diff --git a/config/config.exs b/config/config.exs index 7c831bce..457a7528 100644 --- a/config/config.exs +++ b/config/config.exs @@ -12,8 +12,9 @@ config :opentelemetry, # iex --name a@127.0.0.1 --cookie hyper -S mix # iex --name b@127.0.0.1 --cookie hyper -S mix # -# Swap the strategy for prod (DNSPoll / EC2 tags). -config :libcluster, +# Swap the strategy for prod (DNSPoll / EC2 tags). Hyper forwards this straight +# to libcluster; see `Hyper.Cfg.Cluster`. +config :hyper, Hyper.Cfg.Cluster, topologies: [ hyper: [ strategy: Cluster.Strategy.Epmd, @@ -21,15 +22,10 @@ config :libcluster, ] ] -config :hyper, - cgroup_parent: "hyper", - uid_gid_range: {900_000, 999_999}, - layer_dir: "/srv/hyper/layers" - if config_env() == :test do config :opentelemetry, traces_exporter: :none # No cluster formation during tests. - config :libcluster, topologies: [] + config :hyper, Hyper.Cfg.Cluster, topologies: [] # JUnit XML for Codecov Test Analytics. A fixed report_dir (not the default # app_path) gives CI a stable path; automatic_create_dir? mkdirs it since diff --git a/config/runtime.exs b/config/runtime.exs index a3b4fd0f..6df3f4e9 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -2,29 +2,39 @@ import Config # Per-node resource budget. Lives in runtime config because it builds `Unit.*` # values, which are only loadable once the app's modules are on the code path. -config :hyper, Hyper.Node.Config.Budget, +config :hyper, Hyper.Cfg.Budget, mem_max: Unit.Information.gib(4), disk_max: Unit.Information.gib(4), cpu_max_load: 0.8, + cpu_max_cap: 4.0, disk_bw_cap: Unit.Bandwidth.gibps(1), disk_bw_max_load: 0.8, net_bw_cap: Unit.Bandwidth.gibps(1), net_bw_max_load: 0.8 -# Where to send traces. Defaults to Honeycomb; override OTEL_EXPORTER_OTLP_* -# to point at any OTLP/HTTP backend (Collector, Grafana, etc). +# Operator overrides from a well-known location. An optional Elixir config file +# at /etc/hyper/config.exs (override the path with HYPER_CONFIG) is merged in +# last, so its values win over every default set above. An absent file is a +# no-op — the normal case in dev and CI. Skipped under :test so the suite never +# reads host state. +# +# OpenTelemetry exporter wiring is resolved through Hyper.Cfg.Otel so that the +# operator's `config :hyper, Hyper.Cfg.Otel, ...` stanza (if present) takes +# precedence over the TOML table and environment variables. if config_env() != :test do - endpoint = System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT", "https://api.honeycomb.io") + hyper_config = System.get_env("HYPER_CONFIG") || "/etc/hyper/config.exs" - headers = - case System.get_env("HONEYCOMB_API_KEY") do - nil -> [] - "" -> [] - key -> [{"x-honeycomb-team", key}] - end + operator = + if File.exists?(hyper_config), + do: Config.Reader.read!(hyper_config, env: config_env()), + else: [] - config :opentelemetry_exporter, - otlp_protocol: :http_protobuf, - otlp_endpoint: endpoint, - otlp_headers: headers + otel_exs = get_in(operator, [:hyper, Hyper.Cfg.Otel]) || [] + + case Hyper.Cfg.Otel.exporter_options(otel_exs) do + {:ok, opts} -> config :opentelemetry_exporter, opts + :none -> config :opentelemetry, traces_exporter: :none + end + + for {app, kw} <- operator, do: config(app, kw) end diff --git a/docs/cookbook/config.md b/docs/cookbook/config.md new file mode 100644 index 00000000..5521f5ca --- /dev/null +++ b/docs/cookbook/config.md @@ -0,0 +1,511 @@ +# Configuration + +Configuring `Hyper` is done through four layers, in priority: + +## Configuration Files + +| File | Description | +| ------------------------ | ----------- | +| `/etc/hyper/config.exs` | The `config.exs` file is exclusively used by the unprivileged `hyper` application. The purpose of this file is to allow you to load configuration values at runtime. If you are using a secrets manager, this is the right place to load the secrets. Must be owned by `root` and only writeable by `root`. | +| `/etc/hyper/config.toml` | The `/etc/hyper/config.toml` file is used for static configuration. Unlike `config.exs`, it is used by both `Hyper` and `hyper-suidhelper` which means that it can impact the behavior of a process running under `root`. Must be owned by `root` and only writable by `root`. | +| Compile-Time `config.exs` | The compile-time configuration is generally used to fine-tune the performance of Hyper. You likely do not need to edit most of the configuration fields exposed by this file for day-to-day usage, but they are available for you to tweak. | +| Defaults | `Hyper` has a set of sane defaults for some, but not all config fields. | + +**Note that not all layers allow all configuration fields to be tweaked.** Read +further for more details on where and how each configuration field is set. + +# Configuration Fields + +This section briefly outlines the configuration fields available in `Hyper`. +Note the keys are abbreviated for better layout: + + - `config.exs` refers to `/etc/hyper/config.exs`. + - `config.toml` refers to `/etc/hyper/config.toml`. + - All keys under `config.exs` are written in short-hand form. The parent + group is given as the section title. For example, `.mke2fs` in the tool + configuration section expands to + `:hyper, Hyper.Cfg.Tools, mke2fs: "/path/to/mke2fs"`. + +## Root Keys + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ---------- | ------------ | ------------- | -------------- | ----- | +| `work_dir` | - | `work_dir` | `"/srv/hyper"` | [Absolute Path](#absolute-path) where `Hyper` creates its working tree. | + + +### `config.toml` + +```toml +work_dir = "/srv/hyper" +``` + + +## Tool Configuration + +Hyper relies on a large number of external tools, of which the paths are +configurable: + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ------------- | ------------- | -------------- | ----------------------------------- | ----- | +| `firecracker` | - | `.firecracker` | - | [Safe Path](#safe-path) | +| `jailer` | - | `.jailer` | - | [Safe Path](#safe-path) | +| `dmsetup` | - | `.dmsetup` | `"/usr/sbin/dmsetup"` | [Safe Path](#safe-path) | +| `losetup` | - | `.losetup` | `"/usr/sbin/losetup"` | [Safe Path](#safe-path) | +| `blockdev` | - | `.blockdev` | `"/usr/sbin/blockdev"` | [Safe Path](#safe-path) | +| `mke2fs` | `.mke2fs` | `.mke2fs` | `$PATH["mke2fs"]` | | +| `skopeo` | `.skopeo` | `.skopeo` | `$PATH["skopeo"]` | | +| `umoci` | `.umoci` | `.umoci` | Automatically downloaded. | | +| `suidhelper` | `.suidhelper` | `.suidhelper` | `"/usr/local/bin/hyper-suidhelper"` | [Absolute Path](#absolute-path) | + + +### `config.exs` + +```elixir +# Note that not all the tools are available here. Privileged tools, used by the +# suidhelper, cannot be configured here. +config :hyper, Hyper.Cfg.Tools, + skopeo: "/usr/local/bin/skopeo", + mke2fs: "/usr/local/bin/mke2fs", + umoci: "/usr/local/bin/umoci", + suidhelper: "/usr/local/bin/hyper-suidhelper" +``` + +### `config.toml` + +```toml +[tools] +# Privileged tools -- config.toml only (shared with the suidhelper). +firecracker = "/opt/firecracker/firecracker" +jailer = "/opt/firecracker/jailer" +dmsetup = "/usr/sbin/dmsetup" +losetup = "/usr/sbin/losetup" +blockdev = "/usr/sbin/blockdev" + +# Node tools -- may also be set in config.exs, which wins when both are set. +skopeo = "/usr/local/bin/skopeo" +mke2fs = "/usr/local/bin/mke2fs" +umoci = "/usr/local/bin/umoci" +suidhelper = "/usr/local/bin/hyper-suidhelper" +``` + + +## Jail Confinement + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| --------------- | ------------ | ---------------- | --------- | ----- | +| `cgroup` | - | `.cgroup` | `"hyper"` | Parent cgroup under which each VM's cgroup is nested. Each VM receives its own ephemeral cgroup which lives under the umbrella of this cgroup. | +| `uid_gid_range` | - | `.uid_gid_range` | - | **Required.** [Range](#range) limiting the UID/GID values given to VMs. Each VM receives its own UID/GID pair, within these bounds. Must not be an existing user/group. | + + +### `config.toml` + +```toml +[jails] +cgroup = "hyper" +uid_gid_range = [900000, 999999] +``` + + +## gRPC Configuration + +Hyper supports a [gRPC](https://grpc.io/) interface enabling you to interface +with `Hyper` from any language. + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| -------------- | --------------- | --------------- | ------- | ----- | +| `enabled` | `.enabled` | `.enabled` | `false` | | +| `port` | `.port` | `.port` | `50051` | The port on which to serve the interface. | +| `cred` | `.cred` | `.cred` | `nil` | Either a `GRPC.Credential` or a TOML struct `{ cert = "/path/to/cert.pem", key = "/path/to/key.pem"}`. Cleartext mode when `nil`. | +| `adapter_opts` | `.adapter_opts` | `.adapter_opts` | `[]` | Options forwarded to the gRPC server adapter, e.g. the bind address `ip: {0, 0, 0, 0}`. | + +> #### Uniqueness {: .info} +> +> Note that if you choose to use a homogenous configuration across all your +> nodes and you enable the gRPC server on all of them, you will spawn multiple +> gRPC servers, one-per-node, in your cluster. +> +> This is perfectly legal, if you so desire, but it is important to note that +> you can also conditionally enable the `gRPC` server based on logic in your +> `config.exs`, for example, to only spawn it on your "main" server. + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Grpc, + enabled: true, + port: 50_051, + cred: + GRPC.Credential.new( + ssl: [certfile: "/etc/hyper/tls/cert.pem", keyfile: "/etc/hyper/tls/key.pem"] + ) +``` + +### `config.toml` + +```toml +[grpc] +enabled = true +port = 50051 +cred = { cert = "/etc/hyper/tls/cert.pem", key = "/etc/hyper/tls/key.pem" } +``` + + +## Telemetry Configuration + +You can configure telemetry with Hyper by adding this section to your +configuration and Hyper will emit tracing spans as configured. + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ---------- | ------------ | ------------- | ------- | ----- | +| `proto` | `.proto` | `.proto` | `http_protobuf` | One of `http_protobuf` or `grpc`. | +| `endpoint` | `.endpoint` | `.endpoint` | - | OTLP collector URL. Falls back to the standard `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable; telemetry is disabled when none is set. | +| `headers` | `.headers` | `.headers` | - | Headers sent with each export, e.g. an auth token for your backend. | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Otel, + proto: :http_protobuf, + endpoint: "https://otel-collector.internal:4318", + headers: %{"authorization" => "Bearer YOUR_TOKEN"} +``` + +### `config.toml` + +```toml +[otel] +proto = "http_protobuf" +endpoint = "https://otel-collector.internal:4318" +headers = { "authorization" = "Bearer YOUR_TOKEN" } +``` + + +## Budget Configuration + +Hyper allows you to control the absolute maximal budgets that are available to +all VMs on a particular node. Every field is required except `cpu_max_cap`. + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ------------------ | ------------------- | ------------------- | ------- | ----- | +| `mem_max` | `.mem_max` | `.mem_max` | - | [$\alpha$ (hard) budget](./architecture.md#budgets): total [unit](#unit) of RAM the node offers VMs. Must not exceed available system memory. | +| `disk_max` | `.disk_max` | `.disk_max` | - | [$\alpha$ (hard) budget](./architecture.md#budgets): total [unit](#unit) of disk the node offers VMs. Must not exceed available system disk. | +| `cpu_max_load` | `.cpu_max_load` | `.cpu_max_load` | - | [$\beta$ (soft) budget](./architecture.md#budgets): instantaneous CPU load threshold, a fraction `0.0`–`1.0` of the cap. | +| `cpu_max_cap` | `.cpu_max_cap` | `.cpu_max_cap` | `nil` | [$\beta$ (soft) budget](./architecture.md#budgets): soft CPU capacity in cores (a float). Optional. | +| `disk_bw_cap` | `.disk_bw_cap` | `.disk_bw_cap` | - | [$\beta$ (soft) budget](./architecture.md#budgets): soft disk-bandwidth capacity ([unit](#unit)). | +| `disk_bw_max_load` | `.disk_bw_max_load` | `.disk_bw_max_load` | - | [$\beta$ (soft) budget](./architecture.md#budgets): disk-bandwidth load threshold, a fraction `0.0`–`1.0` of the cap. | +| `net_bw_cap` | `.net_bw_cap` | `.net_bw_cap` | - | [$\beta$ (soft) budget](./architecture.md#budgets): soft network-bandwidth capacity ([unit](#unit)). | +| `net_bw_max_load` | `.net_bw_max_load` | `.net_bw_max_load` | - | [$\beta$ (soft) budget](./architecture.md#budgets): network-bandwidth load threshold, a fraction `0.0`–`1.0` of the cap. | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Budget, + mem_max: Unit.Information.gib(4), + disk_max: Unit.Information.gib(64), + cpu_max_load: 0.8, + cpu_max_cap: 4.0, + disk_bw_cap: Unit.Bandwidth.gibps(1), + disk_bw_max_load: 0.8, + net_bw_cap: Unit.Bandwidth.gibps(1), + net_bw_max_load: 0.8 +``` + +### `config.toml` + +```toml +[budget] +mem_max = "4GiB" +disk_max = "64GiB" +cpu_max_load = 0.8 +cpu_max_cap = 4.0 +disk_bw_cap = "1GiBps" +disk_bw_max_load = 0.8 +net_bw_cap = "1GiBps" +net_bw_max_load = 0.8 +``` + + +## VmLinux Paths + +Hyper requires Linux images for the architectures it runs on: + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ---------- | ------------ | ------------- | -------------------------------------------------------------------------------------------- | ----- | +| `amd64` | `.amd64` | `.amd64` | Automatically downloaded from [hyper-vmlinux](https://github.com/harmont-dev/hyper-vmlinux). | [Absolute Path](#absolute-path). | +| `aarch64` | `.aarch64` | `.aarch64` | Automatically downloaded from [hyper-vmlinux](https://github.com/harmont-dev/hyper-vmlinux). | [Absolute Path](#absolute-path). | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.VmLinux, + amd64: "/opt/hyper/kernels/vmlinux-amd64", + aarch64: "/opt/hyper/kernels/vmlinux-aarch64" +``` + +### `config.toml` + +```toml +[vmlinux] +amd64 = "/opt/hyper/kernels/vmlinux-amd64" +aarch64 = "/opt/hyper/kernels/vmlinux-aarch64" +``` + + +## Image Configuration + +Hyper's image provisioning layer stores read-only layers on a configurable +medium. (The device-mapper geometry behind it is fixed at compile time and +rarely needs tweaking.) + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ---------- | ------------ | ------------- | ------------------- | ----- | +| `store` | `.store` | `.store` | `/layers` | [Absolute Path](#absolute-path) to the [layer storage medium](./architecture.md#storage). | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Img, + store: "/mnt/hyper/layers" +``` + +### `config.toml` + +```toml +[img] +store = "/mnt/hyper/layers" +``` + + +Additionally, sub-sections are available. + +### Database Configuration + +These are secrets, so they are `config.exs`-only. Any key you set overrides the +image-database repo's compiled-in connection config; keys you omit keep that +built-in value. + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ---------- | ------------ | ------------- | ------- | ----- | +| `database` | `.database` | - | - | Postgres database name. | +| `username` | `.username` | - | - | Postgres role. | +| `password` | `.password` | - | - | Postgres password — load it from a secrets manager. | +| `hostname` | `.hostname` | - | - | Postgres host. | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Img.Db, + database: "hyper", + username: "hyper", + password: System.fetch_env!("HYPER_DB_PASSWORD"), + hostname: "db.internal" +``` + + +### Garbage Collector Configuration + +Hyper supports a mechanism to prune unreferenced image layers. Unreferenced +image layers occur when an ungraceful crash happens, resulting in entries in +the layer medium which are not referenced by the database, and, consequently, +unusable. This is always enabled. Since this scans through the whole layer +database, it can have an impact on performance, and tweaking it may be +necessary. + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ------------------ | ------------------- | ------------------- | --------- | ----- | +| `batch_size` | `.batch_size` | `.batch_size` | `200` | | +| `batch_pause` | `.batch_pause` | `.batch_pause` | `100ms` | | +| `sweep_interval` | `.sweep_interval` | `.sweep_interval` | `60s` | | +| `acquire_interval` | `.acquire_interval` | `.acquire_interval` | `5s` | | +| `retry` | `.retry` | `.retry` | `60s` | | +| `timeout` | `.timeout` | `.timeout` | `5s` | | +| `grace_period` | `.grace_period` | `.grace_period` | `1h` | | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Img.Gc, + batch_size: 200, + batch_pause: Unit.Time.ms(100), + sweep_interval: Unit.Time.s(60), + acquire_interval: Unit.Time.s(5), + retry: Unit.Time.s(60), + timeout: Unit.Time.s(5), + grace_period: Unit.Time.s(3600) +``` + +### `config.toml` + +```toml +[img.gc] +batch_size = 200 +batch_pause = "100ms" +sweep_interval = "60s" +acquire_interval = "5s" +retry = "60s" +timeout = "5s" +grace_period = "1h" +``` + + +## Cluster Topology + +Hyper forms a BEAM cluster (Distributed Erlang) so nodes discover each other and +share VM routing and budget state. The topology is given in +[libcluster](https://github.com/bitwalker/libcluster) syntax and forwarded +directly to it. Because a topology references strategy *modules*, it is +`config.exs`-only. Omit it — the default — to run a single, unclustered node. + +| Config Key | `config.exs` | `config.toml` | Default | Notes | +| ------------ | ------------- | ------------- | ------- | ----- | +| `topologies` | `.topologies` | - | `[]` | A [libcluster topology](https://hexdocs.pm/libcluster) keyword list. `[]` is single-node. | + + +### `config.exs` + +```elixir +config :hyper, Hyper.Cfg.Cluster, + topologies: [ + hyper: [ + strategy: Cluster.Strategy.DNSPoll, + config: [query: "hyper.svc.cluster.local", node_basename: "hyper"] + ] + ] +``` + +### `config.toml` + +```toml +# Cluster topologies reference strategy modules, so they are set in config.exs only. +``` + + +## Key Types + +#### Absolute Path + + - Typed as a string in TOML and elixir. + - Must be given as an absolute path. + +#### Safe Path + + - Is an [Absolute Path](#absolute-path). + - The basename **must** match the configuration, eg. `firecracker` must have + a path `/foo/bar/firecracker`. + - Must be owned by `root`. + - Must be only writable by `root`. + - Must not be a symlink. + +#### Range + + - Typed as a 2-tuple in Elixir + - Typed as an array of two elements in TOML. + - `[min, max]` semantics. + +#### Unit + + - Represents a unit as defined in `Unit.*`. + +## Total Examples + +A complete `config.exs` and `config.toml` exercising every section: + + +### `config.exs` + +```elixir +import Config + +config :hyper, Hyper.Cfg.Cluster, + topologies: [ + hyper: [ + strategy: Cluster.Strategy.DNSPoll, + config: [query: "hyper.svc.cluster.local", node_basename: "hyper"] + ] + ] + +# Node-only tools (config.exs wins over config.toml for these). +config :hyper, Hyper.Cfg.Tools, + skopeo: "/usr/local/bin/skopeo", + mke2fs: "/usr/local/bin/mke2fs" + +config :hyper, Hyper.Cfg.Grpc, + enabled: true, + port: 50_051 + +config :hyper, Hyper.Cfg.Otel, + proto: :http_protobuf, + endpoint: "https://otel-collector.internal:4318", + headers: %{"authorization" => System.fetch_env!("OTEL_AUTH_TOKEN")} + +config :hyper, Hyper.Cfg.Budget, + mem_max: Unit.Information.gib(4), + disk_max: Unit.Information.gib(64), + cpu_max_load: 0.8, + cpu_max_cap: 4.0, + disk_bw_cap: Unit.Bandwidth.gibps(1), + disk_bw_max_load: 0.8, + net_bw_cap: Unit.Bandwidth.gibps(1), + net_bw_max_load: 0.8 + +config :hyper, Hyper.Cfg.Img, + store: "/mnt/hyper/layers" + +config :hyper, Hyper.Cfg.Img.Db, + database: "hyper", + username: "hyper", + password: System.fetch_env!("HYPER_DB_PASSWORD"), + hostname: "db.internal" +``` + +### `config.toml` + +```toml +work_dir = "/srv/hyper" + +[tools] +firecracker = "/opt/firecracker/firecracker" +jailer = "/opt/firecracker/jailer" + +[jails] +cgroup = "hyper" +uid_gid_range = [900000, 999999] + +[grpc] +enabled = true +port = 50051 + +[otel] +proto = "http_protobuf" +endpoint = "https://otel-collector.internal:4318" + +[budget] +mem_max = "4GiB" +disk_max = "64GiB" +cpu_max_load = 0.8 +cpu_max_cap = 4.0 +disk_bw_cap = "1GiBps" +disk_bw_max_load = 0.8 +net_bw_cap = "1GiBps" +net_bw_max_load = 0.8 + +[vmlinux] +amd64 = "/opt/hyper/kernels/vmlinux-amd64" +aarch64 = "/opt/hyper/kernels/vmlinux-aarch64" + +[img] +store = "/mnt/hyper/layers" + +[img.gc] +batch_size = 200 +sweep_interval = "60s" +grace_period = "1h" +``` + diff --git a/docs/cookbook/install.md b/docs/cookbook/install.md new file mode 100644 index 00000000..8b22c877 --- /dev/null +++ b/docs/cookbook/install.md @@ -0,0 +1,248 @@ +# Quick Start + +This document provides the quickest start available to get Hyper running. + +## Configuration + +Before you can use `Hyper`, you must do a large amount of configuration. The +following guide must be applied on all nodes you run `Hyper` on. + +Before proceeding, ensure you meet all of these hard requirements: + +| Requirement | Test | +|-------------|------| +| [KVM](https://linux-kvm.org/page/Main_Page) available | `stat /dev/kvm` returns zero. | +| You have root access through `sudo`. | - | +| Your machine has cgroups V2 | `stat -fc %T /sys/fs/cgroup` returns zero. | + +### OS Packages + + + +### Ubuntu + +You can install the required packages by running: + +```sh +sudo apt update && sudo apt install -y \ + coreutils \ + e2fsprogs \ + libc-bin \ + linux-modules-extra-$(uname -r) \ + lvm2 \ + skopeo \ + util-linux +``` + +### Rocky + +You can install the required packages by running: + +```sh +sudo dnf install -y \ + coreutils \ + e2fsprogs \ + glibc-common \ + kernel-modules-extra-$(uname -r) \ + lvm2 \ + skopeo \ + util-linux +``` + +> #### Untested {: .warning} +> +> Rocky has not been tested, but should work. + + + +### Device Mapper Config + +Hyper relies on `dm-snapshot` and `dm-thin` to build COW filesystems. Load the +modules and confirm the targets are present: + +```sh +sudo modprobe dm_snapshot dm_thin_pool loop +sudo dmsetup targets # must list snapshot, thin, and thin-pool +``` + +> #### Persistent Config {: .warning} +> +> Loading modules via `modprobe` is ephemeral and will be reset on next boot. +> To make your config persistent: +> +> ```sh +> printf 'dm_snapshot\ndm_thin_pool\nloop\n' \ +> | sudo tee /etc/modules-load.d/hyper.conf +> ``` + +### PostgreSQL + +Hyper needs a **PostgreSQL** server reachable from every node - it is the image +database and the only stateful external dependency. + +For local development the quickest path is Docker. The connection details below +match the defaults in `config/config.exs` (`Hyper.Img.Db.Repo`): + +```sh +docker run -d --name hyper-pg \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=hyper_dev \ + -p 5432:5432 \ + postgres:16 +``` + +> #### Persistence {: .warning} +> +> Note that the example container should not be used in production -- it will +> be deleted on boot. +> +> We highly suggest you get a managed PostgresSQL instance. The following +> commonly used options are available: +> +> - [AWS RDS](https://aws.amazon.com/rds/postgresql/) if you're in the AWS +> ecosystem. +> - [GCP CloudSQL](https://cloud.google.com/sql) if you're in the GCP +> ecosystem. +> +> The author uses GCP. + +### Configuration + +It is mandatory that you create an `/etc/hyper/config.toml` file on every node. +A reasonable starting point is: + +```toml +# The working directory for hyper. Hyper will create a directory tree in this +# directory and running images, sockets and scratch space will be created in +# this directory. We **strongly** encourage this be mounted on an NVMe drive. +work_dir = "/srv/hyper" + +# Paths to every external binary hyper uses. All paths must be absolute. +# +# The privileged binaries the setuid helper runs (firecracker, jailer, dmsetup, +# losetup, blockdev) must be root-owned and not group/world writable -- the +# helper refuses them otherwise. The node-run tools (skopeo, umoci, mke2fs) have +# no such requirement. +[tools] +# **required**. basename **must** be 'firecracker'. +firecracker = "/opt/firecracker/firecracker" + +# **required**. basename **must** be 'jailer'. +jailer = "/opt/firecracker/jailer" + +# optional -- privileged device tools, default to /usr/sbin/. +# dmsetup = "/usr/sbin/dmsetup" +# losetup = "/usr/sbin/losetup" +# blockdev = "/usr/sbin/blockdev" + +# optional -- node-run tools. skopeo/mke2fs default to the name on PATH; omit +# umoci to let hyper download and cache a pinned release. +# skopeo = "skopeo" +# mke2fs = "mke2fs" +# umoci = "/usr/bin/umoci" +# suidhelper = "/usr/local/bin/hyper-suidhelper" + +[jails] +# The valid range of user/group IDs in which new VMs will be spawned. Hyper +# will create new VM jails for each VM within the given range. +uid_gid_range = [900000, 999999] +# optional +cgroup = "hyper" +``` + +> #### Security {: .error} +> +> This file **must** be owned by `root`, not group and not world writable. +> `Hyper` will refuse to boot otherwise. + +For more details on configuring and tuning Hyper, we suggest you see the +[configuration guide](config.md). + +### Cgroups + +Hyper uses cgroups to impose limits on each VM. Each VM has its own cgroup, +which is spawned ephemerally, for the lifetime of the VM. These cgroups are all +managed by a parent cgroup which you must create. You can name this cgroup +whatever you like, as long as it matches the `jails.cgroup` value in the +`/etc/hyper/config.toml`: + +```sh +sudo mkdir -p /sys/fs/cgroup/hyper +``` + +You must allow permissions on `cpu` and `memory` control on the subtree: + +```sh +echo '+cpu +memory' | sudo tee /sys/fs/cgroup/hyper/cgroup.subtree_control +``` + +> #### Security {: .error} +> +> Note that Hyper does not manage the `cgroup` with its user -- it rather +> delegates to `hyper-suidhelper`, which is why `/sys/fs/cgroup/hyper` should +> be `root:root` owned. + +> #### Persistence {: .warning} +> +> The configuration, as given, will not survive reboots. To persist it, you can +> use `systemd-tempfiles`: +> +> ```sh +> echo 'd /sys/fs/cgroup/hyper 0755 root root -' \ +> | sudo tee /etc/tmpfiles.d/hyper-cgroup.conf +> ``` + +### User Configuration + +Hyper must **not** run as `root`, and you should not run it as your login user +either. Instead, give it a dedicated, unprivileged system user. The BEAM runs +as this user; every operation that genuinely needs root is routed through the +setuid helper (see [SUID Helper](#suid-helper)), so the node itself never holds +privilege. + +Create the user — system account, no login shell: + +```sh +sudo useradd --system --shell /usr/sbin/nologin --home-dir /srv/hyper hyper +``` + +Start Hyper as this user (for example `sudo -u hyper ...`, or `User=hyper` in a +systemd unit). The rest of this section covers the few permissions it needs — +and the ones it deliberately does **not**. + +#### Working directory + +The node builds its entire on-disk tree (`jails`, `socks`, `scratch`, `layers`, +`redist`) under `work_dir` (from `/etc/hyper/config.toml`, default `/srv/hyper`) +**as this user**. It must therefore own that directory: + +```sh +sudo mkdir -p /srv/hyper +sudo chown hyper:hyper /srv/hyper +``` + +## Installation + +### SUID Helper + +Hyper does not run as `root`. Running Hyper as root is considered unsafe and an +anti-pattern. Unfortunately, Hyper needs root for certain classes of system +operations. This is achieved through a side-car binary called +`hyper-setuidhelper`, which you must install manually. + +> #### Versioning {: .warning} +> +> The `hyper-setuidhelper` binary is versioned together with the version of +> `Hyper`, meaning that mismatched versions between the `hyper-setuidhelper` +> and `Hyper` itself will not work and Hyper will fail to boot. + +Installing this binary can be done by downloading it from the [Github Releases +page](https://github.com/harmont-dev/hyper/releases) and executing: + +```sh +sudo install -o root -g root -m 4755 \ + path/to/downloaded/hyper-suidehelper \ + /usr/local/bin/hyper-suidhelper +``` + diff --git a/lib/hyper/application.ex b/lib/hyper/application.ex index 96d28b3e..4a0a280b 100644 --- a/lib/hyper/application.ex +++ b/lib/hyper/application.ex @@ -12,7 +12,7 @@ defmodule Hyper.Application do # the repo's default telemetry_prefix (its module path, underscored). _ = OpentelemetryEcto.setup([:hyper, :img, :db, :repo]) - topologies = Application.get_env(:libcluster, :topologies, []) + topologies = Hyper.Cfg.Cluster.topologies() children = [ diff --git a/lib/hyper/cfg.ex b/lib/hyper/cfg.ex new file mode 100644 index 00000000..9e785834 --- /dev/null +++ b/lib/hyper/cfg.ex @@ -0,0 +1,85 @@ +defmodule Hyper.Cfg do + @moduledoc """ + The one place every Hyper configuration value is read. + + Configuration is layered, highest priority first: + + 1. `/etc/hyper/config.exs` — runtime app env, unprivileged node only. The + right place for secrets loaded at boot. + 2. `/etc/hyper/config.toml` — static, **shared with `hyper-suidhelper`**. + Anything that influences a root process lives here so the two sides can + never drift. + 3. Compile-time `config/config.exs` — performance fine-tuning. + 4. Built-in defaults. + + Each value names *which* of these layers it may be read from. Privileged tool + paths and the helper-shared `[jails]` table are `config.toml`-only, so the + unprivileged `config.exs` can never override a root-impacting path. + + Read values through the focused submodules — `Hyper.Cfg.Tools`, + `Hyper.Cfg.Dirs`, `Hyper.Cfg.Jails`, `Hyper.Cfg.Budget`, ... — never reach for + `Application.get_env` or `Hyper.Cfg.Toml` directly. + """ + + defmodule MissingError do + @moduledoc "Raised when a required config value is absent from every permitted source." + defexception [:message] + end + + @type source :: + {:exs, {keyword(), atom()}} + | {:runtime, atom() | {module(), atom()}} + | {:toml, String.t()} + | {:default, term()} + + @doc false + @spec fetch_cfg([source]) :: {:ok, term()} | :error + def fetch_cfg(sources) when is_list(sources), do: resolve(sources) + + @doc false + @spec get_cfg([source]) :: term() + def get_cfg(sources) when is_list(sources) do + case fetch_cfg(sources) do + {:ok, value} -> + value + + :error -> + raise MissingError, + message: + "required config value is not set in any permitted source: " <> + inspect(Keyword.delete(sources, :default)) + end + end + + @spec resolve([source]) :: {:ok, term()} | :error + defp resolve([]), do: :error + + defp resolve([source | rest]) do + case from(source) do + {:ok, value} -> {:ok, value} + :error -> resolve(rest) + end + end + + @spec from(source) :: {:ok, term()} | :error + defp from({:default, value}), do: {:ok, value} + defp from({:toml, path}), do: Hyper.Cfg.Toml.fetch(path) + + # An explicit keyword list, used when config.exs is read by hand during + # `config/runtime.exs` boot (before it reaches app env) — see `Hyper.Cfg.Otel`. + defp from({:exs, {kw, key}}) when is_list(kw), do: Keyword.fetch(kw, key) + + defp from({:runtime, {mod, key}}) do + case Application.get_env(:hyper, mod) do + kw when is_list(kw) -> Keyword.fetch(kw, key) + _ -> :error + end + end + + defp from({:runtime, key}) when is_atom(key) do + case Application.get_env(:hyper, key) do + nil -> :error + value -> {:ok, value} + end + end +end diff --git a/lib/hyper/cfg/budget.ex b/lib/hyper/cfg/budget.ex new file mode 100644 index 00000000..2d01799e --- /dev/null +++ b/lib/hyper/cfg/budget.ex @@ -0,0 +1,112 @@ +defmodule Hyper.Cfg.Budget do + @moduledoc """ + This node's resource budget. Each field reads from `config.exs` + (`config :hyper, Hyper.Cfg.Budget, ...`), then the `[budget]` table in + `/etc/hyper/config.toml`, then its default. `Unit.*` quantities may be given + as Elixir terms in `config.exs` or as strings (`"4GiB"`, `"1GiBps"`) in TOML. + """ + + import Hyper.Cfg, only: [fetch_cfg: 1] + + @type t :: %__MODULE__{ + mem_max: Unit.Information.t(), + disk_max: Unit.Information.t(), + cpu_max_load: float(), + cpu_max_cap: float() | nil, + disk_bw_cap: Unit.Bandwidth.t(), + disk_bw_max_load: float(), + net_bw_cap: Unit.Bandwidth.t(), + net_bw_max_load: float() + } + defstruct [ + :mem_max, + :disk_max, + :cpu_max_load, + :cpu_max_cap, + :disk_bw_cap, + :disk_bw_max_load, + :net_bw_cap, + :net_bw_max_load + ] + + @spec load :: {:ok, t()} | {:error, term()} + def load do + with {:ok, mem_max} <- information(:mem_max, "budget.mem_max"), + {:ok, disk_max} <- information(:disk_max, "budget.disk_max"), + {:ok, cpu_max_load} <- number(:cpu_max_load, "budget.cpu_max_load"), + {:ok, disk_bw_cap} <- bandwidth(:disk_bw_cap, "budget.disk_bw_cap"), + {:ok, disk_bw_max_load} <- number(:disk_bw_max_load, "budget.disk_bw_max_load"), + {:ok, net_bw_cap} <- bandwidth(:net_bw_cap, "budget.net_bw_cap"), + {:ok, net_bw_max_load} <- number(:net_bw_max_load, "budget.net_bw_max_load") do + config = %__MODULE__{ + mem_max: mem_max, + disk_max: disk_max, + cpu_max_load: cpu_max_load, + cpu_max_cap: optional_number(:cpu_max_cap, "budget.cpu_max_cap"), + disk_bw_cap: disk_bw_cap, + disk_bw_max_load: disk_bw_max_load, + net_bw_cap: net_bw_cap, + net_bw_max_load: net_bw_max_load + } + + :persistent_term.put(__MODULE__, config) + {:ok, config} + end + end + + @spec get :: t() + def get, do: :persistent_term.get(__MODULE__) + + @spec information(atom(), String.t()) :: {:ok, Unit.Information.t()} | {:error, term()} + defp information(key, toml) do + with {:ok, v} <- required(key, toml) do + coerce(v, &Unit.Information.parse/1, Unit.Information, key) + end + end + + @spec bandwidth(atom(), String.t()) :: {:ok, Unit.Bandwidth.t()} | {:error, term()} + defp bandwidth(key, toml) do + with {:ok, v} <- required(key, toml) do + coerce(v, &Unit.Bandwidth.parse/1, Unit.Bandwidth, key) + end + end + + @spec number(atom(), String.t()) :: {:ok, number()} | {:error, term()} + defp number(key, toml) do + case required(key, toml) do + {:ok, n} when is_number(n) -> {:ok, n} + {:ok, other} -> {:error, {:not_a_number, key, other}} + {:error, _} = e -> e + end + end + + @spec optional_number(atom(), String.t()) :: number() | nil + defp optional_number(key, toml) do + case fetch_cfg(runtime: {__MODULE__, key}, toml: toml) do + {:ok, n} when is_number(n) -> n + _ -> nil + end + end + + @spec required(atom(), String.t()) :: {:ok, term()} | {:error, term()} + defp required(key, toml) do + case fetch_cfg(runtime: {__MODULE__, key}, toml: toml) do + {:ok, v} -> {:ok, v} + :error -> {:error, {:missing, key}} + end + end + + # Accept an already-typed struct (from config.exs) or a string to parse (from TOML). + @spec coerce(term(), (String.t() -> {:ok, struct()} | {:error, term()}), module(), atom()) :: + {:ok, struct()} | {:error, term()} + defp coerce(%mod{} = v, _parse, mod, _key), do: {:ok, v} + + defp coerce(s, parse, _mod, key) when is_binary(s) do + case parse.(s) do + {:ok, v} -> {:ok, v} + {:error, _} -> {:error, {:bad_value, key, s}} + end + end + + defp coerce(other, _parse, _mod, key), do: {:error, {:bad_value, key, other}} +end diff --git a/lib/hyper/cfg/cluster.ex b/lib/hyper/cfg/cluster.ex new file mode 100644 index 00000000..523c011b --- /dev/null +++ b/lib/hyper/cfg/cluster.ex @@ -0,0 +1,17 @@ +defmodule Hyper.Cfg.Cluster do + @moduledoc """ + BEAM cluster (Distributed Erlang) topology for Hyper. Set it in `config.exs` + via `config :hyper, Hyper.Cfg.Cluster, topologies: [...]` using + [libcluster](https://github.com/bitwalker/libcluster) topology syntax; + `Hyper.Application` forwards it straight to `Cluster.Supervisor`, which is what + Horde's `members: :auto` registries form over. `config.exs`-only because a + topology references strategy modules. The default — `[]` — is a single, + unclustered node. + """ + + import Hyper.Cfg, only: [get_cfg: 1] + + @doc "The libcluster topologies to form the BEAM cluster with." + @spec topologies :: keyword() + def topologies, do: get_cfg(runtime: {__MODULE__, :topologies}, default: []) +end diff --git a/lib/hyper/cfg/dirs.ex b/lib/hyper/cfg/dirs.ex new file mode 100644 index 00000000..bb3190c0 --- /dev/null +++ b/lib/hyper/cfg/dirs.ex @@ -0,0 +1,47 @@ +defmodule Hyper.Cfg.Dirs do + @moduledoc """ + The node's work directory and every directory derived from it. + + `work_dir` is the single configurable root (`config.toml`-only, shared with the + setuid helper); everything else is a fixed sub-path so the node and helper + agree on layout without a second key to keep in sync. + """ + + import Hyper.Cfg, only: [get_cfg: 1] + + @doc "Root work directory for this node. config.toml `work_dir`, default `/srv/hyper`." + @spec work_dir :: Path.t() + def work_dir, do: get_cfg(toml: "work_dir", default: "/srv/hyper") + + @doc "Read-only image layer store. Delegates to `Hyper.Cfg.Img.store/0`." + @spec layer_dir :: Path.t() + def layer_dir, do: Hyper.Cfg.Img.store() + + @doc "Per-VM control/gRPC sockets (`/socks`)." + @spec socket_dir :: Path.t() + def socket_dir, do: Path.join(work_dir(), "socks") + + @doc "Per-VM copy-on-write writable layers (`/scratch`)." + @spec scratch_dir :: Path.t() + def scratch_dir, do: Path.join(work_dir(), "scratch") + + @doc "Per-VM chroot directories (`/jails`)." + @spec chroot_base :: Path.t() + def chroot_base, do: Path.join(work_dir(), "jails") + + @doc "Node-downloaded binaries (`/redist`)." + @spec redist_dir :: Path.t() + def redist_dir, do: Path.join(work_dir(), "redist") + + @doc "Where guest kernels install (`/redist/vmlinux`)." + @spec vmlinux_install_dir :: Path.t() + def vmlinux_install_dir, do: Path.join(redist_dir(), "vmlinux") + + @doc "Where the default umoci installs (`/redist/umoci`)." + @spec umoci_install_dir :: Path.t() + def umoci_install_dir, do: Path.join(redist_dir(), "umoci") + + @doc "Where a node-downloaded firecracker installs (`/redist/firecracker`)." + @spec firecracker_install_dir :: Path.t() + def firecracker_install_dir, do: Path.join(redist_dir(), "firecracker") +end diff --git a/lib/hyper/cfg/gc.ex b/lib/hyper/cfg/gc.ex new file mode 100644 index 00000000..db74d372 --- /dev/null +++ b/lib/hyper/cfg/gc.ex @@ -0,0 +1,50 @@ +defmodule Hyper.Cfg.Gc do + @moduledoc """ + Layer garbage collector tuning. Each field reads from `config.exs` + (`config :hyper, Hyper.Cfg.Gc, ...`), then the `[img.gc]` table, then its + default. Durations are `Unit.Time` — Elixir terms in `config.exs`, strings + (`"60s"`, `"1h"`) in TOML. + """ + + import Hyper.Cfg, only: [get_cfg: 1] + + @type t :: %__MODULE__{ + batch_size: pos_integer(), + batch_pause: Unit.Time.t(), + sweep_interval: Unit.Time.t(), + acquire_interval: Unit.Time.t(), + retry: Unit.Time.t(), + timeout: Unit.Time.t(), + grace_period: Unit.Time.t() + } + defstruct [ + :batch_size, + :batch_pause, + :sweep_interval, + :acquire_interval, + :retry, + :timeout, + :grace_period + ] + + @spec load :: t() + def load do + struct!(__MODULE__, [ + {:batch_size, + get_cfg(runtime: {__MODULE__, :batch_size}, toml: "img.gc.batch_size", default: 200)}, + {:batch_pause, duration(:batch_pause, "img.gc.batch_pause", Unit.Time.ms(100))}, + {:sweep_interval, duration(:sweep_interval, "img.gc.sweep_interval", Unit.Time.s(60))}, + {:acquire_interval, duration(:acquire_interval, "img.gc.acquire_interval", Unit.Time.s(5))}, + {:retry, duration(:retry, "img.gc.retry", Unit.Time.s(60))}, + {:timeout, duration(:timeout, "img.gc.timeout", Unit.Time.s(5))}, + {:grace_period, duration(:grace_period, "img.gc.grace_period", Unit.Time.s(3600))} + ]) + end + + defp duration(key, toml, default) do + case get_cfg(runtime: {__MODULE__, key}, toml: toml, default: default) do + %_mod{} = t -> t + s when is_binary(s) -> Unit.Time.parse!(s) + end + end +end diff --git a/lib/hyper/cfg/grpc.ex b/lib/hyper/cfg/grpc.ex new file mode 100644 index 00000000..1d2adbec --- /dev/null +++ b/lib/hyper/cfg/grpc.ex @@ -0,0 +1,79 @@ +defmodule Hyper.Cfg.Grpc do + @moduledoc """ + gRPC server configuration, read from application env into a struct: + + config :hyper, Hyper.Cfg.Grpc, + enabled: true, + port: 50_051, + cred: GRPC.Credential.new(ssl: [certfile: "/path/cert.pem", keyfile: "/path/key.pem"]) + + Fields: + + * `:enabled` -- whether the server starts. Defaults to `false`. + * `:port` -- the listen port. Defaults to `50051`. + * `:cred` -- a `GRPC.Credential` for TLS, or `nil` (the default) for + plaintext. + * `:adapter_opts` -- options forwarded to the server adapter, e.g. + `[ip: {0, 0, 0, 0}]`. + + Build the credential where you load your keys (e.g. `config/runtime.exs`); + Hyper never reads the filesystem on your behalf. + + Each field also reads from the `[grpc]` table of `config.toml` when not set in + `config.exs`. There, `cred` is given as an inline table + `{ cert = "/path/cert.pem", key = "/path/key.pem" }`, which is coerced into a + `GRPC.Credential`. + + > #### Co-located nodes {: .info} + > + > Every node binds `:port`. Running multiple nodes on one host (e.g. a local + > cluster) requires giving each a distinct port via its own config. + """ + + import Hyper.Cfg, only: [get_cfg: 1] + + @default_port 50_051 + + defstruct enabled: false, port: @default_port, cred: nil, adapter_opts: [] + + @type t :: %__MODULE__{ + enabled: boolean(), + port: :inet.port_number(), + cred: GRPC.Credential.t() | nil, + adapter_opts: keyword() + } + + @doc "Load the gRPC server configuration: config.exs > [grpc] toml > defaults." + @spec load() :: t() + def load do + %__MODULE__{ + enabled: get_cfg(runtime: {__MODULE__, :enabled}, toml: "grpc.enabled", default: false), + port: get_cfg(runtime: {__MODULE__, :port}, toml: "grpc.port", default: @default_port), + cred: cred(get_cfg(runtime: {__MODULE__, :cred}, toml: "grpc.cred", default: nil)), + adapter_opts: + get_cfg(runtime: {__MODULE__, :adapter_opts}, toml: "grpc.adapter_opts", default: []) + } + end + + @spec cred(term()) :: GRPC.Credential.t() | nil + defp cred(nil), do: nil + defp cred(%GRPC.Credential{} = c), do: c + + defp cred(%{"cert" => cert, "key" => key}), + do: GRPC.Credential.new(ssl: [certfile: cert, keyfile: key]) + + @doc """ + The `GRPC.Server.Supervisor` options for this config: the endpoint and port, + plus the TLS credential and adapter options when set. + """ + @spec server_options(t()) :: keyword() + def server_options(%__MODULE__{} = config) do + [endpoint: Hyper.Grpc.Endpoint, port: config.port, start_server: true] + |> put_unless(:cred, config.cred, nil) + |> put_unless(:adapter_opts, config.adapter_opts, []) + end + + @spec put_unless(keyword(), atom(), term(), term()) :: keyword() + defp put_unless(opts, _key, skip, skip), do: opts + defp put_unless(opts, key, value, _skip), do: Keyword.put(opts, key, value) +end diff --git a/lib/hyper/node/img/config.ex b/lib/hyper/cfg/img.ex similarity index 74% rename from lib/hyper/node/img/config.ex rename to lib/hyper/cfg/img.ex index e0fec815..f1e93b54 100644 --- a/lib/hyper/node/img/config.ex +++ b/lib/hyper/cfg/img.ex @@ -1,4 +1,4 @@ -defmodule Hyper.Node.Config.Img do +defmodule Hyper.Cfg.Img do @moduledoc """ This node's image storage configuration: the device-mapper geometry behind the read-only layer chain (dm-snapshot) and the per-VM writable layers (dm-thin). @@ -7,8 +7,11 @@ defmodule Hyper.Node.Config.Img do * `thin_block_sectors` - dm-thin pool allocation block size. * `thin_pool_data_size` / `thin_pool_meta_size` - sparse sizes of the node's dm-thin pool backing devices. + * `store` - absolute path to the read-only layer store. """ + import Hyper.Cfg, only: [get_cfg: 1] + @chunk_sectors Application.compile_env(:hyper, :chunk_sectors, 8) @thin_block_sectors Application.compile_env(:hyper, :thin_block_sectors, 128) @@ -30,4 +33,17 @@ defmodule Hyper.Node.Config.Img do @doc "Sparse size of the node's dm-thin pool metadata device." @spec thin_pool_meta_size :: Unit.Information.t() def thin_pool_meta_size, do: Unit.Information.gib(1) + + @doc """ + Absolute path to the read-only layer store. config.exs (`store:`) > `[img] store` + toml > `/layers`. + """ + @spec store :: Path.t() + def store, + do: + get_cfg( + runtime: {__MODULE__, :store}, + toml: "img.store", + default: Path.join(Hyper.Cfg.Dirs.work_dir(), "layers") + ) end diff --git a/lib/hyper/cfg/img/db.ex b/lib/hyper/cfg/img/db.ex new file mode 100644 index 00000000..fa4274ef --- /dev/null +++ b/lib/hyper/cfg/img/db.ex @@ -0,0 +1,17 @@ +defmodule Hyper.Cfg.Img.Db do + @moduledoc """ + Image-database (Ecto/Postgres) connection settings — `database`/`username`/ + `password`/`hostname`. These are secrets, so they are read from `config.exs` + only (`config :hyper, Hyper.Cfg.Img.Db, ...`), never the shared `config.toml`. + `Hyper.Img.Db.Repo.init/2` merges these over its compile-time defaults. + """ + + @keys [:database, :username, :password, :hostname] + + @doc "Operator-set repo options (only the keys actually set), for `Repo.init/2`." + @spec repo_opts :: keyword() + def repo_opts do + env = Application.get_env(:hyper, __MODULE__, []) + Enum.filter(env, fn {k, _} -> k in @keys end) + end +end diff --git a/lib/hyper/cfg/jails.ex b/lib/hyper/cfg/jails.ex new file mode 100644 index 00000000..5be5ba2b --- /dev/null +++ b/lib/hyper/cfg/jails.ex @@ -0,0 +1,28 @@ +defmodule Hyper.Cfg.Jails do + @moduledoc """ + VM confinement settings from the `[jails]` table — `config.toml`-only because + the setuid helper enforces the same `uid_gid_range` it reads from this file. + """ + + import Hyper.Cfg, only: [get_cfg: 1] + + @doc "Parent cgroup for every VM cgroup. `[jails] cgroup`, default `\"hyper\"`." + @spec cgroup :: String.t() + def cgroup, do: get_cfg(toml: "jails.cgroup", default: "hyper") + + @doc """ + UID/GID allocation band each VM jail draws from (`[jails] uid_gid_range`, a + required `[min, max]` integer array). Raises `Hyper.Cfg.MissingError` when it + is unset, and `ArgumentError` when it is not a pair of integers — a bogus band + must never silently confine a VM. + """ + @spec uid_gid_range :: {integer(), integer()} + def uid_gid_range, do: range_from(get_cfg(toml: "jails.uid_gid_range")) + + @spec range_from(term()) :: {integer(), integer()} + defp range_from([min, max]) when is_integer(min) and is_integer(max), do: {min, max} + + defp range_from(other) do + raise ArgumentError, "jails.uid_gid_range must be [min, max] integers, got: #{inspect(other)}" + end +end diff --git a/lib/hyper/cfg/otel.ex b/lib/hyper/cfg/otel.ex new file mode 100644 index 00000000..ba084207 --- /dev/null +++ b/lib/hyper/cfg/otel.ex @@ -0,0 +1,51 @@ +defmodule Hyper.Cfg.Otel do + @moduledoc """ + OpenTelemetry exporter configuration. Resolved from `config :hyper, + Hyper.Cfg.Otel, proto:/endpoint:/headers:` (config.exs), the `[otel]` toml + table, then the standard `OTEL_EXPORTER_OTLP_ENDPOINT` env var. + `config/runtime.exs` calls `exporter_options/1` and feeds the result to + `config :opentelemetry_exporter`. + """ + + import Hyper.Cfg, only: [get_cfg: 1] + + @doc "Resolve the `:opentelemetry_exporter` options, or `:none`." + @spec exporter_options(keyword()) :: {:ok, keyword()} | :none + def exporter_options(exs) when is_list(exs) do + endpoint = pick(exs, :endpoint, "otel.endpoint") || env_endpoint() + + case endpoint do + nil -> + :none + + ep -> + {:ok, + [ + otlp_protocol: proto(pick(exs, :proto, "otel.proto")), + otlp_endpoint: ep, + otlp_headers: headers(pick(exs, :headers, "otel.headers")) + ]} + end + end + + # config.exs (the explicit `exs` keyword, since we run during boot) > [otel] toml. + @spec pick(keyword(), atom(), String.t()) :: term() | nil + defp pick(exs, key, toml), do: get_cfg(exs: {exs, key}, toml: toml, default: nil) + + @spec proto(term()) :: :http_protobuf | :grpc + defp proto(p) when p in [:http_protobuf, :grpc], do: p + defp proto("grpc"), do: :grpc + defp proto(_), do: :http_protobuf + + @spec headers(term()) :: [{String.t(), String.t()}] + defp headers(h) when is_map(h), do: Enum.map(h, fn {k, v} -> {to_string(k), to_string(v)} end) + defp headers(h) when is_list(h), do: Enum.map(h, fn {k, v} -> {to_string(k), to_string(v)} end) + defp headers(_), do: [] + + @spec env_endpoint() :: String.t() | nil + defp env_endpoint, do: nonempty(System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT")) + + @spec nonempty(String.t() | nil) :: String.t() | nil + defp nonempty(s) when is_binary(s) and s != "", do: s + defp nonempty(_), do: nil +end diff --git a/lib/hyper/cfg/toml.ex b/lib/hyper/cfg/toml.ex new file mode 100644 index 00000000..b4d20eeb --- /dev/null +++ b/lib/hyper/cfg/toml.ex @@ -0,0 +1,57 @@ +defmodule Hyper.Cfg.Toml do + @moduledoc false + # Internal: the raw read+cache of /etc/hyper/config.toml, the single source of + # truth shared with the setuid helper (native/suidhelper). Read once on first + # access and cached in :persistent_term; an absent file (local dev / CI) yields + # an empty map so the built-in defaults in Hyper.Cfg.* take over and the node + # still agrees with the helper. Only Hyper.Cfg.* may call this module. + + @config_path "/etc/hyper/config.toml" + + @doc "Fetch a dotted key path (e.g. `\"tools.firecracker\"`) from the cached config." + @spec fetch(String.t()) :: {:ok, term()} | :error + def fetch(path), do: fetch_in(config(), path) + + @doc "Pure dotted-path lookup into an already-decoded map (exposed for tests)." + @spec fetch_in(map(), String.t()) :: {:ok, term()} | :error + def fetch_in(map, path) do + Enum.reduce_while(String.split(path, "."), {:ok, map}, fn seg, {:ok, acc} -> + case acc do + %{^seg => v} -> {:cont, {:ok, v}} + _ -> {:halt, :error} + end + end) + end + + @doc "Drop the cache so the next read re-parses the file (test hook)." + @spec reload :: map() + def reload do + :persistent_term.erase({__MODULE__, :config}) + config() + end + + @doc "Seed the cache with a decoded map, bypassing the file (test hook)." + @spec put_cache(map()) :: :ok + def put_cache(cfg), do: :persistent_term.put({__MODULE__, :config}, cfg) + + @spec config :: map() + defp config do + case :persistent_term.get({__MODULE__, :config}, nil) do + nil -> + cfg = load() + :persistent_term.put({__MODULE__, :config}, cfg) + cfg + + cfg -> + cfg + end + end + + @spec load :: map() + defp load do + case File.read(@config_path) do + {:ok, body} -> Toml.decode!(body) + {:error, _} -> %{} + end + end +end diff --git a/lib/hyper/cfg/tools.ex b/lib/hyper/cfg/tools.ex new file mode 100644 index 00000000..3e6593f7 --- /dev/null +++ b/lib/hyper/cfg/tools.ex @@ -0,0 +1,64 @@ +defmodule Hyper.Cfg.Tools do + @moduledoc """ + Paths to every external binary Hyper runs, read from the `[tools]` table. + + The privileged tools (`firecracker`, `jailer`, `dmsetup`, `losetup`, + `blockdev`) are read **only** from `/etc/hyper/config.toml` — the file the + setuid helper also parses, so node and helper can never disagree on a + root-impacting path. The node-only tools (`skopeo`, `mke2fs`, `umoci`, + `suidhelper`) run unprivileged, so `/etc/hyper/config.exs` may override them + (e.g. a path from a secrets manager), then `config.toml`, then the default. + """ + + import Hyper.Cfg, only: [get_cfg: 1] + + @doc "Firecracker binary. Required — raises if `[tools] firecracker` is unset." + @spec firecracker :: Path.t() + def firecracker, do: get_cfg(toml: "tools.firecracker") + + @doc "Non-raising `firecracker/0`." + @spec firecracker_configured :: {:ok, Path.t()} | :error + def firecracker_configured, do: Hyper.Cfg.Toml.fetch("tools.firecracker") + + @doc "Jailer binary. Required — raises if `[tools] jailer` is unset." + @spec jailer :: Path.t() + def jailer, do: get_cfg(toml: "tools.jailer") + + @doc "Non-raising `jailer/0`." + @spec jailer_configured :: {:ok, Path.t()} | :error + def jailer_configured, do: Hyper.Cfg.Toml.fetch("tools.jailer") + + @doc "dmsetup binary (privileged, config.toml-only)." + @spec dmsetup :: Path.t() + def dmsetup, do: get_cfg(toml: "tools.dmsetup", default: "/usr/sbin/dmsetup") + + @doc "losetup binary (privileged, config.toml-only)." + @spec losetup :: Path.t() + def losetup, do: get_cfg(toml: "tools.losetup", default: "/usr/sbin/losetup") + + @doc "blockdev binary (privileged, config.toml-only)." + @spec blockdev :: Path.t() + def blockdev, do: get_cfg(toml: "tools.blockdev", default: "/usr/sbin/blockdev") + + @doc "skopeo binary (node tool). config.exs > config.toml > `skopeo` on PATH." + @spec skopeo :: String.t() + def skopeo, do: get_cfg(runtime: {__MODULE__, :skopeo}, toml: "tools.skopeo", default: "skopeo") + + @doc "mke2fs binary (node tool). config.exs > config.toml > `mke2fs` on PATH." + @spec mke2fs :: String.t() + def mke2fs, do: get_cfg(runtime: {__MODULE__, :mke2fs}, toml: "tools.mke2fs", default: "mke2fs") + + @doc "umoci binary (node tool), or `nil` to let Hyper download a pinned release." + @spec umoci :: String.t() | nil + def umoci, do: get_cfg(runtime: {__MODULE__, :umoci}, toml: "tools.umoci", default: nil) + + @doc "setuid device helper (node tool). config.exs > config.toml > install default." + @spec suidhelper :: String.t() + def suidhelper, + do: + get_cfg( + runtime: {__MODULE__, :suidhelper}, + toml: "tools.suidhelper", + default: "/usr/local/bin/hyper-suidhelper" + ) +end diff --git a/lib/hyper/cfg/vm_linux.ex b/lib/hyper/cfg/vm_linux.ex new file mode 100644 index 00000000..0b3eadbd --- /dev/null +++ b/lib/hyper/cfg/vm_linux.ex @@ -0,0 +1,21 @@ +defmodule Hyper.Cfg.VmLinux do + @moduledoc """ + Per-architecture guest-kernel image paths. Operators set `amd64`/`aarch64` + (mapped to `:x86_64`/`:aarch64`) in `config :hyper, Hyper.Cfg.VmLinux, ...` or + the `[vmlinux]` toml table. An unset architecture is simply absent from the map. + """ + + import Hyper.Cfg, only: [fetch_cfg: 1] + + @archs %{amd64: :x86_64, aarch64: :aarch64} + + @doc "Resolved `%{arch => path}` kernel map (config.exs per key > [vmlinux] toml)." + @spec images :: %{optional(Sys.Arch.t()) => Path.t()} + def images do + for {doc_key, arch} <- @archs, {:ok, path} <- [resolve(doc_key)], into: %{}, do: {arch, path} + end + + @spec resolve(atom()) :: {:ok, Path.t()} | :error + defp resolve(doc_key), + do: fetch_cfg(runtime: {__MODULE__, doc_key}, toml: "vmlinux.#{doc_key}") +end diff --git a/lib/hyper/config.ex b/lib/hyper/config.ex deleted file mode 100644 index 1332d9ee..00000000 --- a/lib/hyper/config.ex +++ /dev/null @@ -1,161 +0,0 @@ -defmodule Hyper.Config do - @moduledoc """ - Host configuration, read from `config :hyper, ...` (see `config/config.exs`). - - `work_dir` is the one value shared with the setuid helper - (`native/suidhelper`); both sides read it from `/etc/hyper/config.toml` so the - data root has a single source of truth (see `work_dir/0`). Everything else is - compile-time. - """ - - # The shared data-root config file, read by both this node and the setuid - # helper. Absent in local dev / CI, where `@dev_work_dir` is used instead. - @config_path "/etc/hyper/config.toml" - @dev_work_dir "/srv/hyper" - - @parent_cgroup Application.compile_env(:hyper, :cgroup_parent, "hyper") - @uid_gid_range Application.compile_env!(:hyper, :uid_gid_range) - @layer_dir Application.compile_env!(:hyper, :layer_dir) - @losetup_path Application.compile_env(:hyper, :losetup_path, "losetup") - @dmsetup_path Application.compile_env(:hyper, :dmsetup_path, "dmsetup") - @blockdev_path Application.compile_env(:hyper, :blockdev_path, "blockdev") - @skopeo_path Application.compile_env(:hyper, :skopeo_path, "skopeo") - @umoci_path Application.compile_env(:hyper, :umoci_path, nil) - @mke2fs_path Application.compile_env(:hyper, :mke2fs_path, "mke2fs") - - @doc """ - Root work directory for this node. All firecracker paths derive from it. - - Read from `#{@config_path}` (the single source of truth shared with the setuid - helper) the first time it is needed, then cached in `:persistent_term`. Falls - back to `#{@dev_work_dir}` when the file is absent (local dev / CI, where the - helper is not installed anyway). - """ - @spec work_dir :: Path.t() - def work_dir do - case :persistent_term.get({__MODULE__, :work_dir}, nil) do - nil -> - work_dir = load_work_dir() - :persistent_term.put({__MODULE__, :work_dir}, work_dir) - work_dir - - work_dir -> - work_dir - end - end - - @spec load_work_dir :: Path.t() - defp load_work_dir do - case File.read(@config_path) do - {:ok, body} -> body |> Toml.decode!() |> Map.fetch!("work_dir") - {:error, _} -> @dev_work_dir - end - end - - @doc "Directory holding redistributable binaries downloaded by the node." - @spec redist_dir :: Path.t() - def redist_dir, do: Path.join(work_dir(), "redist") - - @doc "Directory where `Hyper.Node.FireVMM.Provider` installs the firecracker release." - @spec firecracker_install_dir :: Path.t() - def firecracker_install_dir, do: Path.join(redist_dir(), "firecracker") - - @doc "Directory where `Hyper.Node.FireVMM.VmLinux.Provider` installs guest kernels." - @spec vmlinux_install_dir :: Path.t() - def vmlinux_install_dir, do: Path.join(redist_dir(), "vmlinux") - - @doc "Directory where `Hyper.Img.OciLoader.Umoci` installs the default umoci binary." - @spec umoci_install_dir :: Path.t() - def umoci_install_dir, do: Path.join(redist_dir(), "umoci") - - @doc """ - Path to the directory where all VM chroot's are created (`/jails`). - - If it does not exist, `Hyper.Node` will attempt to create one. - """ - @spec chroot_base :: Path.t() - def chroot_base, do: Path.join(work_dir(), "jails") - - @doc """ - A name for the parent cgroup which is used as a supervision cgroup for all VMs. - """ - def parent_cgroup, do: @parent_cgroup - - @doc """ - Path to the directory where all VM sockets are held. - - Must be stable across all nodes, and must be a directory. If it does not exist, `Hyper.Node` - will attempt to create one. - """ - @spec socket_dir :: Path.t() - def socket_dir, do: Path.join(work_dir(), "socks") - - @doc """ - Range in which `Hyper` will attempt to allocate uid/gids. Whenever a VM is allocated, it will - get a fresh uid/gid pair in this range. It is absolutely critical that this range is not used - by any other process on the system, as that can risk security. - """ - @spec uid_gid_range :: {integer(), integer()} - def uid_gid_range, do: @uid_gid_range - - @doc """ - Location of all image layers on all nodes. - - Hyper expects you to keep your layers in a flat directory, which may be backed by anything you - like: a plain filesystem, an NFS drive. This registry only ever is used to find paths to layers - but not anything more. - - Must be stable across all nodes, and must be a directory. If it does not exist, `Hyper.Node` - will attempt to create one. - """ - @spec layer_dir :: Path.t() - def layer_dir, do: @layer_dir - - @doc "Path to the losetup binary." - def losetup_path, do: @losetup_path - - @doc "Path to the dmsetup binary." - def dmsetup_path, do: @dmsetup_path - - @doc "Path to the blockdev binary." - def blockdev_path, do: @blockdev_path - - @doc "Path to the skopeo binary (used by `Hyper.Img.OciLoader` to pull OCI images)." - def skopeo_path, do: @skopeo_path - - @doc """ - Operator-configured path to the umoci binary, or `nil` (the default) to let - `Hyper.Img.OciLoader.Umoci` download and manage a pinned default. - """ - def umoci_path, do: @umoci_path - - @doc "Path to the mke2fs binary (used by `Hyper.Img.OciLoader` to build the ext4 rootfs)." - def mke2fs_path, do: @mke2fs_path - - @doc """ - Path to the setuid-root device helper (`hyper-suidhelper`). Required: the node - runs unprivileged and routes every `losetup`/`dmsetup`/`blockdev` operation - through it. - - Runtime config (host-specific), so it can be set per node without recompiling. - """ - @spec suid_helper :: Path.t() - def suid_helper, do: Application.fetch_env!(:hyper, :suid_helper) - - @doc """ - Directory for per-VM scratch (writable-layer COW) files. Must be node-local and - writable. If it does not exist, `Hyper.Node` will attempt to create one. - """ - @spec scratch_dir :: Path.t() - def scratch_dir, do: Path.join(work_dir(), "scratch") - - @doc """ - Per-architecture vmlinux (guest kernel) image paths, keyed by `Sys.Arch.t()`. - The operator places the kernels on the host and points these at them; - `Hyper.Node.Vmlinux` resolves and validates them per node. - """ - @spec vmlinux :: %{optional(Sys.Arch.t()) => Path.t()} - # Runtime read, not `compile_env`: an unset map would inline a literal `%{}`, - # which the type checker proves makes every `Map.fetch/2` on it return `:error`. - def vmlinux, do: Application.get_env(:hyper, :vmlinux, %{}) -end diff --git a/lib/hyper/grpc.ex b/lib/hyper/grpc.ex index 65603551..450bd609 100644 --- a/lib/hyper/grpc.ex +++ b/lib/hyper/grpc.ex @@ -9,63 +9,7 @@ defmodule Hyper.Grpc do `Hyper.Grpc.V0.Hyper.Stub` together with `connect/2`. """ - defmodule Config do - @moduledoc """ - gRPC server configuration, read from application env into a struct: - - config :hyper, Hyper.Grpc.Config, - enabled: true, - port: 50_051, - cred: GRPC.Credential.new(ssl: [certfile: "/path/cert.pem", keyfile: "/path/key.pem"]) - - Fields: - - * `:enabled` -- whether the server starts. Defaults to `false`. - * `:port` -- the listen port. Defaults to `50051`. - * `:cred` -- a `GRPC.Credential` for TLS, or `nil` (the default) for - plaintext. - * `:adapter_opts` -- options forwarded to the server adapter, e.g. - `[ip: {0, 0, 0, 0}]`. - - Build the credential where you load your keys (e.g. `config/runtime.exs`); - Hyper never reads the filesystem on your behalf. - - > #### Co-located nodes {: .info} - > - > Every node binds `:port`. Running multiple nodes on one host (e.g. a local - > cluster) requires giving each a distinct port via its own config. - """ - - @default_port 50_051 - - defstruct enabled: false, port: @default_port, cred: nil, adapter_opts: [] - - @type t :: %__MODULE__{ - enabled: boolean(), - port: :inet.port_number(), - cred: GRPC.Credential.t() | nil, - adapter_opts: keyword() - } - - @doc "Load the gRPC server configuration from application env." - @spec load() :: t() - def load, do: struct!(__MODULE__, Application.get_env(:hyper, __MODULE__, [])) - - @doc """ - The `GRPC.Server.Supervisor` options for this config: the endpoint and port, - plus the TLS credential and adapter options when set. - """ - @spec server_options(t()) :: keyword() - def server_options(%__MODULE__{} = config) do - [endpoint: Hyper.Grpc.Endpoint, port: config.port, start_server: true] - |> put_unless(:cred, config.cred, nil) - |> put_unless(:adapter_opts, config.adapter_opts, []) - end - - @spec put_unless(keyword(), atom(), term(), term()) :: keyword() - defp put_unless(opts, _key, skip, skip), do: opts - defp put_unless(opts, key, value, _skip), do: Keyword.put(opts, key, value) - end + alias Hyper.Cfg.Grpc, as: Config @doc """ The gRPC server's supervisor child, or `[]` when the server is disabled (the diff --git a/lib/hyper/img.ex b/lib/hyper/img.ex index f793d168..537dc824 100644 --- a/lib/hyper/img.ex +++ b/lib/hyper/img.ex @@ -6,7 +6,7 @@ defmodule Hyper.Img do `create/2` ingests a prepared image file -- e.g. the ext4 rootfs produced by `Hyper.Img.OciLoader` -- into the shared media store and the image database. It content-addresses the file (sha256 of its bytes = the image id), publishes it - into `Hyper.Config.layer_dir/0` at `layer_.img`, then records it as a + into `Hyper.Cfg.Dirs.layer_dir/0` at `layer_.img`, then records it as a one-layer base image (`blobs` + `images` + `image_layers`). Producers of image files stay decoupled from the store and DB: they hand a path to `create/2`. """ @@ -15,7 +15,6 @@ defmodule Hyper.Img do require Logger - alias Hyper.Config alias Hyper.Img.Db.{Blob, Image, ImageLayer, Repo} @type id :: String.t() @@ -85,7 +84,7 @@ defmodule Hyper.Img do @spec publish(Path.t(), id()) :: {:ok, Path.t(), :created | :reused} | {:error, term()} @decorate with_span("Hyper.Img.publish", include: [:id]) defp publish(src, id) do - File.mkdir_p!(Config.layer_dir()) + File.mkdir_p!(Hyper.Cfg.Dirs.layer_dir()) final = final_path(id) if File.exists?(final) do @@ -125,7 +124,7 @@ defmodule Hyper.Img do end @spec final_path(id()) :: Path.t() - defp final_path(id), do: Path.join(Config.layer_dir(), "layer_#{id}.img") + defp final_path(id), do: Path.join(Hyper.Cfg.Dirs.layer_dir(), "layer_#{id}.img") # Record the base image: one blob, one image (id == blob id), one layer at # position 0. All upserts are idempotent so a re-publish of the same bytes is a diff --git a/lib/hyper/img/db/gc.ex b/lib/hyper/img/db/gc.ex index 02434c27..c414a907 100644 --- a/lib/hyper/img/db/gc.ex +++ b/lib/hyper/img/db/gc.ex @@ -31,9 +31,10 @@ defmodule Hyper.Img.Db.Gc do require Logger import Ecto.Query + alias Hyper.Cfg.Gc, as: Config alias Hyper.Cluster.Routing alias Hyper.Img.Db.{Blob, ImageLayer, Repo} - alias Hyper.Img.Db.Gc.{Config, Sweep} + alias Hyper.Img.Db.Gc.Sweep alias Hyper.Node.Layer.Repo, as: LayerRepo @singleton_key {:singleton, :layer_gc} @@ -59,13 +60,7 @@ defmodule Hyper.Img.Db.Gc do @impl true def init(opts) do config = Keyword.get(opts, :config) || Config.load() - - if config.enabled do - {:ok, %__MODULE__{config: config}, {:continue, :acquire}} - else - Logger.info("layer gc: disabled by config; not starting") - :ignore - end + {:ok, %__MODULE__{config: config}, {:continue, :acquire}} end @impl true @@ -99,7 +94,7 @@ defmodule Hyper.Img.Db.Gc do try do {:noreply, scan_one_batch(state)} rescue - # Only swallow database unavailability (incl. statement_timeout aborts) + # Only swallow database unavailability (incl. timed-out statements) # and retry; let any other exception crash so a real bug surfaces. e in [Postgrex.Error, DBConnection.ConnectionError] -> Logger.warning( @@ -247,11 +242,11 @@ defmodule Hyper.Img.Db.Gc do state |> with_low_priority(fn -> Repo.all(query) end) |> MapSet.new() end - # Run a DB operation at low priority: in a transaction whose statement_timeout - # is capped, so it can never pin a backend and yields under contention. + # Run a DB operation at low priority: in a transaction with a capped per-statement + # timeout, so it can never pin a backend and yields under contention. @spec with_low_priority(t(), (-> result)) :: result when result: var defp with_low_priority(state, fun) do - timeout = Unit.Time.as_ms(state.config.statement_timeout) + timeout = Unit.Time.as_ms(state.config.timeout) {:ok, result} = Repo.transaction(fn -> diff --git a/lib/hyper/img/db/gc/config.ex b/lib/hyper/img/db/gc/config.ex deleted file mode 100644 index 83dea705..00000000 --- a/lib/hyper/img/db/gc/config.ex +++ /dev/null @@ -1,55 +0,0 @@ -defmodule Hyper.Img.Db.Gc.Config do - @moduledoc """ - Configuration for the layer garbage collector (`Hyper.Img.Db.Gc`). - - Every field has a default, so configuration is optional - set only what you want - to change. Durations are `Unit.Time` values, so (like `Hyper.Node.Config.Budget`) - overrides belong in `config/runtime.exs`: - - config :hyper, Hyper.Img.Db.Gc.Config, - enabled: true, - sweep_interval: Unit.Time.s(30), - grace_period: Unit.Time.s(60 * 60) - - Set `enabled: false` to turn the collector off entirely - it then never starts. - - ## Fields - - * `enabled` - run the collector at all (default `true`). - * `batch_size` - rows per keyset page (default `200`). - * `batch_pause` - pause between pages within a sweep (default `100ms`). - * `sweep_interval` - rest between completed sweeps (default `60s`). - * `acquire_interval` - how often a standby retries to become active (default `5s`). - * `retry` - backoff after the medium or database is unavailable (default `60s`). - * `statement_timeout` - cap on each GC DB statement so it can't pin a backend - (default `5s`). - * `grace_period` - never prune a blob younger than this, so a row whose file is - still being published is safe (default `1h`). - """ - - @type t :: %__MODULE__{ - enabled: boolean(), - batch_size: pos_integer(), - batch_pause: Unit.Time.t(), - sweep_interval: Unit.Time.t(), - acquire_interval: Unit.Time.t(), - retry: Unit.Time.t(), - statement_timeout: Unit.Time.t(), - grace_period: Unit.Time.t() - } - - defstruct enabled: true, - batch_size: 200, - batch_pause: Unit.Time.ms(100), - sweep_interval: Unit.Time.s(60), - acquire_interval: Unit.Time.s(5), - retry: Unit.Time.s(60), - statement_timeout: Unit.Time.s(5), - grace_period: Unit.Time.s(60 * 60) - - @doc "Build the config from app env, filling any unset field with its default." - @spec load() :: t() - def load do - struct!(__MODULE__, Application.get_env(:hyper, __MODULE__, [])) - end -end diff --git a/lib/hyper/img/db/repo.ex b/lib/hyper/img/db/repo.ex index a0e8cb81..96ed6735 100644 --- a/lib/hyper/img/db/repo.ex +++ b/lib/hyper/img/db/repo.ex @@ -16,4 +16,9 @@ defmodule Hyper.Img.Db.Repo do use Ecto.Repo, otp_app: :hyper, adapter: Ecto.Adapters.Postgres + + @impl true + def init(_context, config) do + {:ok, Keyword.merge(config, Hyper.Cfg.Img.Db.repo_opts())} + end end diff --git a/lib/hyper/img/oci_loader.ex b/lib/hyper/img/oci_loader.ex index a261c6f6..1ba5fb1c 100644 --- a/lib/hyper/img/oci_loader.ex +++ b/lib/hyper/img/oci_loader.ex @@ -20,7 +20,6 @@ defmodule Hyper.Img.OciLoader do use Unit.Operators - alias Hyper.Config alias Hyper.Img.OciLoader.Umoci alias Unit.Information @@ -81,9 +80,9 @@ defmodule Hyper.Img.OciLoader do def test_system do with {:ok, _arch} <- Sys.Arch.current() do tools = [ - {"skopeo", Config.skopeo_path()}, + {"skopeo", Hyper.Cfg.Tools.skopeo()}, {"umoci", Umoci.bin()}, - {"mke2fs", Config.mke2fs_path()} + {"mke2fs", Hyper.Cfg.Tools.mke2fs()} ] missing = for {name, path} <- tools, System.find_executable(path) == nil, do: name @@ -137,7 +136,7 @@ defmodule Hyper.Img.OciLoader do bundle = Path.join(tmp, "bundle") skopeo = - cmd(Config.skopeo_path(), [ + cmd(Hyper.Cfg.Tools.skopeo(), [ "copy", "--override-os", "linux", @@ -185,14 +184,16 @@ defmodule Hyper.Img.OciLoader do {:ok, Path.t()} | {:error, term()} defp build_ext4(rootfs, {size, inodes}) do Logger.debug("oci: building #{Information.as_mib(size)} MiB ext4 rootfs (#{inodes} inodes)") - File.mkdir_p!(Config.layer_dir()) - staged = Path.join(Config.layer_dir(), ".incoming-#{System.unique_integer([:positive])}.img") + File.mkdir_p!(Hyper.Cfg.Dirs.layer_dir()) + + staged = + Path.join(Hyper.Cfg.Dirs.layer_dir(), ".incoming-#{System.unique_integer([:positive])}.img") args = ["-t", "ext4", "-F", "-q", "-N", to_string(inodes), "-d", rootfs, staged] ++ ["#{Information.as_mib(size)}M"] - case tag(cmd(Config.mke2fs_path(), args), :mke2fs) do + case tag(cmd(Hyper.Cfg.Tools.mke2fs(), args), :mke2fs) do :ok -> {:ok, staged} diff --git a/lib/hyper/img/oci_loader/umoci.ex b/lib/hyper/img/oci_loader/umoci.ex index 06929b58..3f7d3b6d 100644 --- a/lib/hyper/img/oci_loader/umoci.ex +++ b/lib/hyper/img/oci_loader/umoci.ex @@ -5,15 +5,13 @@ defmodule Hyper.Img.OciLoader.Umoci do Two sources, in priority order (mirrors `Hyper.Node.Vmlinux`): - 1. An operator-configured path via `config :hyper, umoci_path: - "/path/to/umoci"` (`Hyper.Config.umoci_path/0`). If set, it wins and is - never downloaded. + 1. An operator-configured path via `[tools] umoci` in + `/etc/hyper/config.toml` (`Hyper.Cfg.Tools.umoci/0`). If set, it wins + and is never downloaded. 2. Otherwise the pinned static binary downloaded by `ensure_installed/0` - into `Hyper.Config.umoci_install_dir/0` (`/redist/umoci`). + into `Hyper.Cfg.Dirs.umoci_install_dir/0` (`/redist/umoci`). """ - alias Hyper.Config - require Logger # Pinned umoci release per architecture: the static binary's filename, its @@ -40,7 +38,7 @@ defmodule Hyper.Img.OciLoader.Umoci do """ @spec ensure_installed() :: :ok | {:error, term()} def ensure_installed do - if Config.umoci_path() != nil do + if Hyper.Cfg.Tools.umoci() != nil do :ok else with {:ok, arch} <- Sys.Arch.current() do @@ -57,19 +55,19 @@ defmodule Hyper.Img.OciLoader.Umoci do """ @spec bin() :: Path.t() def bin do - configured = Config.umoci_path() + case Hyper.Cfg.Tools.umoci() do + nil -> + {:ok, arch} = Sys.Arch.current() + default_path(arch) - if configured != nil do - configured - else - {:ok, arch} = Sys.Arch.current() - default_path(arch) + configured -> + configured end end @spec default_path(Sys.Arch.t()) :: Path.t() defp default_path(arch) do - Path.join(Config.umoci_install_dir(), Map.fetch!(@downloads, arch).asset) + Path.join(Hyper.Cfg.Dirs.umoci_install_dir(), Map.fetch!(@downloads, arch).asset) end @spec install(Sys.Arch.t(), Path.t()) :: :ok | {:error, term()} diff --git a/lib/hyper/node.ex b/lib/hyper/node.ex index 6282eaa7..a5e0f63b 100644 --- a/lib/hyper/node.ex +++ b/lib/hyper/node.ex @@ -143,7 +143,7 @@ defmodule Hyper.Node do @spec test_system :: :ok | {:error, term()} def test_system do - with {:ok, _} <- Hyper.Node.Config.Budget.load(), + with {:ok, _} <- Hyper.Cfg.Budget.load(), :ok <- Hyper.Node.FireVMM.Provider.ensure_installed(), :ok <- Hyper.Node.FireVMM.VmLinux.Provider.ensure_installed(), :ok <- Hyper.Node.Vmlinux.test_system(), @@ -160,10 +160,10 @@ defmodule Hyper.Node do @spec check_helper_base(Path.t()) :: :ok | {:error, {:suid_helper_base_mismatch, Path.t(), Path.t()}} defp check_helper_base(base) do - if base == Hyper.Config.work_dir() do + if base == Hyper.Cfg.Dirs.work_dir() do :ok else - {:error, {:suid_helper_base_mismatch, base, Hyper.Config.work_dir()}} + {:error, {:suid_helper_base_mismatch, base, Hyper.Cfg.Dirs.work_dir()}} end end diff --git a/lib/hyper/node/budget/config.ex b/lib/hyper/node/budget/config.ex deleted file mode 100644 index 30e6218c..00000000 --- a/lib/hyper/node/budget/config.ex +++ /dev/null @@ -1,67 +0,0 @@ -defmodule Hyper.Node.Config.Budget do - @moduledoc """ - This node's resource budget configuration. - - Carries both the hard caps consumed by `Hyper.Node.Budget.Hard` (`mem_max`, - `disk_max`) and the soft load ceilings consumed by `Hyper.Node.Budget.Soft`. - - The soft side names, per metric, the machine's absolute capacity and the - fraction of it past which the node is considered too loaded to take on more - work: - - * `cpu_max_load` - utilization fraction (`0.0..1.0`) above which CPU is full; - CPU capacity is the whole machine, so no separate cap is needed. - * `disk_bw_cap` / `disk_bw_max_load` - absolute disk throughput and the - fraction of it usable before disk is considered saturated. - * `net_bw_cap` / `net_bw_max_load` - the same for network throughput. - """ - - @type t :: %__MODULE__{ - mem_max: Unit.Information.t(), - disk_max: Unit.Information.t(), - cpu_max_load: float(), - disk_bw_cap: Unit.Bandwidth.t(), - disk_bw_max_load: float(), - net_bw_cap: Unit.Bandwidth.t(), - net_bw_max_load: float() - } - defstruct [ - :mem_max, - :disk_max, - :cpu_max_load, - :disk_bw_cap, - :disk_bw_max_load, - :net_bw_cap, - :net_bw_max_load - ] - - @spec load :: {:ok, t()} | {:error, term()} - def load do - case Application.fetch_env(:hyper, __MODULE__) do - {:ok, env} -> - case safe_struct(env) do - {:ok, config} -> - # TODO(markovejnovic): Check whether the limits are under the - # system limits by querying Sys. - :persistent_term.put(__MODULE__, config) - {:ok, config} - - {:error, _} = err -> - err - end - - :error -> - {:error, :config_missing} - end - end - - defp safe_struct(env) do - {:ok, struct!(__MODULE__, env)} - rescue - e in KeyError -> {:error, {:unknown_key, e.key}} - e in ArgumentError -> {:error, {:invalid_config, e.message}} - end - - @spec get :: t() - def get, do: :persistent_term.get(__MODULE__) -end diff --git a/lib/hyper/node/budget/hard.ex b/lib/hyper/node/budget/hard.ex index 8f8e6916..fcd325cb 100644 --- a/lib/hyper/node/budget/hard.ex +++ b/lib/hyper/node/budget/hard.ex @@ -6,7 +6,7 @@ defmodule Hyper.Node.Budget.Hard do "Hard" means the limits are inviolable: a reservation that would push the running total past the node's configured `mem_max`/`disk_max` - (`Hyper.Node.Config.Budget`) is refused outright. Callers reserve through + (`Hyper.Cfg.Budget`) is refused outright. Callers reserve through `with_budget/2`, which holds the budget for the duration of the callback and releases it afterwards. """ @@ -15,7 +15,7 @@ defmodule Hyper.Node.Budget.Hard do use Unit.Operators use OpenTelemetryDecorator - alias Hyper.Node.Config.Budget, as: Config + alias Hyper.Cfg.Budget, as: Config alias Hyper.Vm.Instance defmodule State do diff --git a/lib/hyper/node/budget/node_state.ex b/lib/hyper/node/budget/node_state.ex index ba7921ea..aa9af303 100644 --- a/lib/hyper/node/budget/node_state.ex +++ b/lib/hyper/node/budget/node_state.ex @@ -10,8 +10,8 @@ defmodule Hyper.Node.Budget.NodeState do knowing the target's config or core count. """ + alias Hyper.Cfg.Budget, as: Config alias Hyper.Node.Budget.Hard - alias Hyper.Node.Config.Budget, as: Config alias Hyper.Vm.Instance.Spec alias Sys.Mon alias Sys.Mon.Server.Reading diff --git a/lib/hyper/node/budget/soft.ex b/lib/hyper/node/budget/soft.ex index 758d9699..da50d2f0 100644 --- a/lib/hyper/node/budget/soft.ex +++ b/lib/hyper/node/budget/soft.ex @@ -7,7 +7,7 @@ defmodule Hyper.Node.Budget.Soft do The soft metrics are the ones whose overcommitment degrades speed rather than correctness: CPU utilization, disk bandwidth, and network bandwidth. For each, - the node carries a load ceiling in `Hyper.Node.Config.Budget` (e.g. "never + the node carries a load ceiling in `Hyper.Cfg.Budget` (e.g. "never schedule onto a machine already past 80% CPU"). A spec is admissible only if the measured load plus the spec's nominal demand stays under that ceiling on every metric. @@ -19,7 +19,7 @@ defmodule Hyper.Node.Budget.Soft do use Unit.Operators use OpenTelemetryDecorator - alias Hyper.Node.Config.Budget, as: Config + alias Hyper.Cfg.Budget, as: Config alias Hyper.Vm.Instance alias Sys.Mon alias Sys.Mon.Server.Reading diff --git a/lib/hyper/node/fire_vmm/client.ex b/lib/hyper/node/fire_vmm/client.ex index a336b095..c6901ae2 100644 --- a/lib/hyper/node/fire_vmm/client.ex +++ b/lib/hyper/node/fire_vmm/client.ex @@ -33,8 +33,6 @@ defmodule Hyper.Node.FireVMM.Client do alias Hyper.Node.FireVMM.Jailer - @call_timeout 35_000 - defmodule Opts do @moduledoc """ Start options for `Hyper.Node.FireVMM.Client`. Only `:vm_id` is required; @@ -71,6 +69,9 @@ defmodule Hyper.Node.FireVMM.Client do @spec via(Hyper.Vm.id()) :: GenServer.name() def via(vm_id), do: Hyper.Cluster.Routing.via({vm_id, :client}) + # Cap on a single Firecracker API call. + @call_timeout :timer.seconds(35) + @doc "Run a generated operation against this VM's daemon, serialized." @spec run(GenServer.server(), (keyword() -> result)) :: result when result: var def run(server, op_fun) when is_function(op_fun, 1) do diff --git a/lib/hyper/node/fire_vmm/jailer.ex b/lib/hyper/node/fire_vmm/jailer.ex index 6dd1b800..381f9d23 100644 --- a/lib/hyper/node/fire_vmm/jailer.ex +++ b/lib/hyper/node/fire_vmm/jailer.ex @@ -36,8 +36,6 @@ defmodule Hyper.Node.FireVMM.Jailer do first failure. """ - alias Hyper.Config - @doc "Run every pre-requisite check, halting at the first failure." @spec run() :: :ok | {:error, term()} def run do @@ -63,7 +61,7 @@ defmodule Hyper.Node.FireVMM.Jailer do end defp parent_cgroup_present do - if Sys.Linux.Cgroup.V2.named_exists?(Config.parent_cgroup()), + if Sys.Linux.Cgroup.V2.named_exists?(Hyper.Cfg.Jails.cgroup()), do: :ok, else: {:error, :missing_parent_cgroup} end @@ -79,7 +77,7 @@ defmodule Hyper.Node.FireVMM.Jailer do end defp chroot_writable do - case Sys.Posix.ensure_writable_dir(Config.chroot_base()) do + case Sys.Posix.ensure_writable_dir(Hyper.Cfg.Dirs.chroot_base()) do {:ok} -> :ok {:error, reason} -> {:error, {:chroot_base_unavailable, reason}} end @@ -104,11 +102,11 @@ defmodule Hyper.Node.FireVMM.Jailer do "--gid", to_string(opts.gid), "--chroot-base-dir", - Hyper.Config.chroot_base(), + Hyper.Cfg.Dirs.chroot_base(), "--cgroup-version", "2", "--parent-cgroup", - Hyper.Config.parent_cgroup() + Hyper.Cfg.Jails.cgroup() ] ++ cgroup_flags(opts.type) ++ ["--", "--api-sock", "/" <> @jail_socket] @@ -129,7 +127,7 @@ defmodule Hyper.Node.FireVMM.Jailer do @doc "Host path of the VM's per-VM jail dir (`//`)." @spec chroot_dir(Hyper.Vm.id()) :: Path.t() def chroot_dir(id) do - Path.join([Hyper.Config.chroot_base(), exec_name(), id]) + Path.join([Hyper.Cfg.Dirs.chroot_base(), exec_name(), id]) end @doc "Host path of the VM's chroot root (`///root`)." @@ -145,7 +143,7 @@ defmodule Hyper.Node.FireVMM.Jailer do """ @spec cgroup_dir(Hyper.Vm.id()) :: Path.t() def cgroup_dir(id) do - Path.join(["/sys/fs/cgroup", Hyper.Config.parent_cgroup(), exec_name(), id]) + Path.join(["/sys/fs/cgroup", Hyper.Cfg.Jails.cgroup(), exec_name(), id]) end @doc """ @@ -158,7 +156,7 @@ defmodule Hyper.Node.FireVMM.Jailer do @spec host_socket(Hyper.Vm.id()) :: Path.t() def host_socket(id) do Path.join([ - Hyper.Config.chroot_base(), + Hyper.Cfg.Dirs.chroot_base(), exec_name(), id, "root", diff --git a/lib/hyper/node/fire_vmm/provider.ex b/lib/hyper/node/fire_vmm/provider.ex index e50aee16..0730c588 100644 --- a/lib/hyper/node/fire_vmm/provider.ex +++ b/lib/hyper/node/fire_vmm/provider.ex @@ -1,7 +1,7 @@ defmodule Hyper.Node.FireVMM.Provider do @moduledoc """ Installs the firecracker release for the current architecture into - `Hyper.Config.firecracker_install_dir/0` (`/redist/firecracker`). + `Hyper.Cfg.Dirs.firecracker_install_dir/0` (`/redist/firecracker`). `ensure_installed/0` is idempotent: if the binaries are already present and executable it returns `:ok` without touching the network. Otherwise it fetches @@ -88,5 +88,5 @@ defmodule Hyper.Node.FireVMM.Provider do Path.join(install_dir(), Map.fetch!(dl, key)) end - defp install_dir, do: Hyper.Config.firecracker_install_dir() + defp install_dir, do: Hyper.Cfg.Dirs.firecracker_install_dir() end diff --git a/lib/hyper/node/fire_vmm/vm_linux/provider.ex b/lib/hyper/node/fire_vmm/vm_linux/provider.ex index 0b8f7682..b01daf84 100644 --- a/lib/hyper/node/fire_vmm/vm_linux/provider.ex +++ b/lib/hyper/node/fire_vmm/vm_linux/provider.ex @@ -1,7 +1,7 @@ defmodule Hyper.Node.FireVMM.VmLinux.Provider do @moduledoc """ Installs the guest-kernel (vmlinux) images for the current architecture into - `Hyper.Config.vmlinux_install_dir/0` (`/redist/vmlinux`). + `Hyper.Cfg.Dirs.vmlinux_install_dir/0` (`/redist/vmlinux`). The available kernels and their SHA-256 sums come from the statically-embedded `Hyper.Node.FireVMM.VmLinux.Manifest`. `ensure_installed/0` installs *every* @@ -89,5 +89,5 @@ defmodule Hyper.Node.FireVMM.VmLinux.Provider do defp build_path(dir, build), do: Path.join(dir, build.asset) - defp install_dir, do: Hyper.Config.vmlinux_install_dir() + defp install_dir, do: Hyper.Cfg.Dirs.vmlinux_install_dir() end diff --git a/lib/hyper/node/img/mutable.ex b/lib/hyper/node/img/mutable.ex index 386e36f0..57de460b 100644 --- a/lib/hyper/node/img/mutable.ex +++ b/lib/hyper/node/img/mutable.ex @@ -24,8 +24,6 @@ defmodule Hyper.Node.Img.Mutable do use OpenTelemetryDecorator - @idle_timeout_ms :timer.seconds(30) - defmodule Opts do @moduledoc false @enforce_keys [:img_id, :vm_id] @@ -149,9 +147,13 @@ defmodule Hyper.Node.Img.Mutable do end end + # How long an idle mutable layer lingers with no users before it is torn down. + @idle_grace :timer.seconds(30) + defp arm_idle(state) do state = cancel_idle(state) - %{state | idle_ref: Process.send_after(self(), :idle_timeout, @idle_timeout_ms)} + + %{state | idle_ref: Process.send_after(self(), :idle_timeout, @idle_grace)} end defp cancel_idle(%State{idle_ref: nil} = state), do: state diff --git a/lib/hyper/node/img/server.ex b/lib/hyper/node/img/server.ex index d51fe3ba..9587049c 100644 --- a/lib/hyper/node/img/server.ex +++ b/lib/hyper/node/img/server.ex @@ -21,9 +21,6 @@ defmodule Hyper.Node.Img.Server do use OpenTelemetryDecorator - # Grace period after the last holder leaves before the image is torn down. - @idle_timeout_ms :timer.seconds(30) - defmodule State do @moduledoc false @@ -225,10 +222,14 @@ defmodule Hyper.Node.Img.Server do end end + # How long an idle image lingers with no users before it is torn down. + @idle_grace :timer.seconds(30) + @spec arm_idle(State.t()) :: State.t() defp arm_idle(state) do state = cancel_idle(state) - %{state | idle_ref: Process.send_after(self(), :idle_timeout, @idle_timeout_ms)} + + %{state | idle_ref: Process.send_after(self(), :idle_timeout, @idle_grace)} end @spec cancel_idle(State.t()) :: State.t() diff --git a/lib/hyper/node/img/thin_pool.ex b/lib/hyper/node/img/thin_pool.ex index f3ec3e19..7a50f7ec 100644 --- a/lib/hyper/node/img/thin_pool.ex +++ b/lib/hyper/node/img/thin_pool.ex @@ -13,7 +13,7 @@ defmodule Hyper.Node.Img.ThinPool do use GenServer - alias Hyper.Node.Config.Img, as: ImgConfig + alias Hyper.Cfg.Img, as: ImgConfig alias Hyper.SuidHelper alias Unit.Information @@ -50,7 +50,7 @@ defmodule Hyper.Node.Img.ThinPool do def init(_opts) do Process.flag(:trap_exit, true) - with :ok <- File.mkdir_p(Hyper.Config.scratch_dir()), + with :ok <- File.mkdir_p(Hyper.Cfg.Dirs.scratch_dir()), {:ok, meta} <- ensure_backing(@meta_file, ImgConfig.thin_pool_meta_size()), {:ok, data} <- ensure_backing(@data_file, ImgConfig.thin_pool_data_size()), :ok <- zero_metadata(meta), @@ -114,7 +114,7 @@ defmodule Hyper.Node.Img.ThinPool do # Create a sparse file of `size` if absent; reuse it if already present. @spec ensure_backing(String.t(), Information.t()) :: {:ok, Path.t()} | {:error, term()} defp ensure_backing(file, size) do - path = Path.join(Hyper.Config.scratch_dir(), file) + path = Path.join(Hyper.Cfg.Dirs.scratch_dir(), file) case File.open(path, [:write, :read]) do {:ok, io} -> diff --git a/lib/hyper/node/layer/repo.ex b/lib/hyper/node/layer/repo.ex index 6604a08c..f9cf0be5 100644 --- a/lib/hyper/node/layer/repo.ex +++ b/lib/hyper/node/layer/repo.ex @@ -15,8 +15,8 @@ defmodule Hyper.Node.Layer.Repo do @decorate with_span("Hyper.Node.Layer.Repo.test_system") def test_system do cond do - not File.exists?(Hyper.Config.layer_dir()) -> {:error, :layer_dir_not_found} - not File.dir?(Hyper.Config.layer_dir()) -> {:error, :layer_dir_not_dir} + not File.exists?(Hyper.Cfg.Dirs.layer_dir()) -> {:error, :layer_dir_not_found} + not File.dir?(Hyper.Cfg.Dirs.layer_dir()) -> {:error, :layer_dir_not_dir} true -> :ok end end @@ -25,7 +25,7 @@ defmodule Hyper.Node.Layer.Repo do @spec find_layer(Hyper.Layer.id()) :: {:ok, Path.t()} | {:error, File.posix()} @decorate with_span("Hyper.Node.Layer.Repo.find_layer") def find_layer(id) do - path = Path.join([Hyper.Config.layer_dir(), layer_basename(id)]) + path = Path.join([Hyper.Cfg.Dirs.layer_dir(), layer_basename(id)]) case File.stat(path) do {:ok, _stat} -> {:ok, path} diff --git a/lib/hyper/node/layer/server.ex b/lib/hyper/node/layer/server.ex index 188e135b..79e70491 100644 --- a/lib/hyper/node/layer/server.ex +++ b/lib/hyper/node/layer/server.ex @@ -5,7 +5,8 @@ defmodule Hyper.Node.Layer.Server do Reference-counted via process monitors: each holder `acquire/1`s the layer and the server monitors it, so a holder that crashes is released automatically (no leaked count). When the last holder goes away the server waits a short idle - grace period and then stops, unmounting the block device in `terminate/2`. + grace period and then stops, unmounting the block device in `terminate/2`. The + grace period keeps bursty acquire/release cycles from thrashing the mount. """ use GenServer @@ -15,10 +16,6 @@ defmodule Hyper.Node.Layer.Server do alias Hyper.Node.Layer.Repo alias Hyper.SuidHelper - # Grace period after the last holder leaves before the layer is unmounted. Keeps - # bursty acquire/release cycles from thrashing the mount. - @idle_timeout_ms :timer.seconds(30) - defmodule State do @moduledoc false @@ -149,10 +146,14 @@ defmodule Hyper.Node.Layer.Server do end end + # How long an idle layer mount lingers with no users before it is torn down. + @idle_grace :timer.seconds(30) + @spec arm_idle(State.t()) :: State.t() defp arm_idle(state) do state = cancel_idle(state) - %{state | idle_ref: Process.send_after(self(), :idle_timeout, @idle_timeout_ms)} + + %{state | idle_ref: Process.send_after(self(), :idle_timeout, @idle_grace)} end @spec cancel_idle(State.t()) :: State.t() diff --git a/lib/hyper/node/users.ex b/lib/hyper/node/users.ex index 72777d1f..6643da79 100644 --- a/lib/hyper/node/users.ex +++ b/lib/hyper/node/users.ex @@ -77,7 +77,7 @@ defmodule Hyper.Node.Users do def test_system do alias Sys.Linux.Subid - {min, max} = Hyper.Config.uid_gid_range() + {min, max} = Hyper.Cfg.Jails.uid_gid_range() cond do passwd_conflicts(min, max) != [] -> @@ -99,7 +99,7 @@ defmodule Hyper.Node.Users do @impl true def init(_opts) do - {min, max} = Hyper.Config.uid_gid_range() + {min, max} = Hyper.Cfg.Jails.uid_gid_range() {:ok, %State{max: max, next: min}} end diff --git a/lib/hyper/node/vmlinux.ex b/lib/hyper/node/vmlinux.ex index 92f7f688..027f0a6b 100644 --- a/lib/hyper/node/vmlinux.ex +++ b/lib/hyper/node/vmlinux.ex @@ -5,7 +5,7 @@ defmodule Hyper.Node.Vmlinux do Two sources, in priority order: 1. An operator-configured path for the node's architecture, via - `config :hyper, vmlinux: %{ => }` (see `Hyper.Config.vmlinux/0`). + `config :hyper, Hyper.Cfg.VmLinux, amd64: ..., aarch64: ...` (see `Hyper.Cfg.VmLinux.images/0`). If set, it wins - the operator can pin a custom kernel. 2. Otherwise, the default kernel downloaded by `Hyper.Node.FireVMM.VmLinux.Provider` (highest version for the arch). @@ -24,7 +24,7 @@ defmodule Hyper.Node.Vmlinux do """ @spec path(Sys.Arch.t()) :: Path.t() def path(arch) do - case Map.fetch(Hyper.Config.vmlinux(), arch) do + case Map.fetch(Hyper.Cfg.VmLinux.images(), arch) do {:ok, path} -> path @@ -44,7 +44,7 @@ defmodule Hyper.Node.Vmlinux do @spec test_system() :: :ok | {:error, term()} def test_system do with {:ok, arch} <- Sys.Arch.current() do - case Map.fetch(Hyper.Config.vmlinux(), arch) do + case Map.fetch(Hyper.Cfg.VmLinux.images(), arch) do {:ok, path} -> present(path) diff --git a/lib/hyper/suid_helper.ex b/lib/hyper/suid_helper.ex index 690f004a..1a11992a 100644 --- a/lib/hyper/suid_helper.ex +++ b/lib/hyper/suid_helper.ex @@ -30,7 +30,7 @@ defmodule Hyper.SuidHelper do @spec exec([String.t()]) :: {:ok, map()} | {:error, err()} @decorate with_span("Hyper.SuidHelper.exec", include: [:argv]) def exec(argv) do - case System.cmd(Hyper.Config.suid_helper(), argv, stderr_to_stdout: true) do + case System.cmd(Hyper.Cfg.Tools.suidhelper(), argv, stderr_to_stdout: true) do {out, 0} -> {:ok, Jason.decode!(out)} {out, code} -> {:error, {code, String.trim(out)}} end @@ -91,7 +91,7 @@ defmodule Hyper.SuidHelper do @spec helper_present() :: :ok | {:error, :suid_helper_not_found} defp helper_present do - if System.find_executable(Hyper.Config.suid_helper()), + if System.find_executable(Hyper.Cfg.Tools.suidhelper()), do: :ok, else: {:error, :suid_helper_not_found} end diff --git a/lib/hyper/suid_helper/blockdev.ex b/lib/hyper/suid_helper/blockdev.ex index 9f675ea5..be9bc39e 100644 --- a/lib/hyper/suid_helper/blockdev.ex +++ b/lib/hyper/suid_helper/blockdev.ex @@ -11,7 +11,7 @@ defmodule Hyper.SuidHelper.Blockdev do @spec device_sectors(Path.t()) :: {:ok, pos_integer()} | {:error, err()} @decorate with_span("Hyper.SuidHelper.Blockdev.device_sectors", include: [:path]) def device_sectors(path) do - case SuidHelper.exec(["blockdev", "--bin", Hyper.Config.blockdev_path(), "--getsz", path]) do + case SuidHelper.exec(["blockdev", "--bin", Hyper.Cfg.Tools.blockdev(), "--getsz", path]) do {:ok, %{"sectors" => n}} -> {:ok, n} {:error, _} = err -> err end @@ -21,7 +21,7 @@ defmodule Hyper.SuidHelper.Blockdev do @spec test_system() :: :ok | {:error, :blockdev_not_found} @decorate with_span("Hyper.SuidHelper.Blockdev.test_system") def test_system do - if System.find_executable(Hyper.Config.blockdev_path()), + if System.find_executable(Hyper.Cfg.Tools.blockdev()), do: :ok, else: {:error, :blockdev_not_found} end diff --git a/lib/hyper/suid_helper/dmsetup.ex b/lib/hyper/suid_helper/dmsetup.ex index ef670634..c8dabc88 100644 --- a/lib/hyper/suid_helper/dmsetup.ex +++ b/lib/hyper/suid_helper/dmsetup.ex @@ -18,7 +18,7 @@ defmodule Hyper.SuidHelper.Dmsetup do {:ok, Path.t()} | {:error, err()} @decorate with_span("Hyper.SuidHelper.Dmsetup.create_snapshot", include: [:name]) def create_snapshot(name, origin_dev, cow_dev, sectors) do - table = snapshot_table(origin_dev, cow_dev, sectors, Hyper.Node.Config.Img.chunk_sectors()) + table = snapshot_table(origin_dev, cow_dev, sectors, Hyper.Cfg.Img.chunk_sectors()) create(name, table, ["--readonly"]) end @@ -62,7 +62,7 @@ defmodule Hyper.SuidHelper.Dmsetup do case SuidHelper.exec([ "dmsetup", "--bin", - Hyper.Config.dmsetup_path(), + Hyper.Cfg.Tools.dmsetup(), "remove", "--retry", name @@ -77,7 +77,7 @@ defmodule Hyper.SuidHelper.Dmsetup do @decorate with_span("Hyper.SuidHelper.Dmsetup.message", include: [:name, :message]) def message(name, message) do argv = - ["dmsetup", "--bin", Hyper.Config.dmsetup_path(), "message", name, "--message", message] + ["dmsetup", "--bin", Hyper.Cfg.Tools.dmsetup(), "message", name, "--message", message] case SuidHelper.exec(argv) do {:ok, _} -> :ok @@ -92,7 +92,7 @@ defmodule Hyper.SuidHelper.Dmsetup do @spec test_system() :: :ok | {:error, term()} @decorate with_span("Hyper.SuidHelper.Dmsetup.test_system") def test_system do - if System.find_executable(Hyper.Config.dmsetup_path()), + if System.find_executable(Hyper.Cfg.Tools.dmsetup()), do: test_targets(), else: {:error, :dmsetup_not_found} end @@ -101,7 +101,7 @@ defmodule Hyper.SuidHelper.Dmsetup do @spec test_targets() :: :ok | {:error, term()} @decorate with_span("Hyper.SuidHelper.Dmsetup.test_targets") def test_targets do - case System.cmd(Hyper.Config.dmsetup_path(), ["targets"], stderr_to_stdout: true) do + case System.cmd(Hyper.Cfg.Tools.dmsetup(), ["targets"], stderr_to_stdout: true) do {out, 0} -> have = parse_targets(out) missing = Enum.reject(@required_targets, &MapSet.member?(have, &1)) @@ -146,7 +146,7 @@ defmodule Hyper.SuidHelper.Dmsetup do @spec create(String.t(), String.t(), [String.t()]) :: {:ok, Path.t()} | {:error, err()} defp create(name, table, flags) do argv = - ["dmsetup", "--bin", Hyper.Config.dmsetup_path(), "create", name] ++ + ["dmsetup", "--bin", Hyper.Cfg.Tools.dmsetup(), "create", name] ++ flags ++ ["--table", table] case SuidHelper.exec(argv) do diff --git a/lib/hyper/suid_helper/losetup.ex b/lib/hyper/suid_helper/losetup.ex index d825b731..9ee9191a 100644 --- a/lib/hyper/suid_helper/losetup.ex +++ b/lib/hyper/suid_helper/losetup.ex @@ -11,7 +11,7 @@ defmodule Hyper.SuidHelper.Losetup do @spec attach_ro(Path.t()) :: {:ok, Path.t()} | {:error, err()} @decorate with_span("Hyper.SuidHelper.Losetup.attach_ro", include: [:path]) def attach_ro(path) do - case SuidHelper.exec(["losetup", "--bin", Hyper.Config.losetup_path(), "attach", path]) do + case SuidHelper.exec(["losetup", "--bin", Hyper.Cfg.Tools.losetup(), "attach", path]) do {:ok, %{"device" => dev}} -> {:ok, dev} {:error, _} = err -> err end @@ -24,7 +24,7 @@ defmodule Hyper.SuidHelper.Losetup do case SuidHelper.exec([ "losetup", "--bin", - Hyper.Config.losetup_path(), + Hyper.Cfg.Tools.losetup(), "attach", "--rw", path @@ -38,7 +38,7 @@ defmodule Hyper.SuidHelper.Losetup do @spec detach(Path.t()) :: :ok | {:error, err()} @decorate with_span("Hyper.SuidHelper.Losetup.detach", include: [:dev]) def detach(dev) do - case SuidHelper.exec(["losetup", "--bin", Hyper.Config.losetup_path(), "detach", dev]) do + case SuidHelper.exec(["losetup", "--bin", Hyper.Cfg.Tools.losetup(), "detach", dev]) do {:ok, _} -> :ok {:error, _} = err -> err end @@ -48,7 +48,7 @@ defmodule Hyper.SuidHelper.Losetup do @spec test_system() :: :ok | {:error, :losetup_not_found} @decorate with_span("Hyper.SuidHelper.Losetup.test_system") def test_system do - if System.find_executable(Hyper.Config.losetup_path()), + if System.find_executable(Hyper.Cfg.Tools.losetup()), do: :ok, else: {:error, :losetup_not_found} end diff --git a/lib/sys/mon/cpu.ex b/lib/sys/mon/cpu.ex index 47d6d312..55e0635e 100644 --- a/lib/sys/mon/cpu.ex +++ b/lib/sys/mon/cpu.ex @@ -3,26 +3,26 @@ defmodule Sys.Mon.Cpu do alias Sys.Linux.Proc.Stat alias Sys.Mon.Server - alias Unit.Time - - @period_ms 23 - @tau_s 30 @moduledoc """ Monitors instantaneous CPU utilization (the soft beta_vcpus signal). - Samples `/proc/stat` every #{@period_ms} ms and reports the busy fraction + Samples `/proc/stat` every 23 ms and reports the busy fraction (`0.0..1.0`, normalized across all cores) between consecutive reads - never the load average, which has different semantics. The first read only establishes a - baseline (`:skip`). Readings are smoothed with a #{@tau_s}-second time constant + baseline (`:skip`). Readings are smoothed with a 30-second time constant (sampling fast only de-noises the filter; the smoothing window is set by `tau`). """ + # Prime sampling period, co-prime with the sibling monitors so their reads + # rarely land on the same tick. @impl true - def period, do: Time.ms(@period_ms) + @spec period :: Unit.Time.t() + def period, do: Unit.Time.ms(23) @impl true - def tau, do: Time.s(@tau_s) + @spec tau :: Unit.Time.t() + def tau, do: Unit.Time.s(30) @doc "The latest instantaneous + filtered CPU utilization (fractions `0.0..1.0`)." @spec value() :: Server.Reading.t() diff --git a/lib/sys/mon/disk_bw.ex b/lib/sys/mon/disk_bw.ex index 5cc97c73..a886ad5a 100644 --- a/lib/sys/mon/disk_bw.ex +++ b/lib/sys/mon/disk_bw.ex @@ -5,25 +5,25 @@ defmodule Sys.Mon.DiskBw do alias Sys.Linux.Proc.Diskstats alias Sys.Mon.Server alias Unit.Bandwidth - alias Unit.Time - - @period_ms 31 - @tau_s 20 @moduledoc """ Monitors instantaneous disk bandwidth (the soft beta_disk_bw signal). Samples cumulative read+write bytes across whole physical disks from - `/proc/diskstats` every #{@period_ms} ms and differentiates them into bytes/sec + `/proc/diskstats` every 31 ms and differentiates them into bytes/sec via `Controls.Rate` (the first read only establishes a baseline). The rate series - is smoothed with a #{@tau_s}-second time constant. Readings are `Unit.Bandwidth`. + is smoothed with a 20-second time constant. Readings are `Unit.Bandwidth`. """ + # Prime sampling period, co-prime with the sibling monitors so their reads + # rarely land on the same tick. @impl true - def period, do: Time.ms(@period_ms) + @spec period :: Unit.Time.t() + def period, do: Unit.Time.ms(31) @impl true - def tau, do: Time.s(@tau_s) + @spec tau :: Unit.Time.t() + def tau, do: Unit.Time.s(20) @doc "The latest instantaneous + filtered disk bandwidth (`Unit.Bandwidth` readings)." @spec value() :: Server.Reading.t() diff --git a/lib/sys/mon/mem.ex b/lib/sys/mon/mem.ex index 603e5133..6536fb57 100644 --- a/lib/sys/mon/mem.ex +++ b/lib/sys/mon/mem.ex @@ -4,25 +4,25 @@ defmodule Sys.Mon.Mem do alias Sys.Linux.Proc.Meminfo alias Sys.Mon.Server alias Unit.Information - alias Unit.Time - - @period_ms 29 - @tau_s 30 @moduledoc """ Monitors instantaneous memory pressure. - Samples `/proc/meminfo` every #{@period_ms} ms and reports *used* memory as - `MemTotal - MemAvailable`, smoothed with a #{@tau_s}-second time constant. Although + Samples `/proc/meminfo` every 29 ms and reports *used* memory as + `MemTotal - MemAvailable`, smoothed with a 30-second time constant. Although memory is an alpha (hard) budget tracked from VM specs, the live figure is useful for detecting actual pressure. Readings are `Unit.Information`. """ + # Prime sampling period, co-prime with the sibling monitors so their reads + # rarely land on the same tick. @impl true - def period, do: Time.ms(@period_ms) + @spec period :: Unit.Time.t() + def period, do: Unit.Time.ms(29) @impl true - def tau, do: Time.s(@tau_s) + @spec tau :: Unit.Time.t() + def tau, do: Unit.Time.s(30) @doc "The latest instantaneous + filtered used memory (`Unit.Information` readings)." @spec value() :: Server.Reading.t() diff --git a/lib/sys/mon/net_bw.ex b/lib/sys/mon/net_bw.ex index c877a52f..4687f2e9 100644 --- a/lib/sys/mon/net_bw.ex +++ b/lib/sys/mon/net_bw.ex @@ -5,25 +5,25 @@ defmodule Sys.Mon.NetBw do alias Sys.Linux.Proc.NetDev alias Sys.Mon.Server alias Unit.Bandwidth - alias Unit.Time - - @period_ms 37 - @tau_s 20 @moduledoc """ Monitors instantaneous network bandwidth (the soft beta_net_bw signal). Samples cumulative rx+tx bytes across physical interfaces from `/proc/net/dev` - every #{@period_ms} ms and differentiates them into bytes/sec via `Controls.Rate` + every 37 ms and differentiates them into bytes/sec via `Controls.Rate` (the first read only establishes a baseline). The rate series is smoothed with a - #{@tau_s}-second time constant. Readings are `Unit.Bandwidth`. + 20-second time constant. Readings are `Unit.Bandwidth`. """ + # Prime sampling period, co-prime with the sibling monitors so their reads + # rarely land on the same tick. @impl true - def period, do: Time.ms(@period_ms) + @spec period :: Unit.Time.t() + def period, do: Unit.Time.ms(37) @impl true - def tau, do: Time.s(@tau_s) + @spec tau :: Unit.Time.t() + def tau, do: Unit.Time.s(20) @doc "The latest instantaneous + filtered network bandwidth (`Unit.Bandwidth` readings)." @spec value() :: Server.Reading.t() diff --git a/lib/unit/bandwidth.ex b/lib/unit/bandwidth.ex index 5811d444..1a45da2a 100644 --- a/lib/unit/bandwidth.ex +++ b/lib/unit/bandwidth.ex @@ -37,6 +37,28 @@ defmodule Unit.Bandwidth do @doc "The zero throughput (additive identity)." @spec zero() :: t() def zero, do: %__MODULE__{bytes_per_sec: 0} + + @units %{"Bps" => 1, "KiBps" => @kib, "MiBps" => @mib, "GiBps" => @gib, "TiBps" => @tib} + + @doc "Parse a string like `\"1GiBps\"`. Suffixes: Bps/KiBps/MiBps/GiBps/TiBps." + @spec parse(String.t()) :: {:ok, t()} | {:error, {:bad_unit, String.t()}} + def parse(s) when is_binary(s) do + with {n, suffix} when n >= 0 <- Integer.parse(String.trim(s)), + {:ok, mult} <- Map.fetch(@units, String.trim(suffix)) do + {:ok, %__MODULE__{bytes_per_sec: n * mult}} + else + _ -> {:error, {:bad_unit, s}} + end + end + + @doc "Like `parse/1` but raises `ArgumentError` on bad input." + @spec parse!(String.t()) :: t() + def parse!(s) do + case parse(s) do + {:ok, v} -> v + {:error, _} -> raise ArgumentError, "invalid Bandwidth string: #{inspect(s)}" + end + end end defimpl Unit.Quantity, for: Unit.Bandwidth do diff --git a/lib/unit/information.ex b/lib/unit/information.ex index be9a31a0..c2ef0f41 100644 --- a/lib/unit/information.ex +++ b/lib/unit/information.ex @@ -43,6 +43,28 @@ defmodule Unit.Information do @doc "The zero quantity (additive identity)." @spec zero() :: t() def zero, do: %__MODULE__{bytes: 0} + + @units %{"B" => 1, "KiB" => @kib, "MiB" => @mib, "GiB" => @gib, "TiB" => @tib} + + @doc "Parse a string like `\"4GiB\"` into an `Information`. Suffixes: B/KiB/MiB/GiB/TiB." + @spec parse(String.t()) :: {:ok, t()} | {:error, {:bad_unit, String.t()}} + def parse(s) when is_binary(s) do + with {n, suffix} when n >= 0 <- Integer.parse(String.trim(s)), + {:ok, mult} <- Map.fetch(@units, String.trim(suffix)) do + {:ok, %__MODULE__{bytes: n * mult}} + else + _ -> {:error, {:bad_unit, s}} + end + end + + @doc "Like `parse/1` but raises `ArgumentError` on bad input." + @spec parse!(String.t()) :: t() + def parse!(s) do + case parse(s) do + {:ok, v} -> v + {:error, _} -> raise ArgumentError, "invalid Information string: #{inspect(s)}" + end + end end defimpl Unit.Quantity, for: Unit.Information do diff --git a/lib/unit/time.ex b/lib/unit/time.ex index b15950e5..86071e4e 100644 --- a/lib/unit/time.ex +++ b/lib/unit/time.ex @@ -42,6 +42,35 @@ defmodule Unit.Time do @doc "The zero duration (additive identity)." @spec zero() :: t() def zero, do: %__MODULE__{ns: 0} + + @units %{ + "ns" => 1, + "us" => @us, + "ms" => @ms, + "s" => @s, + "m" => 60 * @s, + "h" => 3600 * @s + } + + @doc ~s(Parse a duration string like `"60s"`/`"100ms"`/`"1h"`. Suffixes: ns/us/ms/s/m/h.) + @spec parse(String.t()) :: {:ok, t()} | {:error, {:bad_unit, String.t()}} + def parse(s) when is_binary(s) do + with {n, suffix} when n >= 0 <- Integer.parse(String.trim(s)), + {:ok, mult} <- Map.fetch(@units, String.trim(suffix)) do + {:ok, %__MODULE__{ns: n * mult}} + else + _ -> {:error, {:bad_unit, s}} + end + end + + @doc "Like `parse/1` but raises `ArgumentError` on bad input." + @spec parse!(String.t()) :: t() + def parse!(s) do + case parse(s) do + {:ok, v} -> v + {:error, _} -> raise ArgumentError, "invalid Time string: #{inspect(s)}" + end + end end defimpl Unit.Quantity, for: Unit.Time do diff --git a/mix.exs b/mix.exs index 7906c6a1..7ccae729 100644 --- a/mix.exs +++ b/mix.exs @@ -67,6 +67,10 @@ defmodule Hyper.MixProject do {:junit_formatter, "~> 3.4", only: :test, runtime: false}, {:dialyxir, "~> 1.4", only: [:dev], runtime: false}, {:ex_doc, "~> 0.34", only: :dev, runtime: false}, + # Syntect-backed Makeup lexer: covers the doc languages that have no + # dedicated Makeup lexer (markdown, toml, bash, sh, python). Elixir/erlang + # still use their native lexers; this fills the rest in one dep. + {:makeup_syntect, "~> 0.1", only: :dev, runtime: false}, {:ecto_sql, "~> 3.13"}, {:grpc, "~> 1.0"}, {:grpc_server, "~> 1.0"}, @@ -105,6 +109,8 @@ defmodule Hyper.MixProject do extras: [ "README.md", "docs/cookbook/intro.md", + "docs/cookbook/install.md", + "docs/cookbook/config.md", "docs/cookbook/architecture.md", "docs/grpc.md" ], @@ -183,6 +189,32 @@ defmodule Hyper.MixProject do end # `mix check` - the strict gate. Runs fast checks first, slow ones (dialyzer) last. + # Give ```bash / ```sh real syntax highlighting in the docs. + # + # Two obstacles, both about *who registers the lexer last*: + # 1. makeup_syntect registers the shell grammar only under its raw syntect + # name "Shell-Unix-Generic" (ExDoc resolves fences by lexer name, not + # file extension), so ```bash / ```sh never reach it. + # 2. ExDoc itself registers a minimal `ExDoc.ShellLexer` for sh/bash/shell/ + # zsh (it only de-selects the `$ ` prompt; everything else is plain text) + # from `ExDoc.Application.start`, which runs during the `docs` task. + # + # So we start :ex_doc and :makeup_syntect here first (idempotent — the later + # `docs` task won't re-run their `start/2`), then register our shell aliases + # LAST so they win. Dev-only; runs as the `docs` alias's first step. + defp register_doc_lexers(_args) do + {:ok, _} = Application.ensure_all_started(:makeup_syntect) + {:ok, _} = Application.ensure_all_started(:ex_doc) + + Makeup.Registry.register_lexer(MakeupSyntect.Lexer, + options: [language: "Shell-Unix-Generic"], + names: ["bash", "sh", "shell", "zsh"], + extensions: [] + ) + + :ok + end + defp aliases do [ check: [ @@ -192,6 +224,12 @@ defmodule Hyper.MixProject do "test --warnings-as-errors", "dialyzer" ], + # makeup_syntect registers the shell grammar only under its raw syntect + # name "Shell-Unix-Generic", and ExDoc resolves fences by lexer *name* + # (not file extension), so ```bash / ```sh would fall back to plain text. + # Alias them to the shell grammar before ExDoc runs (same VM, so the + # registration is visible to the highlighter). + docs: ["loadpaths", ®ister_doc_lexers/1, "docs"], # Force a regeneration of the Firecracker bindings (ignores staleness). "firecracker.gen": ["compile.firecracker_gen --force"], # Force a regeneration of the gRPC bindings (ignores staleness). diff --git a/mix.lock b/mix.lock index 06780d8a..8012573e 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,7 @@ %{ "acceptor_pool": {:hex, :acceptor_pool, "1.0.1", "d88c2e8a0be9216cf513fbcd3e5a4beb36bee3ff4168e85d6152c6f899359cdb", [:rebar3], [], "hexpm", "f172f3d74513e8edd445c257d596fc84dbdd56d2c6fa287434269648ae5a421e"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, + "castore": {:hex, :castore, "1.0.19", "6903cabdfd9d1af46454126e7c8385186659dd33ecfb74a885cae52221ad6109", [:mix], [], "hexpm", "3669e6cab13f54c2df26b3e6833745d647f35b6e30d8ddd5975df0d5c842ca98"}, "chatterbox": {:hex, :ts_chatterbox, "0.15.1", "5cac4d15dd7ad61fc3c4415ce4826fc563d4643dee897a558ec4ea0b1c835c9c", [:rebar3], [{:hpack, "~> 0.3.0", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "4f75b91451338bc0da5f52f3480fa6ef6e3a2aeecfc33686d6b3d0a0948f31aa"}, "cowboy": {:hex, :cowboy, "2.16.1", "fa04080b602ff25c40a7700f2dc0152dbc1ba26b42093ae0fa9bb7a337d5a242", [:make, :rebar3], [{:cowlib, ">= 2.16.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "b8ea4dd317a043e3177ec840cfa3bcb47cfb41035d3abb24d954dc7d51def399"}, "cowlib": {:hex, :cowlib, "2.17.1", "3e6053016d1ab245730f0af688755476dcedb1c25ed8fb5751f59a2bfdc0c9af", [:make, :rebar3], [], "hexpm", "ff08bd17e6dd931445b18af77315b9b5fe052407110964ad2588c686b57b5e3f"}, @@ -38,6 +39,7 @@ "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, + "makeup_syntect": {:hex, :makeup_syntect, "0.1.4", "e1230c9e0513c667b226b21c83eb182e1ab581f65af9441edab1f9ac626acba6", [:mix], [{:makeup, "~> 1.2", [hex: :makeup, repo: "hexpm", optional: false]}, {:rustler, "~> 0.37.1", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.8.2", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "5b624a434d9665786b9a352a5f3502b6c98e1996ede9936b20035ec140daef70"}, "merkle_map": {:hex, :merkle_map, "0.2.2", "f36ff730cca1f2658e317a3c73406f50bbf5ac8aff54cf837d7ca2069a6e251c", [:mix], [], "hexpm", "383107f0503f230ac9175e0631647c424efd027e89ea65ab5ea12eeb54257aaf"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mint": {:hex, :mint, "1.9.0", "d6f534c2a3e98b2a8cc749b4796eb77e9e3af79a76f96e4c74035a827de0d318", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "007154c7d8c43916aed3c93afd1f11aebbaa9c5ff4b7ba55ebe0d17ee0296042"}, @@ -57,6 +59,7 @@ "protobuf": {:hex, :protobuf, "0.17.0", "39e24e43c9648e148feba16ed51100b5b2028ea900b55460377b0476f6e10613", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "ca6c91f6f63e2c147b47f03eefd10b80538aa6fc55ff4b12b795efb786b0152f"}, "ranch": {:hex, :ranch, "2.2.0", "25528f82bc8d7c6152c57666ca99ec716510fe0925cb188172f41ce93117b1b0", [:make, :rebar3], [], "hexpm", "fa0b99a1780c80218a4197a59ea8d3bdae32fbff7e88527d7d8a4787eff4f8e7"}, "req": {:hex, :req, "0.6.2", "b9b2024f35bcf60a92cc8cad2eaaf9d4e7aace463ff74be1afe5986830184413", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.21", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cc9cd30a2ddd04989929b887178e1610c940456d962c6c3a52df6146d2eef9bf"}, + "rustler_precompiled": {:hex, :rustler_precompiled, "0.8.4", "700a878312acfac79fb6c572bb8b57f5aae05fe1cf70d34b5974850bbf2c05bf", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "3b33d99b540b15f142ba47944f7a163a25069f6d608783c321029bc1ffb09514"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, diff --git a/test/hyper/cfg/resolver_test.exs b/test/hyper/cfg/resolver_test.exs new file mode 100644 index 00000000..41cabbea --- /dev/null +++ b/test/hyper/cfg/resolver_test.exs @@ -0,0 +1,45 @@ +defmodule Hyper.Cfg.ResolverTest do + use ExUnit.Case, async: false + + import Hyper.Cfg, only: [get_cfg: 1] + + setup do + on_exit(fn -> Application.delete_env(:hyper, :__cfg_test) end) + end + + test "list order is priority: runtime wins over toml wins over default" do + Application.put_env(:hyper, :__cfg_test, "from_runtime") + + assert get_cfg(runtime: :__cfg_test, toml: "nope", default: "d") == "from_runtime" + end + + test "falls through absent runtime to the default" do + Application.delete_env(:hyper, :__cfg_test) + + assert get_cfg(runtime: :__cfg_test, default: "d") == "d" + end + + test "nested {mod, key} runtime source reads a keyword under the module env" do + Application.put_env(:hyper, __MODULE__, foo: "bar") + + assert get_cfg(runtime: {__MODULE__, :foo}, default: "d") == "bar" + assert get_cfg(runtime: {__MODULE__, :absent}, default: "d") == "d" + after + Application.delete_env(:hyper, __MODULE__) + end + + test "a required key (no default) with every source absent raises a named error" do + assert_raise Hyper.Cfg.MissingError, ~r/required/, fn -> + get_cfg(toml: "definitely.absent") + end + end + + test "fetch_cfg/1 returns {:ok, value} or :error without raising" do + Application.put_env(:hyper, :__cfg_test, "v") + assert Hyper.Cfg.fetch_cfg(runtime: :__cfg_test) == {:ok, "v"} + Application.delete_env(:hyper, :__cfg_test) + assert Hyper.Cfg.fetch_cfg(runtime: :__cfg_test) == :error + assert Hyper.Cfg.fetch_cfg(toml: "definitely.absent") == :error + assert Hyper.Cfg.fetch_cfg(runtime: :__cfg_test, default: "d") == {:ok, "d"} + end +end diff --git a/test/hyper/cfg/toml_test.exs b/test/hyper/cfg/toml_test.exs new file mode 100644 index 00000000..6c9fb6f8 --- /dev/null +++ b/test/hyper/cfg/toml_test.exs @@ -0,0 +1,35 @@ +defmodule Hyper.Cfg.TomlTest do + use ExUnit.Case, async: false + + alias Hyper.Cfg.Toml + + setup do + on_exit(fn -> Toml.reload() end) + :ok + end + + @cfg %{"work_dir" => "/data", "tools" => %{"firecracker" => "/opt/fc"}} + + for {path, expected} <- [ + {"work_dir", {:ok, "/data"}}, + {"tools.firecracker", {:ok, "/opt/fc"}}, + {"tools.jailer", :error}, + {"tools.firecracker.extra", :error}, + {"missing", :error} + ] do + test "fetch_in #{inspect(path)} -> #{inspect(expected)}" do + assert Toml.fetch_in(unquote(Macro.escape(@cfg)), unquote(path)) == + unquote(Macro.escape(expected)) + end + end + + test "fetch/1 reads the seeded cache; absent keys return :error" do + Toml.put_cache(@cfg) + assert Toml.fetch("work_dir") == {:ok, "/data"} + assert Toml.fetch("tools.firecracker") == {:ok, "/opt/fc"} + assert Toml.fetch("tools.jailer") == :error + + Toml.put_cache(%{}) + assert Toml.fetch("work_dir") == :error + end +end diff --git a/test/hyper/cfg/vm_linux_test.exs b/test/hyper/cfg/vm_linux_test.exs new file mode 100644 index 00000000..0569d927 --- /dev/null +++ b/test/hyper/cfg/vm_linux_test.exs @@ -0,0 +1,29 @@ +defmodule Hyper.Cfg.VmLinuxTest do + use ExUnit.Case, async: false + + alias Hyper.Cfg.VmLinux + + setup do + Application.delete_env(:hyper, VmLinux) + Hyper.Cfg.Toml.put_cache(%{}) + + on_exit(fn -> + Application.delete_env(:hyper, VmLinux) + Hyper.Cfg.Toml.reload() + end) + + :ok + end + + test "maps the amd64/aarch64 doc keys to their arch atoms" do + Application.put_env(:hyper, VmLinux, amd64: "/k/amd64", aarch64: "/k/arm64") + assert VmLinux.images() == %{x86_64: "/k/amd64", aarch64: "/k/arm64"} + end + + test "omits an architecture that has no configured path" do + assert VmLinux.images() == %{} + + Application.put_env(:hyper, VmLinux, amd64: "/k/amd64") + assert VmLinux.images() == %{x86_64: "/k/amd64"} + end +end diff --git a/test/hyper/img/oci_loader_test.exs b/test/hyper/img/oci_loader_test.exs index 89277779..d959a4dd 100644 --- a/test/hyper/img/oci_loader_test.exs +++ b/test/hyper/img/oci_loader_test.exs @@ -2,7 +2,7 @@ defmodule Hyper.Img.OciLoaderTest do use ExUnit.Case, async: false use ExUnitProperties - alias Hyper.Config + alias Hyper.Cfg.Dirs alias Hyper.Img.Db.{Blob, Repo} alias Hyper.Img.OciLoader alias Unit.Information @@ -35,7 +35,7 @@ defmodule Hyper.Img.OciLoaderTest do assert {:ok, id} = OciLoader.load("docker.io/library/busybox:1.36") - path = Path.join(Config.layer_dir(), "layer_#{id}.img") + path = Path.join(Dirs.layer_dir(), "layer_#{id}.img") assert File.exists?(path) assert File.stat!(path).size > 0 diff --git a/test/hyper/node/fire_vmm/vm_linux/provider_test.exs b/test/hyper/node/fire_vmm/vm_linux/provider_test.exs index 4bd99330..7ddf1db6 100644 --- a/test/hyper/node/fire_vmm/vm_linux/provider_test.exs +++ b/test/hyper/node/fire_vmm/vm_linux/provider_test.exs @@ -35,12 +35,12 @@ defmodule Hyper.Node.FireVMM.VmLinux.ProviderTest do test "default_path/1 resolves under the configured install dir", %{builds: _} do assert {:ok, path} = Provider.default_path(:x86_64) - assert path == Path.join(Hyper.Config.vmlinux_install_dir(), "vmlinux-x86_64-6.1") + assert path == Path.join(Hyper.Cfg.Dirs.vmlinux_install_dir(), "vmlinux-x86_64-6.1") end test "path/1 resolves a known build under the install dir" do assert Provider.path("x86_64-6.1") == - {:ok, Path.join(Hyper.Config.vmlinux_install_dir(), "vmlinux-x86_64-6.1")} + {:ok, Path.join(Hyper.Cfg.Dirs.vmlinux_install_dir(), "vmlinux-x86_64-6.1")} end test "path/1 rejects an unknown build name" do diff --git a/test/unit/bandwidth_parse_test.exs b/test/unit/bandwidth_parse_test.exs new file mode 100644 index 00000000..86ddda76 --- /dev/null +++ b/test/unit/bandwidth_parse_test.exs @@ -0,0 +1,31 @@ +defmodule Unit.BandwidthParseTest do + use ExUnit.Case, async: true + use ExUnitProperties + + alias Unit.Bandwidth + + for {input, expected} <- [ + {"100Bps", Bandwidth.bps(100)}, + {"4KiBps", Bandwidth.kibps(4)}, + {"512MiBps", Bandwidth.mibps(512)}, + {"1GiBps", Bandwidth.gibps(1)}, + {"1 GiBps", Bandwidth.gibps(1)} + ] do + test "parses #{inspect(input)}" do + assert Bandwidth.parse!(unquote(input)) == unquote(Macro.escape(expected)) + end + end + + for input <- ["1GiB", "fast", ""] do + test "rejects #{inspect(input)}" do + assert {:error, _} = Bandwidth.parse(unquote(input)) + assert_raise ArgumentError, fn -> Bandwidth.parse!(unquote(input)) end + end + end + + property "parse! inverts the gibps constructor across a range" do + check all(n <- integer(0..1024)) do + assert Bandwidth.parse!("#{n}GiBps") == Bandwidth.gibps(n) + end + end +end diff --git a/test/unit/information_parse_test.exs b/test/unit/information_parse_test.exs new file mode 100644 index 00000000..5850794e --- /dev/null +++ b/test/unit/information_parse_test.exs @@ -0,0 +1,32 @@ +defmodule Unit.InformationParseTest do + use ExUnit.Case, async: true + use ExUnitProperties + + alias Unit.Information + + for {input, expected} <- [ + {"100B", Information.bytes(100)}, + {"4KiB", Information.kib(4)}, + {"512MiB", Information.mib(512)}, + {"4GiB", Information.gib(4)}, + {"2TiB", Information.tib(2)}, + {"4 GiB", Information.gib(4)} + ] do + test "parses #{inspect(input)}" do + assert Information.parse!(unquote(input)) == unquote(Macro.escape(expected)) + end + end + + for input <- ["", "GiB", "4 Gigs", "4.5GiB", "nope"] do + test "rejects #{inspect(input)}" do + assert {:error, _} = Information.parse(unquote(input)) + assert_raise ArgumentError, fn -> Information.parse!(unquote(input)) end + end + end + + property "parse! inverts the gib constructor across a range" do + check all(n <- integer(0..4096)) do + assert Information.parse!("#{n}GiB") == Information.gib(n) + end + end +end diff --git a/test/unit/time_parse_test.exs b/test/unit/time_parse_test.exs new file mode 100644 index 00000000..f847de98 --- /dev/null +++ b/test/unit/time_parse_test.exs @@ -0,0 +1,33 @@ +defmodule Unit.TimeParseTest do + use ExUnit.Case, async: true + use ExUnitProperties + + alias Unit.Time + + for {input, expected} <- [ + {"500ns", Time.ns(500)}, + {"100us", Time.us(100)}, + {"100ms", Time.ms(100)}, + {"60s", Time.s(60)}, + {"30m", Time.s(30 * 60)}, + {"1h", Time.s(3600)}, + {"1 h", Time.s(3600)} + ] do + test "parses #{inspect(input)}" do + assert Time.parse!(unquote(input)) == unquote(Macro.escape(expected)) + end + end + + for input <- ["5", "5 secs", "soon", ""] do + test "rejects #{inspect(input)}" do + assert {:error, _} = Time.parse(unquote(input)) + assert_raise ArgumentError, fn -> Time.parse!(unquote(input)) end + end + end + + property "parse! inverts the s constructor across a range" do + check all(n <- integer(0..100_000)) do + assert Time.parse!("#{n}s") == Time.s(n) + end + end +end