chore: Get a VM running

docs/cookbook/intro.md

@@ -13,51 +13,41 @@ of the aforementioned systems.
 The absolute best way to understand `Hyper` and how it works is to play around
 with it.
 
-## Getting Started
+#### Auto-redistributed
 
-The absolute best way to get started with `Hyper` is to play with it.
+`umoci` and the guest `vmlinux` kernels are downloaded, checksum-verified, and
+managed by Hyper itself; you do not install them.
 
-### Requirements
-
-Hyper requires the following software be installed on each node running it:
-
-  - [`skopeo`](https://github.com/containers/skopeo)
-  - [`e2fsprogs`](https://github.com/tytso/e2fsprogs)
-
-Hyper has more runtime dependencies, but they are automatically redistributed
-by Hyper.
+`firecracker` and `jailer` are not auto-downloaded. Install them with
+`mix firecracker.install [--prefix <dir>]` (default prefix `/opt/firecracker`),
+which downloads the pinned v1.16.0 release, places the binaries at
+`<prefix>/firecracker` and `<prefix>/jailer`, and prints the `/etc/hyper/config.toml`
+snippet to paste in.
 
 ### Installation
 
 <!-- TODO(markovejnovic): Write this out. -->
 
 ### Configuration
 
-Running `Hyper` is involved and requires a large number of pre-requisites. The
-configuration of `:hyper` can be done by creating a `config :hyper` entry in
-your `config.exs`. Refer to the given snippet for details on each
-configuration.
+Almost all host configuration — `work_dir`, the `[tools]` binary paths
+(`firecracker`, `jailer`, `dmsetup`, ...), and the `[jails]` table (`cgroup`,
+`uid_gid_range`) — lives in `/etc/hyper/config.toml` (the single source of
+truth shared with the setuid helper, shown above), and every node-local path
+(`jails`, `socks`, `scratch`, `layers`) is derived from `work_dir`. None of it is
+repeated in `config :hyper`.
+
+The node's own tool paths (`skopeo`, `mke2fs`, `umoci`, `suidhelper`) now live in
+the `[tools]` table of `/etc/hyper/config.toml` alongside the privileged binaries,
+so `config :hyper` holds only the per-architecture guest kernels (each with a
+default, so the block may be omitted):
 
 ```elixir
 config :hyper,
-  # TODO(markovejnovic): Remove this after it gets auto-downloaded.
-  jailer_bin: "/opt/firecracker/jailer-v1.16.0-x86_64",
-  # TODO(markovejnovic): Remove this after it gets auto-downloaded.
-  firecracker_bin: "/opt/firecracker/firecracker-v1.16.0-x86_64",
-  # You must create a parent cgroup on your system. Continue reading for
-  # further details.
-  cgroup_parent: "hyper",
-  # TODO(markovejnovic): Merge these directories into one.
-  jailer_chroot_base: "/srv/hyper/jails",
-  socket_dir: "/srv/hyper/socks",
-  scratch_dir: "/srv/hyper/scratch",
-  # Hyper requires that each VM you pass 
-  uid_gid_range: {900_000, 999_999},
-  layer_dir: "/srv/hyper/layers"
+  # Per-architecture guest kernel images placed on the host.
+  vmlinux: %{x86_64: "/srv/hyper/redist/vmlinux/vmlinux-x86_64"}
 ```
 
-<!-- TODO(markovejnovic): Update the config section. -->
-
 ### Usage
 
 <!-- TODO(markovejnovic): Write out how to boot hyper etc -->

lib/hyper.ex

@@ -18,7 +18,7 @@ defmodule Hyper do
   @spec create_vm(Hyper.Vm.Spec.t()) :: {:ok, Hyper.Vm.t()} | {:error, term()}
   def create_vm(%Hyper.Vm.Spec{} = spec) do
     with {:ok, arch} <- resolve_arch(spec.arch) do
-      vm_id = gen_vm_id()
+      vm_id = Hyper.Vm.Id.generate()
       spec = %{spec | arch: arch}
       instance_spec = Hyper.Vm.Instance.spec(spec.type)
 
@@ -34,17 +34,13 @@ defmodule Hyper do
     end
   end
 
-  @doc "Generate a fresh VM id (url-safe base64, dm-name compatible)."
-  @spec gen_vm_id() :: Hyper.Vm.id()
-  def gen_vm_id, do: Base.url_encode64(:crypto.strong_rand_bytes(9), padding: false)
-
   @spec resolve_arch(Hyper.Vm.Instance.arch() | nil) ::
           {:ok, Hyper.Vm.Instance.arch()} | {:error, term()}
   defp resolve_arch(nil), do: Sys.Arch.current()
   defp resolve_arch(arch), do: {:ok, arch}
 
   @doc "Cluster-wide: which node currently runs `vm_id`? `nil` if unknown."
-  @spec whereis(Hyper.Vm.id()) :: node() | nil
+  @spec whereis(Hyper.Vm.Id.t()) :: node() | nil
   def whereis(vm_id), do: Hyper.Cluster.Routing.whereis(vm_id)
 
   @doc """
@@ -57,7 +53,7 @@ defmodule Hyper do
   died with its host, so "unknown" is the truthful answer. Only `:erpc`'s own
   transport failures are swallowed; a genuine fault in the lookup still raises.
   """
-  @spec id(Hyper.Vm.t()) :: Hyper.Vm.id() | nil
+  @spec id(Hyper.Vm.t()) :: Hyper.Vm.Id.t() | nil
   def id(pid) when is_pid(pid) do
     :erpc.call(node(pid), Hyper.Cluster.Routing, :id_for, [pid])
   catch

lib/hyper/cluster/routing.ex

@@ -28,8 +28,28 @@ defmodule Hyper.Cluster.Routing do
   @spec via(term()) :: {:via, module(), {atom(), term()}}
   def via(key), do: {:via, Horde.Registry, {@name, key}}
 
+  @doc """
+  Register the calling process under `key` from inside its own `init`.
+
+  Prefer this over starting a process with a `{:via, Horde.Registry, _}` name.
+  OTP's post-start name check (`gen:get_proc_name`) calls `whereis_name`
+  immediately after the synchronous `register`, but Horde materialises the name
+  into its local ETS only asynchronously, via the DeltaCRDT diff loop. Under
+  registry churn that read loses the race and OTP aborts startup with
+  `{:process_not_registered_via, Horde.Registry}`. Registering from within
+  `init` carries no such self-check, while leaving the name cluster-resolvable
+  through `via/1` once the diff propagates (callers already tolerate that lag).
+  """
+  @spec register_self(term()) :: :ok | {:error, {:already_registered, pid()}}
+  def register_self(key) do
+    case Horde.Registry.register(@name, key, nil) do
+      {:ok, _pid} -> :ok
+      {:error, {:already_registered, _pid}} = err -> err
+    end
+  end
+
   @doc "Which node currently runs `vm_id`? `nil` if unknown."
-  @spec whereis(Hyper.Vm.id()) :: node() | nil
+  @spec whereis(Hyper.Vm.Id.t()) :: node() | nil
   @decorate with_span("Hyper.Cluster.Routing.whereis", include: [:vm_id])
   def whereis(vm_id) do
     case Horde.Registry.lookup(@name, {vm_id, :supervisor}) do
@@ -43,7 +63,7 @@ defmodule Hyper.Cluster.Routing do
   replica via a registry match spec; intended to be called on the node that owns
   `pid` (see `Hyper.id/1`).
   """
-  @spec id_for(pid()) :: Hyper.Vm.id() | nil
+  @spec id_for(pid()) :: Hyper.Vm.Id.t() | nil
   @decorate with_span("Hyper.Cluster.Routing.id_for")
   def id_for(pid) when is_pid(pid) do
     case Horde.Registry.select(@name, [
@@ -55,7 +75,7 @@ defmodule Hyper.Cluster.Routing do
   end
 
   @doc "Every VM the cluster currently knows about, paired with the node it runs on."
-  @spec all() :: [{Hyper.Vm.id(), node()}]
+  @spec all() :: [{Hyper.Vm.Id.t(), node()}]
   @decorate with_span("Hyper.Cluster.Routing.all")
   def all do
     @name

lib/hyper/cluster/scheduler.ex

@@ -16,6 +16,8 @@ defmodule Hyper.Cluster.Scheduler do
   alias Hyper.Vm.Instance.Spec
   alias Unit.Information
 
+  require Logger
+
   use OpenTelemetryDecorator
 
   @type layer_sizes :: [{Hyper.Layer.id(), Unit.Information.t()}]
@@ -45,8 +47,15 @@ defmodule Hyper.Cluster.Scheduler do
     |> candidates(layers)
     |> Enum.reduce_while({:error, :no_capacity}, fn node, acc ->
       case attempt.(node) do
-        {:ok, result} -> {:halt, {:ok, {node, result}}}
-        {:error, _reason} -> {:cont, acc}
+        {:ok, result} ->
+          {:halt, {:ok, {node, result}}}
+
+        {:error, reason} ->
+          # The candidate fit the snapshot but refused at confirmation time.
+          # Log the real reason: otherwise an actual boot failure on the only
+          # candidate is indistinguishable from genuine `:no_capacity`.
+          Logger.warning("scheduler: #{inspect(node)} refused placement: #{inspect(reason)}")
+          {:cont, acc}
       end
     end)
   end

lib/hyper/grpc/codec.ex

@@ -85,15 +85,15 @@ defmodule Hyper.Grpc.Codec do
   end
 
   @doc "Convert a domain result to an outbound response message, or an error to `GRPC.RPCError`."
-  @spec to_grpc({:created, Hyper.Vm.id(), node()}) :: CreateVmResponse.t()
+  @spec to_grpc({:created, Hyper.Vm.Id.t(), node()}) :: CreateVmResponse.t()
   def to_grpc({:created, vm_id, node}) when is_binary(vm_id),
     do: %CreateVmResponse{vm_id: vm_id, node: to_string(node)}
 
-  @spec to_grpc({:located, Hyper.Vm.id(), node()}) :: GetVmResponse.t()
+  @spec to_grpc({:located, Hyper.Vm.Id.t(), node()}) :: GetVmResponse.t()
   def to_grpc({:located, vm_id, node}),
     do: %GetVmResponse{vm_id: vm_id, node: to_string(node)}
 
-  @spec to_grpc({:vms, [{Hyper.Vm.id(), node()}]}) :: ListVmsResponse.t()
+  @spec to_grpc({:vms, [{Hyper.Vm.Id.t(), node()}]}) :: ListVmsResponse.t()
   def to_grpc({:vms, vms}),
     do: %ListVmsResponse{vms: Enum.map(vms, &vm/1)}
 
@@ -117,7 +117,7 @@ defmodule Hyper.Grpc.Codec do
   def to_grpc({:exit, {:nodedown, _}}), do: rpc_error(:machine_unreachable)
   def to_grpc({:exit, reason}), do: rpc_error({:stop_failed, reason})
 
-  @spec vm({Hyper.Vm.id(), node()}) :: Vm.t()
+  @spec vm({Hyper.Vm.Id.t(), node()}) :: Vm.t()
   defp vm({vm_id, node}), do: %Vm{vm_id: vm_id, node: to_string(node)}
 
   @spec instance_type(instance_enum()) :: {:ok, Hyper.Vm.Instance.t()}

lib/hyper/img/db/lease.ex

@@ -49,7 +49,7 @@ defmodule Hyper.Img.Db.Lease do
   Upserts on `(node_id, vm_id)` - the same call both takes a fresh lease and
   heartbeats a live one.
   """
-  @spec bump(Hyper.Img.id(), Hyper.Vm.id(), Unit.Time.t()) ::
+  @spec bump(Hyper.Img.id(), Hyper.Vm.Id.t(), Unit.Time.t()) ::
           {:ok, %__MODULE__{}} | {:error, Ecto.Changeset.t()}
   @decorate with_span("Hyper.Img.Db.Lease.bump", include: [:image_id, :vm_id])
   def bump(image_id, vm_id, ttl) do
@@ -72,7 +72,7 @@ defmodule Hyper.Img.Db.Lease do
   Release the lease issued to the given node_id and the given vm_id. Since each VM only ever uses
   one image, it is not necessary to specify the image id.
   """
-  @spec release(Hyper.Vm.id()) :: :ok
+  @spec release(Hyper.Vm.Id.t()) :: :ok
   @decorate with_span("Hyper.Img.Db.Lease.release", include: [:vm_id])
   def release(vm_id) do
     query = from(l in __MODULE__, where: l.node_id == ^to_string(node()) and l.vm_id == ^vm_id)

lib/hyper/node.ex

@@ -26,6 +26,11 @@ defmodule Hyper.Node do
       real-time monitors backing the soft budget (`Hyper.Node.Budget.Soft`).
       Lives here, not at the application root, because both are per-node and only
       meaningful while this node hosts VMs.
+
+    * `Hyper.Node.Reaper` - a periodic, liveness-aware GC for per-VM host
+      resources (orphaned firecracker cgroups and `hyper-rw-*` dm volumes) stranded
+      by an unclean death whose vm_id never reboots. Started last so the VM
+      supervisor it consults for liveness is already up.
   """
 
   use Supervisor
@@ -40,19 +45,31 @@ defmodule Hyper.Node do
 
   def start_link(opts \\ []) do
     case test_system() do
-      :ok -> Supervisor.start_link(__MODULE__, opts, name: __MODULE__)
-      {:error, reason} -> {:error, reason}
+      :ok ->
+        # Clear any dm/loop devices a previous unclean shutdown left behind,
+        # before the device-owning children start and collide with them.
+        :ok = Hyper.Node.Reclaim.run()
+        Supervisor.start_link(__MODULE__, opts, name: __MODULE__)
+
+      {:error, reason} ->
+        {:error, reason}
     end
   end
 
   @impl true
   def init(_opts) do
     children = [
       Hyper.Node.Users,
+      # Layer owns Hyper.Node.Layer.Registry, which Budget.Advertiser queries
+      # (via Hyper.Node.Layer.active/0) as it advertises on init - so Layer must
+      # be up first.
+      Hyper.Node.Layer,
       Hyper.Node.Budget.Supervisor,
       {DynamicSupervisor, name: @vm_sup, strategy: :one_for_one},
-      Hyper.Node.Layer,
-      Hyper.Node.Img
+      Hyper.Node.Img,
+      # Last child: :one_for_one starts in order, so the VM supervisor and Img are
+      # up before the reaper's first tick can read their liveness.
+      Hyper.Node.Reaper
     ]
 
     Supervisor.init(children, strategy: :one_for_one)
@@ -63,7 +80,7 @@ defmodule Hyper.Node do
   layer, resolve the kernel, and start the VM supervisor. The uid is freed and
   the mutable layer torn down automatically when the VM supervisor dies.
   """
-  @spec start_image_vm(Hyper.Vm.id(), Hyper.Vm.Spec.t()) :: {:ok, pid()} | {:error, term()}
+  @spec start_image_vm(Hyper.Vm.Id.t(), Hyper.Vm.Spec.t()) :: {:ok, pid()} | {:error, term()}
   @decorate with_span("Hyper.Node.start_image_vm", include: [:vm_id, :spec])
   def start_image_vm(vm_id, %Hyper.Vm.Spec{} = spec) do
     with {:ok, uid} <- Users.claim(),
@@ -89,7 +106,7 @@ defmodule Hyper.Node do
   end
 
   @doc false
-  @spec build_opts(Hyper.Vm.id(), Hyper.Vm.Spec.t(), Users.id(), pid(), Path.t()) ::
+  @spec build_opts(Hyper.Vm.Id.t(), Hyper.Vm.Spec.t(), Users.id(), pid(), Path.t()) ::
           FireVMM.Opts.t()
   def build_opts(vm_id, %Hyper.Vm.Spec{} = spec, uid, mutable, kernel) do
     %FireVMM.Opts{
@@ -144,10 +161,11 @@ defmodule Hyper.Node do
   @spec test_system :: :ok | {:error, term()}
   def test_system do
     with {:ok, _} <- Hyper.Cfg.Budget.load(),
-         :ok <- Hyper.Node.FireVMM.Provider.ensure_installed(),
+         :ok <- check_firecracker_bins(),
          :ok <- Hyper.Node.FireVMM.VmLinux.Provider.ensure_installed(),
          :ok <- Hyper.Node.Vmlinux.test_system(),
          :ok <- Hyper.Img.OciLoader.Umoci.ensure_installed(),
+         :ok <- Hyper.Img.OciLoader.test_system(),
          :ok <- Hyper.Node.Users.test_system(),
          :ok <- Hyper.Node.Layer.Repo.test_system(),
          :ok <- Hyper.SuidHelper.test_system(),
@@ -157,6 +175,24 @@ defmodule Hyper.Node do
     end
   end
 
+  @spec check_firecracker_bins ::
+          :ok
+          | {:error, {:firecracker_bin_missing | :jailer_bin_missing, Path.t()}}
+          | {:error, :firecracker_not_configured | :jailer_not_configured}
+  defp check_firecracker_bins do
+    with {:fc, {:ok, fc}} <- {:fc, Hyper.Cfg.Tools.firecracker_configured()},
+         {:jail, {:ok, jail}} <- {:jail, Hyper.Cfg.Tools.jailer_configured()} do
+      cond do
+        not Sys.Posix.executable?(fc) -> {:error, {:firecracker_bin_missing, fc}}
+        not Sys.Posix.executable?(jail) -> {:error, {:jailer_bin_missing, jail}}
+        true -> :ok
+      end
+    else
+      {:fc, :error} -> {:error, :firecracker_not_configured}
+      {:jail, :error} -> {:error, :jailer_not_configured}
+    end
+  end
+
   @spec check_helper_base(Path.t()) ::
           :ok | {:error, {:suid_helper_base_mismatch, Path.t(), Path.t()}}
   defp check_helper_base(base) do

lib/hyper/node/fire_vmm.ex

@@ -34,7 +34,7 @@ defmodule Hyper.Node.FireVMM do
     defstruct [:vm_id, :uid, :gid, :type, :arch, :mutable, :kernel, :boot_args]
 
     @type t :: %__MODULE__{
-            vm_id: Hyper.Vm.id(),
+            vm_id: Hyper.Vm.Id.t(),
             uid: Hyper.Node.Users.id(),
             gid: Hyper.Node.Users.id(),
             type: Hyper.Vm.Instance.t(),
@@ -47,14 +47,15 @@ defmodule Hyper.Node.FireVMM do
 
   @spec start_link(Opts.t()) :: Supervisor.on_start()
   def start_link(opts) do
-    Supervisor.start_link(__MODULE__, opts, name: via(opts.vm_id))
+    Supervisor.start_link(__MODULE__, opts)
   end
 
+  @spec child_spec(Opts.t()) :: Supervisor.child_spec()
   def child_spec(opts) do
     # Keyed by VM id and :transient so a cleanly-stopped VM is not rebooted by
     # the node-level DynamicSupervisor.
     %{
-      vm_id: {__MODULE__, opts.vm_id},
+      id: {__MODULE__, opts.vm_id},
       start: {__MODULE__, :start_link, [opts]},
       type: :supervisor,
       restart: :transient
@@ -63,18 +64,26 @@ defmodule Hyper.Node.FireVMM do
 
   @impl true
   def init(opts) do
-    children = [
-      # Client must be registered before Core: Core starts the State machine,
-      # which calls Client.run while waiting for the daemon's API. Client depends
-      # only on vm_id (an independent peer), so it has no reverse dependency.
-      {Client, %Client.Opts{vm_id: opts.vm_id}},
-      {Core, opts}
-    ]
+    # Self-register the cluster routing entry here rather than via a start name;
+    # see `Hyper.Cluster.Routing.register_self/1`. A fresh random vm_id never
+    # collides, so `:already_registered` only happens against a stale dead
+    # incarnation - decline the start and let the supervisor retry clean.
+    case Hyper.Cluster.Routing.register_self({opts.vm_id, :supervisor}) do
+      :ok ->
+        children = [
+          # Client must be registered before Core: Core starts the State machine,
+          # which calls Client.run while waiting for the daemon's API. Client
+          # depends only on vm_id (an independent peer), so no reverse dependency.
+          {Client, %Client.Opts{vm_id: opts.vm_id}},
+          {Core, opts}
+        ]
 
-    Supervisor.init(children, strategy: :one_for_one)
-  end
+        Supervisor.init(children, strategy: :one_for_one)
 
-  defp via(vm_id), do: Hyper.Cluster.Routing.via({vm_id, :supervisor})
+      {:error, _} ->
+        :ignore
+    end
+  end
 
   @doc "Test whether the system can run firecracker VMMs."
   @spec test_system() :: :ok | {:error, term()}