"""Whether there is room to hold one more provider. This is what the budgets are for.
Every manifest declares `[gpu] vram_budget_gb`, measured rather than guessed, and
STANDARD.md §6 says it is "what a host adds up to decide how many providers may
be resident at once". Twenty-seven providers now declare one and nothing added
them up: `backend.running()` returns `{slug: budget}` with a docstring saying a
scheduler above it reads that, and no caller. This module is the addition.
IT EXISTS BECAUSE THE MACHINE WAS WEDGED, by this project, with the numbers in
hand. Four providers' workers were held, their RSS summed to about 12 GiB, and a
fifth provider's 28 GiB smoke was started on top. The host stopped responding.
Two things made that possible and both are requirements here:
1. **The sum of what THIS process holds is not what the machine holds.** The
workers that turned 12 GiB into a wedge were `boltzgen` workers from a
different checkout entirely, started by a session that had no reason to
announce them. `backend.running()` is blind in exactly the direction that
caused the harm, so the holdings come from `machine_holdings` -- the process
table, which sees every checkout's workers and counts two processes of one
provider as two holders. The two gates then cover different unknowns: the
declared sum protects against peaks that have not happened yet, and the live
reading protects against everything on this machine that is not a resident
worker at all.
2. **On unified memory the two budgets are one budget.** `nvidia-smi
--query-gpu=memory.total` answers `[N/A]` on this GB10 because device memory
IS system memory: 121 GiB for the host and the card together. So capacity is
`MemTotal`, a VRAM budget is spent from the same pool as a page cache, and a
provider's budget cannot be checked against a device total that does not
exist. STANDARD.md already recorded that this breaks a kit's own arithmetic;
it breaks a scheduler's the same way one level up.
THE LIVE GATE ASKS A NARROWER QUESTION WHEN IT HAS CO-TENANTS, which is the
second thing this project got wrong before measuring it. The first guard gated
on `MemAvailable` and waved a worker through at 60.4 GiB available with 5 GiB
free; it died with `CUDA error: out of memory`, because the other 56 was
reclaimable page cache that a device allocation did not get to reclaim in time.
Changing it to `MemFree` then made it too strict: a bounded probe allocated to 24
GiB starting from 18.7 GiB free, with `Cached` falling 72.2 -> 63.4 while free
held near 4 -- reclaim works. Both readings are true and they answer different
questions, so which one gates depends on whether anything else is competing for
the reclaim: `MemFree` when there are co-tenants, because losing a race to one
is what the OOM was, and `MemAvailable` when there are none, because then
reclaim is not a race.
THE LIMIT OF THAT EVIDENCE, stated because the rule extrapolates past it: the
uncontended probe reached 24 GiB starting from 18.7 GiB free. An uncontended
admission of more than that is trusting the kernel's own estimate of what it can
reclaim, which is exactly what `MemAvailable` is, and which has not been
falsified here -- but it has only been confirmed once and at one size. A host
that wants a harder floor passes a larger `reserve_gb`; the honest default is
not to invent a bound nobody measured.
A refusal here names the numbers and does not round them, per §8.
"""
from __future__ import annotations
import os
import re
import subprocess
from dataclasses import dataclass, field
from typing import Dict, Mapping, Optional, Tuple
GIB = 1024 * 1024 * 1024
[docs]
class AdmissionRefused(RuntimeError):
"""There is not room to hold this provider. The message carries the numbers."""
[docs]
@dataclass
class Verdict:
"""The decision, the reason, and every number it was made from."""
admit: bool
reason: str
slug: str = ""
budget_gb: Optional[float] = None
capacity_gib: Optional[float] = None
capacity_source: str = ""
# `held_gb`, not `declared_held_gb`: on a machine whose pool is its own RAM
# the holdings are read from the process table, and only on a discrete device
# is this sum the declared budgets it was originally named for.
held_gb: float = 0.0
free_gib: Optional[float] = None
available_gib: Optional[float] = None
gate: str = ""
co_tenants: int = 0
unknown_budgets: Tuple[str, ...] = field(default_factory=tuple)
def __bool__(self) -> bool: # `if decide(...):` reads as intended
return self.admit
[docs]
def raise_if_refused(self) -> "Verdict":
if not self.admit:
raise AdmissionRefused(self.reason)
return self
[docs]
def meminfo_gib() -> Dict[str, float]:
"""`MemTotal`, `MemFree` and `MemAvailable` in GiB, from /proc/meminfo."""
want = {"MemTotal": None, "MemFree": None, "MemAvailable": None}
with open("/proc/meminfo") as fh:
for line in fh:
key, _, rest = line.partition(":")
if key in want:
want[key] = int(rest.split()[0]) / (1024 * 1024)
missing = [k for k, v in want.items() if v is None]
if missing:
raise RuntimeError(f"/proc/meminfo has no {', '.join(missing)}")
return want # type: ignore[return-value]
# Where a capacity figure came from, named because callers act on the difference and not
# only on the number: on unified memory a model's weights, a CPU-only provider's RSS and
# the page cache are all spent from the one pool, and on a discrete card they are not.
DEVICE_CAPACITY_SOURCE = "nvidia-smi memory.total (discrete device)"
UNIFIED_CAPACITY_SOURCE = "/proc/meminfo MemTotal (unified memory)"
[docs]
def pool_is_shared_with_the_host() -> Optional[bool]:
"""Whether a model's memory and this machine's RAM come out of one pool.
WHICH DECIDES WHAT ``[gpu] policy = "none"`` IS WORTH AS A COST. Three of this
catalog's manifests declare it, each off a measurement: their kit has no
GPU-capable wheel for this architecture, so they hold no device memory at all. One
of them says in as many words that it is therefore "admitted while the card is
full", and on a discrete-GPU host that is exactly right. On unified memory it is
not: that provider's ~10 GiB of host RSS is spent from the same pool the next
model's weights are, so "no device memory" stops meaning "no cost".
True on unified memory, False on a discrete device, None where the machine cannot
be read -- the same three machines :func:`capacity_gib` answers for, decided by
which of them it answered from rather than by a second detection that could drift
from the first.
"""
capacity, source = capacity_gib()
if capacity is None:
return None
return source == UNIFIED_CAPACITY_SOURCE
[docs]
def capacity_gib() -> Tuple[Optional[float], str]:
"""The size of the one pool a budget is spent from, and where the figure came from.
On unified memory there is no device total to ask for -- `nvidia-smi` returns
`[N/A]` -- and the honest capacity is `MemTotal`, because a VRAM allocation
and a page cache come out of the same 121 GiB. On a discrete card the device
total is the right ceiling for a VRAM budget and `MemTotal` is not.
Returns `(None, reason)` when neither can be established, which is a thing a
caller must handle rather than a zero it can compare against.
"""
try:
out = subprocess.run(
["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"],
capture_output=True,
text=True,
timeout=15,
)
except (OSError, subprocess.SubprocessError) as exc:
return None, f"nvidia-smi did not run ({type(exc).__name__})"
first = (out.stdout or "").strip().splitlines()
value = first[0].strip() if first else ""
if value and "N/A" not in value:
try:
return float(value) / 1024.0, DEVICE_CAPACITY_SOURCE
except ValueError:
pass
# `[N/A]` is the unified-memory answer, not an error: the device has no total
# of its own because it is sharing the host's.
try:
return meminfo_gib()["MemTotal"], UNIFIED_CAPACITY_SOURCE
except (OSError, RuntimeError) as exc:
return None, f"no device total and no MemTotal ({exc})"
[docs]
def pool_free_gib(timeout: float = 15.0) -> Tuple[Optional[float], str]:
"""How much of that one pool is free right now, and where the figure came from.
The mirror of `capacity_gib`, for the same reason and across the same three
machines. A discrete card knows what it has free and that is the answer. On
unified memory `nvidia-smi` returns `[N/A]` for `memory.free` exactly as it
does for `memory.total`, and the honest reading is `MemAvailable`, because
the model's allocation and the page cache come out of one 121 GiB pool.
A host with no NVIDIA driver at all gets `(None, reason)` rather than its
free RAM: a machine with no device is not a machine with an empty device,
and reporting host RAM as free VRAM would admit a model onto hardware that
cannot hold it. `None` means DO NOT REASON ABOUT MEMORY, which a caller has
to handle as its own case rather than compare against zero.
"""
try:
out = subprocess.run(
["nvidia-smi", "--query-gpu=memory.free", "--format=csv,noheader,nounits"],
capture_output=True,
text=True,
timeout=timeout,
)
except (OSError, subprocess.SubprocessError) as exc:
return None, f"nvidia-smi did not run ({type(exc).__name__})"
if out.returncode != 0:
# A driver that errors is not a card that is empty. Falling through to
# MemAvailable here would report host RAM as free VRAM on a discrete box
# and admit a model the card cannot hold -- so a failed query is the
# "do not reason" answer, not the unified-memory one.
return None, f"nvidia-smi exited {out.returncode}"
values: list[int] = []
unavailable = False
for line in (out.stdout or "").splitlines():
line = line.strip()
if not line:
continue
if "N/A" in line:
unavailable = True
continue
try:
values.append(int(line))
except ValueError:
return None, "nvidia-smi answered with neither a number nor [N/A]"
if values and unavailable:
# One device answered and another did not: admitting on half a view of
# the hardware is worse than not reasoning about memory at all.
return None, "nvidia-smi answered for some devices and [N/A] for others"
if values:
# The most free device, not the sum: one worker lands on one card, so a
# sum would offer room that no single model can use.
return max(values) / 1024.0, "nvidia-smi memory.free (discrete device)"
if not unavailable:
return None, "nvidia-smi listed no devices"
try:
return (
meminfo_gib()["MemAvailable"],
"/proc/meminfo MemAvailable (unified memory)",
)
except (OSError, RuntimeError) as exc:
return None, f"no device free and no MemAvailable ({exc})"
def _nvidia_pids() -> int:
"""How many processes the driver says are holding the card right now.
Counted rather than named: the point is only whether this decision has
competition for reclaim, and the processes that wedged the machine belonged
to another checkout, so their identities were not available anyway.
"""
try:
out = subprocess.run(
["nvidia-smi", "--query-compute-apps=pid", "--format=csv,noheader"],
capture_output=True,
text=True,
timeout=15,
)
except (OSError, subprocess.SubprocessError):
return 0
return len([ln for ln in (out.stdout or "").splitlines() if ln.strip()])
# .../remote/<slug>/resident/<slug>_resident_worker.py, and the segment after
# `worktrees` when the holder is a different checkout of this repository.
SLUG_RE = re.compile(r"^[a-z][a-z0-9_]*$")
def _holder_of(path: str) -> Tuple[str, Optional[str]]:
parts = path.split("/")
slug = parts[-3] if len(parts) >= 3 else "?"
tree = next(
(
parts[i + 1]
for i, seg in enumerate(parts)
if seg == "worktrees" and i + 1 < len(parts)
),
None,
)
return slug, tree
PS_COLUMNS = ["ps", "-eo", "pid,ppid,rss,args"]
def _read_ps() -> Optional[str]:
"""One snapshot of the process table, or None if `ps` could not be run."""
try:
return subprocess.run(
PS_COLUMNS,
capture_output=True,
text=True,
timeout=20,
).stdout
except (OSError, subprocess.SubprocessError):
return None
def _process_rows(ps_output: str) -> list[Tuple[str, str, int, str]]:
"""`(pid, ppid, rss KiB, args)` for each real row of a `PS_COLUMNS` snapshot.
A row is four fields with three leading integers. That excludes `ps`'s own
header, and it excludes a shell whose command line merely quotes a worker's
-- one was in this machine's process table while this was being written, and
it passed every other test a holder is given. The field count is also a
guard and not only a filter: the four names below are unpacked from it, so a
shorter row would raise rather than be ignored, and this module returning
nothing is the wedged host it exists to prevent.
"""
rows = []
for line in (ps_output or "").splitlines():
fields = line.split(None, 3)
if len(fields) < 4 or not all(f.isdigit() for f in fields[:3]):
continue
pid, ppid, rss, args = fields
rows.append((pid, ppid, int(rss), args))
return rows
def _tree_rss_gib(root: str, rows: list[Tuple[str, str, int, str]]) -> float:
"""This process and everything under it, from one snapshot of the table.
ONE SNAPSHOT AND NOT A /PROC WALK PER PROCESS: a walk reads a different
moment for each one, and a child that exits between two of them makes the
sum describe no instant that ever existed. The gate decides in one moment,
so it reads one.
A cycle cannot occur in a parent/child relation, but a snapshot can still
disagree with itself -- a process reparented to init between two columns --
so `seen` is what keeps this from looping.
"""
rss = {pid: kib for pid, _ppid, kib, _args in rows}
children: Dict[str, list[str]] = {}
for pid, ppid, _kib, _args in rows:
children.setdefault(ppid, []).append(pid)
total, seen, frontier = 0, set(), [root]
while frontier:
current = frontier.pop()
if current in seen:
continue
seen.add(current)
total += rss.get(current, 0)
frontier.extend(children.get(current, ()))
return total / (1024 * 1024)
[docs]
def machine_holdings(
*,
ps_output: Optional[str] = None,
budget_of=None,
pool_is_shared=None,
) -> Dict[str, Optional[float]]:
"""Every resident worker on this machine, not only this process's.
THIS IS THE HOLE THE REST OF THIS MODULE WAS WRITTEN AROUND. `backend.running()`
knows what this interpreter started; the workers that wedged this host were
started by a session in another worktree of this same repository, and the
whole mistake was not knowing they were there. The process table does know:
a resident worker is a `*_resident_worker.py` invoked with `--socket`, and
its own path names the provider and the checkout.
Scanning sockets instead would undercount, because the socket path is one per
slug per uid: two checkouts running the same provider are one socket and two
processes, and the second process is the one that turns a comfortable sum
into a wedge. A stale socket also outlives its worker -- there is one in that
directory as this is written -- so a socket is evidence of a past worker and
a process is evidence of a present one.
Keys are `slug` for a holder in this tree and `slug@tree` for one elsewhere,
so a restart can release its own budget without releasing someone else's.
WHAT A HOLDER COSTS, AND WHY A DECLARED BUDGET IS NOT IT ON THIS MACHINE.
A declared `[gpu] vram_budget_gb` is one process's peak DEVICE allocation,
measured on the machine that verified the provider. Where the pool is the
host's own memory -- which `pool_is_shared_with_the_host` answers -- what
admission spends is the RESIDENT SET OF THE WHOLE PROCESS TREE, and those two
are not the same number by a factor of two. Measured live here on
2026-10-26, boltz answering one call:
boltz_mcp_server 0.07 GiB
worker_launch (the worker) 5.70 GiB
worker_launch (its fork) 1.00 GiB
boltz2_opt.affinity_leg --serve 4.91 GiB
worker's tree 11.61 GiB
against `vram_budget_gb = 5.54`. The affinity leg is a SECOND model server,
started by the first call and still up 400 s later, reused by the second
call rather than restarted -- so it is residency, and the
`*_resident_worker.py --socket` pattern cannot see it. Twelve of the resident
workers in this catalog spawn children.
So on a shared pool a holder costs the LARGER of its declared budget and its
tree's RSS, read from the same `ps` snapshot that found it. The declared
figure still matters: it is a PEAK and the reading is the residency of this
moment, so a worker that has not been called yet is costed at what it will
need rather than at the little it holds. On a discrete device the reading is
not a candidate at all -- host RSS is not device memory there, and the
declared budget is the only figure that is.
A slug this checkout does not have, on a machine where the tree cannot be
costed, returns None, which `decide` refuses on rather than treating as free.
The PPID is read with the command line because a launcher that FORKS is one
residency and not two: boltz's worker is started through
`boltz2_opt.worker_launch`, and parent and child both carry the worker path
and `--socket`, so one residency read as `{'boltz': 5.54, 'boltz#2': 5.54}`
-- measured live on this machine -- and this gate refused admissions that
would have fit. A match whose parent is also a match is therefore dropped.
Two INDEPENDENT workers of one slug are still two holders, because neither is
the other's child, and that is the case that wedged this host. A dropped fork
is still inside its parent's tree, so its memory is counted once and by the
holder that owns it.
"""
if ps_output is None:
ps_output = _read_ps()
if ps_output is None:
return {}
if budget_of is None:
budget_of = _declared_budget
if pool_is_shared is None:
pool_is_shared = pool_is_shared_with_the_host
rows = _process_rows(ps_output)
matches: list[Tuple[str, str, str, Optional[str]]] = []
for pid, ppid, _rss, args in rows:
if "resident_worker.py" not in args or "--socket" not in args:
continue
worker = next((tok for tok in args.split() if "/resident/" in tok), "")
if not worker:
continue # a shell wrapper that merely mentions the worker
slug, tree = _holder_of(worker)
# A SHELL WHOSE COMMAND LINE QUOTES A WORKER PATH IS NOT A WORKER, and
# the token test above does not catch it: the first run of this function
# reported a holder called `<slug>`, matched out of the `bash -c` that
# was writing this very function and had the path template in a comment.
# Checking that the file exists would not do -- a worker outlives a
# deleted worktree, and that holder is real -- so the shape of the slug
# is the test, which a template or a glob fails and a deleted checkout
# still passes.
if not SLUG_RE.match(slug):
continue
if tree is not None and not SLUG_RE.match(tree.replace("-", "_")):
tree = None
matches.append((pid, ppid, slug, tree))
shared = pool_is_shared()
holders = {pid for pid, _ppid, _slug, _tree in matches}
out: Dict[str, Optional[float]] = {}
for pid, ppid, slug, tree in matches:
if ppid in holders:
continue # the other half of one forked launcher, counted already
key = f"{slug}@{tree}" if tree else slug
# Two processes of the same slug in the same tree are two holders: the
# second is exactly what made 12 GiB into a wedged host.
suffix = 2
base = key
while key in out:
key = f"{base}#{suffix}"
suffix += 1
declared = budget_of(slug)
if shared is not True:
out[key] = declared
continue
held = _tree_rss_gib(pid, rows)
out[key] = held if declared is None else max(declared, held)
return out
[docs]
def manifest_path(slug: str) -> str:
"""Where this checkout keeps `slug`'s manifest, whether or not it has one."""
return os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"remote",
slug,
"provider.toml",
)
def _manifest(slug: str):
"""`slug`'s loaded manifest, or None when this checkout has none to load."""
from . import manifest as M
path = manifest_path(slug)
if not os.path.exists(path):
return None
try:
return M.load(path)
except Exception: # noqa: BLE001 - a broken manifest is `tu remote check`'s to report
return None
def _declared_budget(slug: str) -> Optional[float]:
"""`[gpu] vram_budget_gb` from this checkout's manifest for `slug`.
None means UNDECLARED, which `decide` reports rather than summing as zero:
a provider whose cost is unknown cannot be added up.
**A CPU-ONLY PROVIDER IS 0.0, NOT None**, and the distinction is the whole
difference between a useful co-tenant and a blocker. A manifest with
`[gpu] policy = "none"` is the one case where an absent `vram_budget_gb` is
legal -- the manifest layer enforces that -- and it means the provider holds
no device memory at all, not that nobody measured it. Mapping it to None
made a held esm_if1 or proteinmpnn worker refuse every GPU provider on the
box with "already held without a declared budget, so what the card is
carrying cannot be summed": exactly inverted, since a CPU provider is the
one thing that can safely be resident while the card is full. Found by
running `tu remote room boltz` with a real proteinmpnn worker up, which
reported `held: {"proteinmpnn": null}` and refused boltz on a card holding
nothing.
Another checkout's copy of the same provider could in principle declare a
different figure; using the local one assumes the checkouts agree, which is
worth stating because it is an assumption and not a reading.
"""
m = _manifest(slug)
if m is None:
return None
if m.vram_budget_gb is None and (m.gpu_policy or "").lower() == "none":
return 0.0
return m.vram_budget_gb
# The public name for the same reading. `_declared_budget` stays because this
# module and its tests call it by that name; anything outside the package asks
# for a declared budget rather than reaching for a private helper.
declared_budget_gb = _declared_budget
[docs]
def declares_resident_worker(slug: str) -> bool:
"""Whether `slug` holds its model in a worker process of its own.
WHICH IS WHAT DECIDES WHETHER A LOADED MODEL CAN BE SEEN FROM OUTSIDE, and that
is the only reason anything outside this module asks. Neither kind of provider
has its model in memory when its port starts answering -- a resident one loads
in a worker that the first request starts, and a provider without a manifest
loads lazily in the server process, on the first request too. The difference is
the evidence: a worker is a process, so `machine_holdings` can say whether it is
up, where a model inside an MCP server leaves nothing a scheduler can read.
So a scheduler measuring what a provider costs uses this to know whether it can
check that a model loaded at all, rather than attributing an empty process to a
model. False for the 28 reviewed slugs with no manifest, and for a manifest that
declares any other mode.
"""
m = _manifest(slug)
return bool(m is not None and m.mode == "resident")
[docs]
def decide(
slug: str,
budget_gb: Optional[float],
held: Optional[Mapping[str, Optional[float]]] = None,
*,
reserve_gb: float = 4.0,
mem: Optional[Mapping[str, float]] = None,
capacity: Optional[Tuple[Optional[float], str]] = None,
co_tenants: Optional[int] = None,
) -> Verdict:
"""Whether `slug` may be started now, given what is already held.
`held` is `{slug: budget}` as `backend.running()` returns it, and a None in
it is an UNDECLARED budget, which is reported rather than treated as zero: a
provider whose cost is unknown cannot be added up, and a sum that silently
drops it would be an admission made on a number nobody measured.
`reserve_gb` is left for the host itself. It is not a safety fudge over the
budgets -- those are already upper bounds -- but the room the operating
system needs to not be the thing that fails: the wedge happened with the
pool nominally sufficient and the machine unable to schedule anything.
The injectable arguments exist so the decision is testable without a card.
"""
held = dict(held or {})
held.pop(slug, None) # restarting what is already held frees its own budget
unknown = tuple(sorted(k for k, v in held.items() if v is None))
held_total = sum(v for v in held.values() if v is not None)
cap, cap_source = capacity if capacity is not None else capacity_gib()
m = dict(mem) if mem is not None else meminfo_gib()
free, avail = m.get("MemFree"), m.get("MemAvailable")
tenants = _nvidia_pids() if co_tenants is None else co_tenants
gate = "MemFree" if tenants else "MemAvailable"
gate_value = free if tenants else avail
v = Verdict(
admit=False,
reason="",
slug=slug,
budget_gb=budget_gb,
capacity_gib=cap,
capacity_source=cap_source,
held_gb=round(held_total, 2),
free_gib=None if free is None else round(free, 1),
available_gib=None if avail is None else round(avail, 1),
gate=gate,
co_tenants=tenants,
unknown_budgets=unknown,
)
if budget_gb is None:
v.reason = (
f"{slug} declares no vram_budget_gb, so there is nothing to add up. "
f"A provider is admitted on a measured budget or not at all: measure "
f"it with scripts/remote_validation/verify_provider.py and put the "
f"figure in [gpu] vram_budget_gb."
)
return v
if unknown:
v.reason = (
f"{', '.join(unknown)} {'is' if len(unknown) == 1 else 'are'} already "
f"held without a declared budget, so what the card is carrying cannot "
f"be summed and admitting {slug} ({budget_gb} GiB) would be a guess. "
f"Declare {'its' if len(unknown) == 1 else 'their'} budget, or stop "
f"{'it' if len(unknown) == 1 else 'them'} before starting {slug}."
)
return v
if cap is None:
v.reason = (
f"the size of the pool {slug}'s {budget_gb} GiB would come out of "
f"could not be established ({cap_source}), so no sum means anything. "
f"On unified memory this is MemTotal; on a discrete card it is the "
f"device total."
)
return v
wanted = held_total + budget_gb + reserve_gb
if wanted > cap:
v.reason = (
f"{slug} needs {budget_gb} GiB and {round(held_total, 2)} GiB is "
f"already held by {len(held)} worker(s); with {reserve_gb} GiB left for the "
f"host that is {round(wanted, 2)} GiB against a pool of "
f"{round(cap, 1)} GiB ({cap_source}). Stop a resident provider, or "
f"run this one on a quieter machine -- do not start it anyway: that "
f"is the sequence that wedged this host, with the budgets in hand."
)
return v
if gate_value is None:
v.reason = f"/proc/meminfo gave no {gate}, so the live gate cannot be read"
return v
if gate_value < budget_gb + reserve_gb:
detail = (
f"{round(free or 0, 1)} GiB free and {round(avail or 0, 1)} GiB "
f"available, {tenants} process(es) on the card"
)
why = (
"MemFree is the gate because something else is competing for reclaim, "
"and losing that race is what a CUDA out-of-memory here actually is"
if tenants
else "MemAvailable is the gate because nothing else is competing, so "
"reclaim is not a race"
)
v.reason = (
f"the pool is big enough on paper -- {round(held_total, 2)} GiB held of "
f"{round(cap, 1)} -- but the machine does not have the room right "
f"now: {detail}. {slug} needs {budget_gb} GiB plus {reserve_gb} GiB "
f"for the host. {why}. Wait for a co-tenant to finish; "
f"scripts/remote_validation/wait_for_memory.py --need "
f"{int(budget_gb + reserve_gb)} does the waiting."
)
return v
v.admit = True
v.reason = (
f"{slug} fits: {budget_gb} GiB on top of {round(held_total, 2)} GiB held, "
f"{round(wanted, 2)} of {round(cap, 1)} GiB ({cap_source}), and "
f"{gate} reads {round(gate_value, 1)} GiB with {tenants} process(es) on "
f"the card."
)
return v
[docs]
def room_for(slug: str, budget_gb: Optional[float], **kw) -> Verdict:
"""`decide` against what the MACHINE is holding, not what this process is.
`backend.running()` was the obvious source and is the wrong one: it sees the
workers this interpreter started, and the ones that wedged this host came
from another worktree. `machine_holdings` reads the process table instead,
which covers both, so this function does not need to add them up.
A holder in this tree with the same slug is dropped, because restarting a
worker releases its own budget. A holder of the same slug in ANOTHER tree is
kept, because that one is still holding it.
"""
held = machine_holdings()
held.pop(slug, None)
return decide(slug, budget_gb, held, **kw)
