fix(issue-17): review R1 - the finalizer ends with its wrapper
finalize-end ran the disk grace and the broker re-observation as a child of the wrapper; TERM to the wrapper left it running, and at the deadline it still closed the job and published the error (codex R1, reproduced with the real registry). The bash loops it replaced ended with the wrapper. The wrapper now passes its own pid (--watch-pid "$$", as for the stall watcher). finalize-end checks it every 0.2 s through both waits and before the close and the event, and leaves without either once it is gone. A comparison test sends TERM to the old and the new wrapper inside each grace: both exit at once, leave no process behind and make no call. Also: no blank line at the end of the old-decisions fixture, and the #42 inventory follows main() one line down. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
66d1490230
commit
c5cebe6c32
5 files changed
+137
-7
No files matched your search
@@ -15,7 +15,11 @@ what a subcommand prints or returns:
|
||||
herdr diagnostic probe bash still runs
|
||||
|
||||
finalize-end answers on fd 3 (stdout when fd 3 is closed): its stdout carries
|
||||
registry.py's close output to the wrapper's stdout, as before.
|
||||
registry.py's close output to the wrapper's stdout, as before. It runs as a
|
||||
child of the wrapper and must not outlive it (review R1): with --watch-pid it
|
||||
checks that the wrapper is still alive throughout both waits and before each
|
||||
side effect, and leaves without closing or publishing once it is gone - as the
|
||||
same loops in bash ended with the wrapper.
|
||||
|
||||
Stdlib only and Python 3.9 (macOS /usr/bin/python3); registry.py and
|
||||
publish_event.py run as the processes the wrapper ran, with its interpreter.
|
||||
@@ -111,6 +115,36 @@ def recent_disk_heartbeat(registry_dir: str, job_id: str, window: float) -> bool
|
||||
return time.time() - seen <= window
|
||||
|
||||
|
||||
class OwnerGone(Exception):
|
||||
"""The wrapper that ran finalize-end is gone."""
|
||||
|
||||
|
||||
def _alive(pid: int) -> bool:
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
def _check_owner(a: argparse.Namespace) -> None:
|
||||
if a.watch_pid and not _alive(a.watch_pid):
|
||||
raise OwnerGone(a.watch_pid)
|
||||
|
||||
|
||||
def _sleep(a: argparse.Namespace, seconds: float) -> None:
|
||||
"""time.sleep that looks at the wrapper every 0.2 s."""
|
||||
end = time.time() + seconds
|
||||
while True:
|
||||
_check_owner(a)
|
||||
left = end - time.time()
|
||||
if left <= 0:
|
||||
return
|
||||
time.sleep(min(left, 0.2))
|
||||
|
||||
|
||||
def _tail(path: str, n: int) -> List[str]:
|
||||
with open(path, errors="replace") as fh:
|
||||
return fh.read().splitlines()[-n:]
|
||||
@@ -163,7 +197,7 @@ def finalize_end(a: argparse.Namespace) -> str:
|
||||
left = deadline - _now()
|
||||
if left <= 0:
|
||||
break
|
||||
time.sleep(min(left, 3))
|
||||
_sleep(a, min(left, 3))
|
||||
disk = disk_status(reg, job)
|
||||
if disk in TERMINAL:
|
||||
status = final = disk
|
||||
@@ -191,7 +225,7 @@ def finalize_end(a: argparse.Namespace) -> str:
|
||||
break
|
||||
if deadline - _now() <= 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
_sleep(a, 1)
|
||||
if _now() >= deadline:
|
||||
clock = "wall"
|
||||
|
||||
@@ -211,6 +245,7 @@ def finalize_end(a: argparse.Namespace) -> str:
|
||||
print(f"finalize: {job} closed by {'wall_clock' if clock == 'wall' else 'no_event'} clock "
|
||||
f"(budget {a.timeout}s / idle {a.idle_timeout}s)", file=sys.stderr)
|
||||
tok = ["--expect-dispatch-token", a.dispatch_token] if a.dispatch_token else []
|
||||
_check_owner(a)
|
||||
close_args = ["close", "--job", job, "--request", close_req] + (["--detail", detail] if detail else []) + tok
|
||||
if _run(a, "registry.py", close_args, merge=True) == 4:
|
||||
# a dispatch token mismatch (stale attempt): no secondary event (CAS/ABA)
|
||||
@@ -221,6 +256,7 @@ def finalize_end(a: argparse.Namespace) -> str:
|
||||
return "completed"
|
||||
# direct path only: publish the error/cancelled event (loop/discuss forbidden: O-1)
|
||||
if a.type == "direct":
|
||||
_check_owner(a)
|
||||
if close_req == "cancelled":
|
||||
_run(a, "publish_event.py", ["--job", job, "--event", "cancelled", "--detail", detail] + tok, merge=False)
|
||||
elif a.sub_rc != SUB_RC_ERROR or a.phase == "inject":
|
||||
@@ -270,6 +306,7 @@ def _parser() -> argparse.ArgumentParser:
|
||||
s.add_argument("--preflight-reason", default="")
|
||||
s.add_argument("--python", default=sys.executable)
|
||||
s.add_argument("--scripts-dir", required=True)
|
||||
s.add_argument("--watch-pid", type=int, default=None, help="the wrapper's pid; gone -> no close, no event")
|
||||
return p
|
||||
|
||||
|
||||
@@ -288,7 +325,10 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
elif a.cmd == "finalize-begin":
|
||||
print(finalize_begin(a))
|
||||
else:
|
||||
_answer(finalize_end(a))
|
||||
try:
|
||||
_answer(finalize_end(a))
|
||||
except OwnerGone:
|
||||
print(f"finalize: wrapper pid {a.watch_pid} is gone; job {a.job} left unclosed", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -821,11 +821,12 @@ finalize_unfinished() {
|
||||
echo "liveness: no wire event for ${IDLE_TIMEOUT}s on job $job_id; herdr/pane says busy=$diag (diagnostic only, not evidence)" >&2
|
||||
fi
|
||||
# The answer comes back on fd 3: stdout carries registry.py's close output.
|
||||
# --watch-pid: the waits end, unclosed, with this wrapper (review R1).
|
||||
{ out="$(_decide finalize-end --job "$job_id" --registry-dir "$REGISTRY_DIR" --status "$job_status" \
|
||||
${START_TIME:+--start-time "$START_TIME"} --timeout "$TIMEOUT" --idle-timeout "$IDLE_TIMEOUT" \
|
||||
--sub-rc "$sub_rc" --phase "$phase" --type "$TYPE" --final "$final" --evidence-until "$evidence_until" \
|
||||
--dispatch-token "${_disp_tok:-}" --preflight-reason "${PREFLIGHT_REASON:-}" \
|
||||
--python "$PY" --scripts-dir "$SCRIPT_DIR/scripts" 3>&1 1>&4 4>&-)"; } 4>&1
|
||||
--python "$PY" --scripts-dir "$SCRIPT_DIR/scripts" --watch-pid "$$" 3>&1 1>&4 4>&-)"; } 4>&1
|
||||
FINAL_JOB_STATUS="$out"
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user