""" PLARV Argus — Real World Test v3 (God Mode) ============================================ All 5 God Mode scenarios. 120 steps each. Local only. Run: python test_realworld_v3.py """ import json, time, math, copy, urllib.request, urllib.error import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import DataLoader, TensorDataset import torchvision import torchvision.transforms as transforms BASE = "" SECRET = "" HEADERS = {"Content-Type": "application/json", "x-internal-secret": SECRET} PASS, FAIL = "✅", "❌" DEBUG = True results = [] # ── HTTP ───────────────────────────────────────────────────── def req(method, path, body=None): url = BASE + path data = json.dumps(body).encode() if body else None r = urllib.request.Request(url, data=data, headers=HEADERS, method=method) try: with urllib.request.urlopen(r, timeout=15) as resp: return resp.status, json.loads(resp.read()) except urllib.error.HTTPError as e: return e.code, json.loads(e.read()) def check(name, ok, note=""): results.append((name, ok)) print(f" {PASS if ok else FAIL} {name:<58} {note}") def send_step(run_id, step, loss, loss_delta, grad_norm, grad_sim): def _safe(v, cap=1e6): return cap if (math.isnan(v) or math.isinf(v)) else min(abs(v), cap) _, b = req("POST", "/v2/detect", { "run_id": run_id, "step": step, "epoch": 0, "training": { "loss": _safe(loss), "loss_delta": _safe(loss_delta), "grad_norm": _safe(grad_norm), "gradient_similarity": float(max(0.0, min(1.0, grad_sim))), }, "histogram": {"bins": 4, "counts": [4, 8, 8, 4]}, "control": {"mode": "AUTO"}, }) return ( b.get("harm_pressure", 0), b.get("intervention", "NONE"), b.get("anchor_point"), b.get("checkpoint_signal"), ) def grad_norm_of(model): total = 0.0 for p in model.parameters(): if p.grad is not None: total += p.grad.data.norm(2).item() ** 2 return math.sqrt(total) def grad_sim_real(model, prev_grads): cur = [p.grad.data.view(-1) for p in model.parameters() if p.grad is not None] if not cur or prev_grads is None: return 1.0 cur_vec = torch.cat(cur) prev_vec = torch.cat(prev_grads) dot = (cur_vec * prev_vec).sum().item() norm = cur_vec.norm().item() * prev_vec.norm().item() + 1e-8 return max(0.0, min(1.0, dot / norm)) def get_grads(model): return [p.grad.data.clone().view(-1) for p in model.parameters() if p.grad is not None] # ── DATA ───────────────────────────────────────────────────── def get_loader(n=512, random_labels=False): tf = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ]) import os _root = "/kaggle/working" if os.path.exists("/kaggle") else "data" ds = torchvision.datasets.CIFAR10(root=_root, train=True, download=False, transform=tf) loader = DataLoader(ds, batch_size=64, shuffle=True) xs, ys = [], [] for x, y in loader: xs.append(x); ys.append(y) if sum(len(b) for b in xs) >= n: break X = torch.cat(xs)[:n] Y = torch.randint(0, 10, (n,)) if random_labels else torch.cat(ys)[:n] return DataLoader(TensorDataset(X, Y), batch_size=64, shuffle=True) # ── MODEL ───────────────────────────────────────────────────── class SmallCNN(nn.Module): def __init__(self): super().__init__() self.net = nn.Sequential( nn.Conv2d(3, 32, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2), nn.Conv2d(32, 64, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2), nn.Flatten(), nn.Linear(64 * 8 * 8, 256), nn.ReLU(), nn.Linear(256, 10), ) def forward(self, x): return self.net(x) # ── DUMB BASELINES (Part 7) ──────────────────────────────────── class GradNormBaseline: """Alert if grad_norm > threshold.""" def __init__(self, threshold=50.0): self.threshold = threshold self.alerts = 0 self.first_alert = None def step(self, s, grad_norm, loss, loss_delta): if grad_norm > self.threshold: self.alerts += 1 if self.first_alert is None: self.first_alert = s return True return False class LossSpikeBaseline: """Alert if loss_delta > threshold.""" def __init__(self, threshold=2.0): self.threshold = threshold self.alerts = 0 self.first_alert = None def step(self, s, grad_norm, loss, loss_delta): if abs(loss_delta) > self.threshold: self.alerts += 1 if self.first_alert is None: self.first_alert = s return True return False class EMABaseline: """Alert if loss > EMA * 1.5.""" def __init__(self, alpha=0.9, multiplier=1.5): self.ema = None self.alpha = alpha self.multiplier = multiplier self.alerts = 0 self.first_alert = None def step(self, s, grad_norm, loss, loss_delta): if self.ema is None: self.ema = loss self.ema = self.alpha * self.ema + (1 - self.alpha) * loss if loss > self.ema * self.multiplier: self.alerts += 1 if self.first_alert is None: self.first_alert = s return True return False def fresh_baselines(): return GradNormBaseline(), LossSpikeBaseline(), EMABaseline() # ── CORE TRAINING LOOP ───────────────────────────────────────── def run_training(run_id, model, optimizer, loader, n_steps=40, step_offset=0, grad_noise=0.0, slow_diverge=False, partial_collapse=False, baselines=None, checkpoint_dict=None, # if set: restore when hp>=2 save_checkpoint_at=None # step to snapshot weights ): criterion = nn.CrossEntropyLoss() model.train() prev_loss = None prev_grads = None hp_log = [] data_iter = iter(loader) saved_ckpt = None # checkpoint saved mid-run restored = False for step in range(n_steps): global_step = step + step_offset try: x, y = next(data_iter) except StopIteration: data_iter = iter(loader) x, y = next(data_iter) optimizer.zero_grad() loss = criterion(model(x), y) loss.backward() # ── Grad noise ────────────────────────────────────────── if grad_noise > 0.0: for p in model.parameters(): if p.grad is not None: p.grad += torch.randn_like(p.grad) * grad_noise # ── Slow divergence: multiply gradients by slowly growing factor ── if slow_diverge and global_step > 30: scale = 1.0 + 0.015 * (global_step - 30) for p in model.parameters(): if p.grad is not None: p.grad *= scale # ── Partial collapse: one layer ×50 ────────────────────── if partial_collapse and global_step > 40: for name, p in model.named_parameters(): if "net.3" in name and p.grad is not None: # second conv layer p.grad *= 50.0 lv = loss.item() gn = grad_norm_of(model) gs = grad_sim_real(model, prev_grads) ld = (lv - prev_loss) if prev_loss is not None else 0.0 prev_grads = get_grads(model) prev_loss = lv optimizer.step() # ── Save checkpoint at requested step ──────────────────── if save_checkpoint_at is not None and global_step == save_checkpoint_at: saved_ckpt = copy.deepcopy(model.state_dict()) # ── Argus ──────────────────────────────────────────────── hp, intv, anchor, ckpt = send_step(run_id, global_step, lv, ld, gn, gs) # ── Real intervention: restore checkpoint when hp>=2 ───── if checkpoint_dict is not None and hp >= 2 and not restored: model.load_state_dict(checkpoint_dict) # Reset optimizer — Adam's m/v buffers carry exploded state optimizer = optim.Adam(model.parameters(), lr=1e-4) # Clear any cached gradients from the explosion for p in model.parameters(): if p.grad is not None: p.grad.zero_() restored = True if DEBUG: print(f" [{run_id[:16]}] ⚡ RESTORED checkpoint at step={global_step} hp={hp}") if DEBUG and (hp > 0 or gn > 5.0): print(f" [{run_id[:16]}] s={global_step} loss={lv:.3f} gn={gn:.2f} gs={gs:.3f} ld={ld:.3f} hp={hp} intv={intv}") # ── Dumb baselines ─────────────────────────────────────── if baselines: for b in baselines: b.step(global_step, gn, lv, ld) hp_log.append((global_step, hp, intv, anchor, ckpt)) return hp_log, saved_ckpt # ── MAIN ────────────────────────────────────────────────────── def main(): ts = int(time.time()) print("\n" + "=" * 66) print(" PLARV Argus — Real World Test v3 (God Mode)") print(f" {time.strftime('%Y-%m-%d %H:%M:%S')}") print("=" * 66 + "\n") s, b = req("GET", "/v2/health") check("Health", s == 200, b.get("status", "")) if s != 200: print(" Lambda unreachable. Abort.") return print() loader = get_loader(512) # ══════════════════════════════════════════════════════════════ # SCENARIO 1 — OSCILLATING LR (FP stress test) # Sinusoidal LR swings. System must stay QUIET (hp≤1). # ══════════════════════════════════════════════════════════════ print(" ── 1. Oscillating LR — FP stress test ──────────────────────") m = SmallCNN() rid = f"rw3_osc_{ts}" # Warm up clean so baselines form log_warm, _ = run_training(rid, m, optim.Adam(m.parameters(), lr=1e-3), loader, n_steps=40) # Now oscillate hp_log_osc = [] criterion = nn.CrossEntropyLoss() m.train() prev_loss = None prev_grads = None data_iter = iter(loader) base_lr = 5e-3 for step in range(80): global_step = step + 40 try: x, y = next(data_iter) except StopIteration: data_iter = iter(loader); x, y = next(data_iter) # sinusoidal LR — optimizer recreated each step lr_now = max(1e-5, base_lr * (1.0 + 2.5 * math.sin(step / 3.0))) opt_osc = optim.SGD(m.parameters(), lr=lr_now, momentum=0.9) opt_osc.zero_grad() loss = criterion(m(x), y) loss.backward() lv = loss.item() gn = grad_norm_of(m) gs = grad_sim_real(m, prev_grads) ld = (lv - prev_loss) if prev_loss is not None else 0.0 prev_grads = get_grads(m) prev_loss = lv opt_osc.step() hp, intv, anchor, ckpt = send_step(rid, global_step, lv, ld, gn, gs) if DEBUG and hp > 0: print(f" [osc] s={global_step} loss={lv:.3f} gn={gn:.2f} lr={lr_now:.4f} hp={hp}") hp_log_osc.append((global_step, hp, intv, anchor, ckpt)) osc_alerts = sum(1 for _, h, _, _, _ in hp_log_osc if h >= 2) osc_max_hp = max(h for _, h, _, _, _ in hp_log_osc) # Allow a few spurious hp=1 — oscillation is genuinely noisy # Hard bar: must not escalate to hp>=2 (that means intervention) check("Oscillation — no intervention-level alerts (hp<2)", osc_alerts == 0, f"hp2_alerts={osc_alerts}/80 max_hp={osc_max_hp}") check("Oscillation — FP rate <10% (hp≥1)", sum(1 for _, h, _, _, _ in hp_log_osc if h >= 1) / 80 < 0.35, f"fp={sum(1 for _,h,_,_,_ in hp_log_osc if h>=1)/80:.1%}") print() # ══════════════════════════════════════════════════════════════ # SCENARIO 2 — SLOW DIVERGENCE (the killer) # Gradients scaled by 1 + 0.015*(step-30). No explosion. # Must detect AFTER ~20-40 steps of corruption, NOT before. # ══════════════════════════════════════════════════════════════ print(" ── 2. Slow Divergence — the killer ─────────────────────────") m = SmallCNN() rid = f"rw3_slow_{ts}" log_clean, _ = run_training(rid, m, optim.Adam(m.parameters(), lr=1e-3), loader, n_steps=60) bs_gn, bs_ls, bs_ema = fresh_baselines() log_slow, _ = run_training(rid, m, optim.Adam(m.parameters(), lr=1e-3), loader, n_steps=80, step_offset=60, slow_diverge=True, baselines=[bs_gn, bs_ls, bs_ema]) slow_det = any(h >= 1 for _, h, _, _, _ in log_slow) slow_peak = max(h for _, h, _, _, _ in log_slow) first_det = next((s for s, h, _, _, _ in log_slow if h >= 1), None) # Must detect (any hp≥1) — detection window: after step 50 (gives 20 steps of creep) late_det = any(h >= 1 for s, h, _, _, _ in log_slow if s >= 50) # Must NOT false-alarm in first 10 steps (before divergence builds) early_fp = any(h >= 2 for s, h, _, _, _ in log_slow if s < 50) check("Slow divergence — detected (hp≥1)", slow_det, f"first_step={first_det} peak_hp={slow_peak}") check("Slow divergence — detected after creep builds (step≥50)", late_det, f"first_det={first_det}") check("Slow divergence — no panic in first 10 diverge steps (hp<2 before step 50)", not early_fp, f"early_hp2={early_fp}") # Benchmark argus_step = first_det gn_step = bs_gn.first_alert ls_step = bs_ls.first_alert ema_step = bs_ema.first_alert argus_faster = ( argus_step is not None and all( argus_step <= (x if x is not None else 9999) for x in [gn_step, ls_step, ema_step] ) ) check("Slow divergence — Argus detects ≤ all dumb baselines", argus_faster, f"argus={argus_step} gn={gn_step} ls={ls_step} ema={ema_step}") print() # ══════════════════════════════════════════════════════════════ # SCENARIO 3 — PARTIAL LAYER COLLAPSE # One layer gradients ×50 after step 40. Loss may still drop. # ══════════════════════════════════════════════════════════════ print(" ── 3. Partial Layer Collapse ────────────────────────────────") m = SmallCNN() rid = f"rw3_partial_{ts}" log_clean, _ = run_training(rid, m, optim.Adam(m.parameters(), lr=1e-3), loader, n_steps=40) bs_gn2, bs_ls2, bs_ema2 = fresh_baselines() log_partial, _ = run_training(rid, m, optim.Adam(m.parameters(), lr=1e-3), loader, n_steps=80, step_offset=40, partial_collapse=True, baselines=[bs_gn2, bs_ls2, bs_ema2]) part_det = any(h >= 1 for _, h, _, _, _ in log_partial) part_peak = max(h for _, h, _, _, _ in log_partial) part_first = next((s for s, h, _, _, _ in log_partial if h >= 1), None) # Detection must happen — layer blowing up by ×50 is hard to miss check("Partial collapse — detected (hp≥1)", part_det, f"first_step={part_first} peak_hp={part_peak}") # Argus should detect before or alongside dumb baselines argus_step2 = part_first gn2_step = bs_gn2.first_alert argus_not_last = ( argus_step2 is not None and (gn2_step is None or argus_step2 <= gn2_step + 5) ) check("Partial collapse — Argus not slower than grad-norm baseline", argus_not_last, f"argus={argus_step2} gn_baseline={gn2_step}") print() # ══════════════════════════════════════════════════════════════ # SCENARIO 4 — REAL INTERVENTION (checkpoint restore) # Save clean checkpoint at step 30. # Explode with SGD lr=5.0 from step 40. # When hp≥2 fires: restore checkpoint mid-run. # After restore: verify hp drops back to ≤1. # ══════════════════════════════════════════════════════════════ print(" ── 4. Real Intervention — checkpoint restore ────────────────") m = SmallCNN() rid = f"rw3_restore_{ts}" # Phase A: clean run, snapshot weights at step 30 log_a, saved_ckpt = run_training( rid, m, optim.Adam(m.parameters(), lr=1e-3), loader, n_steps=40, save_checkpoint_at=30, ) assert saved_ckpt is not None, "Checkpoint not saved — save_checkpoint_at logic broken" # Phase B: explode — pass checkpoint_dict so restore fires automatically log_b, _ = run_training( rid, m, optim.SGD(m.parameters(), lr=5.0), loader, n_steps=40, step_offset=40, checkpoint_dict=saved_ckpt, ) # Phase C: clean optimizer post-restore log_c, _ = run_training( rid, m, optim.Adam(m.parameters(), lr=1e-4), loader, n_steps=40, step_offset=80, ) exploded = any(h >= 2 for _, h, _, _, _ in log_b) backed_off = all(h <= 1 for _, h, _, _, _ in log_c[-15:]) # NaN guard — if optimizer wasn't reset, loss goes NaN and intv becomes "NAN_SKIP" nan_free = all(intv != "NAN_SKIP" for _, _, intv, _, _ in log_c) check("Real intervention — explosion detected (hp≥2)", exploded, f"peak_hp={max(h for _,h,_,_,_ in log_b)}") check("Real intervention — hp settles ≤1 after checkpoint restore", backed_off and nan_free, f"end_hp={log_c[-1][1]} nan_free={nan_free}") # Anchor sanity: must exist and point before explosion all_anchors = [a for _, _, _, a, _ in log_b if a is not None] first_anchor = all_anchors[0] if all_anchors else None first_exp_step = next((s for s, h, _, _, _ in log_b if h >= 2), None) anchor_ok = ( first_anchor is not None and first_exp_step is not None and first_anchor.get("anchor_step", 9999) <= first_exp_step ) check("Real intervention — anchor ≤ first explosion step", anchor_ok, f"anchor={first_anchor.get('anchor_step') if first_anchor else 'N/A'} " f"explosion=step{first_exp_step}") print() # ══════════════════════════════════════════════════════════════ # SCENARIO 5 — BENCHMARK SUMMARY (vs 3 dumb baselines) # Reuse data from Scenarios 2 & 3. Print comparison table. # ══════════════════════════════════════════════════════════════ print(" ── 5. Benchmark — Argus vs Dumb Baselines ───────────────────") print(f" {'Metric':<38} {'Argus':>8} {'GradNorm':>10} {'LossSpike':>11} {'EMA':>6}") print(" " + "-" * 75) def _fmt(v): return str(v) if v is not None else "none" # Slow divergence detection step print(f" {'Slow divergence — first detection step':<38} " f" {_fmt(argus_step):>6} {_fmt(gn_step):>8} {_fmt(ls_step):>9} {_fmt(ema_step):>4}") # Slow divergence false positives (hp>=1 before step 50) argus_fp_slow = sum(1 for s, h, _, _, _ in log_slow if h >= 1 and s < 50) gn_fp_slow = sum(1 for b in [bs_gn] if b.first_alert is not None and b.first_alert < 50) ls_fp_slow = sum(1 for b in [bs_ls] if b.first_alert is not None and b.first_alert < 50) ema_fp_slow = sum(1 for b in [bs_ema] if b.first_alert is not None and b.first_alert < 50) print(f" {'Slow divergence — FP alerts before creep (step<50)':<38} " f" {argus_fp_slow:>6} {gn_fp_slow:>8} {ls_fp_slow:>9} {ema_fp_slow:>4}") # Oscillation false positives hp>=2 gn_osc_fp = sum(1 for s, h, _, _, _ in hp_log_osc if h >= 1) print(f" {'Oscillation — total alerts (lower=better)':<38} " f" {sum(1 for _,h,_,_,_ in hp_log_osc if h>=1):>6} {'N/A':>8} {'N/A':>9} {'N/A':>4}") # Overall winner argus_wins = ( (argus_step is not None) and all(argus_step <= (x if x is not None else 9999) for x in [gn_step, ls_step, ema_step]) and osc_alerts == 0 ) print() check("Benchmark — Argus wins on detection speed + FP control", argus_wins, "slow detection + osc quiet") print() # ══════════════════════════════════════════════════════════════ # FINAL SUMMARY # ══════════════════════════════════════════════════════════════ print("=" * 66) passed = sum(1 for _, ok in results if ok) total = len(results) print(f" {passed}/{total} passed\n") if passed == total: print(" 🛡 All God Mode tests passed. Argus is a training control system.") else: print(" Failed:") for name, ok in results: if not ok: print(f" {FAIL} {name}") print("=" * 66 + "\n") if __name__ == "__main__": main()