diff --git a/README.md b/README.md index 603afe1..cf38427 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Characters (bytes) does not pass through a fixed stack of layers as it would be Three distinct blocks, up to 26 block-applications per character. -- **Adaptive depth.** A halting head scores every character at every row, and the character stops as soon as another row would not change the answer. Easy characters take one row, hard ones take many. This is the PonderNet recipe: while training, every depth is computed and weighted by its halting probability, so the halting head learns through those weights. +- **NoProp-FM Continuous-Time Denoising.** Instead of adaptive depth per character, this architecture uses a Flow Matching objective to integrate the latent representations through a vector field via Euler integration. Backpropagation through time (BPTT) is eliminated. - **Routing per block-application, not per character.** Each of the 26 applications picks its own top-8 experts, so one character touches far more of the pool than "top-8" suggests, and the same expert can be selected several times at different depths. What varies is *which* eight at each point. - **No expert is assigned a subject.** There are no labels anywhere. Soft top-k routing distributes capability across the pool by itself, and a character can combine fragments from several experts. The cost is that capabilities share parameters and so *can* interfere. @@ -250,7 +250,6 @@ python3 -m corpora expand # .bin -> the text files read python3 train.py read --help # every knob the reader has python3 train.py stream --steps 140000 --lr 2e-4 # the packed-corpus path -python3 train.py ponder-probe --ckpt weights # depth against difficulty ``` Every tool takes `--ckpt weights` - the directory is the model, and there are no `.pt` files to keep track of. @@ -318,8 +317,8 @@ Byte level - vocabulary 265: the 256 byte values plus 9 structural markers (` self.rope_cos.shape[0]: - # Reading past the rotary tables gives an empty slice and a - # shape error four frames deeper, which says nothing useful raise ValueError( f"reading at position {pos_offset + T:,} but the rotary " - f"tables were built to {self.rope_cos.shape[0]:,}. The window " - f"may not exceed the ceiling the model was built with " - f"(model.context_end in config.yaml).") + f"tables were built to {self.rope_cos.shape[0]:,}.") cos = self.rope_cos[pos_offset:pos_offset + T] sin = self.rope_sin[pos_offset:pos_offset + T] @@ -256,97 +231,87 @@ def slot(i): x = blk(x, cos, sin, slot(ci)) ci += 1 - h = torch.zeros_like(x) - cum = torch.ones(B, T, 1, device=x.device, dtype=torch.float32) - loss_terms, p_terms = [], [] - halted_logits = None - halted = torch.zeros(B, T, 1, device=x.device, dtype=torch.bool) - steps_used = torch.zeros(B, T, device=x.device) - per_step = [] - - n_steps = self.sample_depth() - for n in range(n_steps): - # truncated backprop: only the last few passes carry gradient, so - # memory does not grow with depth - if (targets is not None and cfg.bptt_window > 0 - and n < n_steps - cfg.bptt_window): - h = h.detach() - h = self.adapter(torch.cat([h, x], dim=-1)) + h_x = x # Clean context from prelude + + if targets is not None: + # Training: NoProp-FM continuous time dynamics + u_y = self.tok_emb(targets) + + # Sample time t ~ U[0, 1] + t = torch.rand(B, 1, 1, device=x.device, dtype=x.dtype) + + # Sample initial noise z_0 ~ N(0, I) + z_0 = torch.randn_like(u_y) + + # Interpolate to get noisy state z_t + z_t = t * u_y + (1 - t) * z_0 + + # Get time embedding + t_emb = self.get_time_embedding(t.view(B, 1)) + t_emb = t_emb.expand(B, T, -1) + + # Pass through single recurrent block (acting point-wise, no self-attention) + # Concat z_t, h_x, and t_emb + h = self.adapter(torch.cat([z_t, h_x, t_emb], dim=-1)) + + ci_recur_start = ci for blk in self.recur: - h = blk(h, cos, sin, slot(ci)) + h = blk(h, cos, sin, slot(ci), use_attn=False) ci += 1 - y = h for blk in self.coda: - y = blk(y, cos, sin, slot(ci)) + h = blk(h, cos, sin, slot(ci), use_attn=False) ci += 1 - yf = self.ln_f(y) - logits_n = self.head(yf) - - lam = torch.sigmoid(self.halt(yf).float()) # [B,T,1] - if n == n_steps - 1: - lam = torch.ones_like(lam) # must stop - elif n < cfg.min_steps - 1: - lam = torch.zeros_like(lam) # must continue - p_n = cum * lam - cum = cum * (1.0 - lam) - - if targets is not None: - ce = F.cross_entropy( - logits_n.reshape(-1, logits_n.size(-1)).float(), - targets.reshape(-1), reduction="none").view(B, T) - loss_terms.append(p_n.squeeze(-1) * ce) - p_terms.append(p_n.squeeze(-1)) - # the halting-weighted mixture of every depth's logits, so - # what is returned is what the model would actually emit - term = logits_n * p_n.to(logits_n.dtype) - halted_logits = (term if halted_logits is None - else halted_logits + term) - else: - # Each token halts on its own schedule: the first step whose - # cumulative halting mass crosses the threshold is the one - # whose logits that token keeps. The forced lam=1 on the last - # step guarantees every token halts somewhere. - if halted_logits is None: - halted_logits = logits_n.clone() - steps_used = torch.ones(B, T, device=x.device) - newly = (~halted) & ((1.0 - cum) >= cfg.halt_thresh) - if bool(newly.any()): - halted_logits = torch.where(newly, logits_n, halted_logits) - steps_used = torch.where( - newly.squeeze(-1), - torch.full_like(steps_used, float(n + 1)), steps_used) - halted = halted | newly - if collect: - per_step.append({"step": n + 1, - "halt_p": float(p_n.mean()), - "cum": float((1 - cum).mean())}) - - # what this stretch of text looked like, for the next segment's choice - self.end_segment(x) - - if targets is None: - out = {"steps": steps_used} if collect else None - if collect: - out["per_step"] = per_step - return halted_logits, out - - # PonderNet: expected loss under the halting distribution, plus a KL - # pull toward a geometric prior so it does not simply always run long. - P = torch.stack(p_terms, 0) # [N,B,T] - L = torch.stack(loss_terms, 0) - loss = L.sum(0).mean() - prior = torch.tensor( - [cfg.halt_prior * (1 - cfg.halt_prior) ** n - for n in range(len(p_terms))], device=x.device) - prior = (prior / prior.sum()).view(-1, 1, 1) - kl = (P.clamp_min(1e-8) * (P.clamp_min(1e-8).log() - prior.log())).sum(0) - loss = loss + cfg.ponder_beta * kl.mean() - steps = (P * torch.arange(1, len(p_terms) + 1, device=x.device) - .view(-1, 1, 1)).sum(0) - self.last_steps = float(steps.mean()) - # logits are the halting-weighted mixture, so top-1 accuracy measured - # downstream reflects what the model would actually have emitted - return halted_logits, loss + + yf = self.ln_f(h) + v_theta = self.v_proj(yf) + + # Flow Matching Loss + # Target vector field is (u_y - z_0) + target_v = u_y - z_0 + loss_fm = F.mse_loss(v_theta.float(), target_v.float(), reduction='none').mean() + + # Anchor Loss (Extrapolated Linear Estimate) + z_hat_1 = z_t + (1 - t) * v_theta + logits = self.head(z_hat_1) + loss_anchor = F.cross_entropy( + logits.reshape(-1, logits.size(-1)).float(), + targets.reshape(-1), reduction="none").mean() + + loss = loss_fm + cfg.lambda_anchor * loss_anchor + + self.end_segment(h_x) + self.last_steps = 1.0 # only 1 step during training + return logits, loss + else: + # Inference: Euler integration over euler_steps + z_t = torch.randn_like(h_x) + dt = 1.0 / cfg.euler_steps + + for step in range(cfg.euler_steps): + t_val = step * dt + t = torch.full((B, 1, 1), t_val, device=x.device, dtype=x.dtype) + t_emb = self.get_time_embedding(t.view(B, 1)).expand(B, T, -1) + + h = self.adapter(torch.cat([z_t, h_x, t_emb], dim=-1)) + + ci = len(self.prelude) + for blk in self.recur: + h = blk(h, cos, sin, slot(ci), use_attn=False) + ci += 1 + for blk in self.coda: + h = blk(h, cos, sin, slot(ci), use_attn=False) + ci += 1 + + yf = self.ln_f(h) + v_theta = self.v_proj(yf) + z_t = z_t + dt * v_theta + + logits = self.head(z_t) + + self.end_segment(h_x) + + out = {"steps": torch.full((B, T), float(cfg.euler_steps), device=x.device)} if collect else None + return logits, out @torch.no_grad() def choose_for(self, idx, free=False): diff --git a/train.py b/train.py index 4e65e67..cad429d 100644 --- a/train.py +++ b/train.py @@ -9,7 +9,6 @@ one chunk of characters at a time. Peak memory is set by --chunk and nothing else, so --context is nearly free to extend. - probe what depth the model chooses, per character. Everything here runs at batch 1 behind a cache. The fixed-window regime that preceded it - several windows per step, one autograd graph over each whole @@ -427,7 +426,7 @@ def write(self, cfg, precision, chunk): meta=_json.dumps({"chunks": self.meta, "swaps": self.swaps, "chunk": chunk, "top_k": cfg.pool_top_k, - "max_steps": cfg.max_steps, + "euler_steps": cfg.euler_steps, "n_recur": cfg.n_recur, "n_coda": cfg.n_coda, "n_prelude": cfg.n_prelude, "context": cfg.block, @@ -738,58 +737,9 @@ def _in_steps(chars, least=1): birth_gate=args.birth_gate, recent_mult=args.recent_mult) \ if args.grow_k else None - # The checkpoint carries whatever depth policy it was trained under, but - # this is a knob about how to spend compute now, not a property of the - # weights - so config.yaml wins over the manifest every time. - cfg.train_steps_mean = float(args.train_steps_mean) - cfg.min_steps = max(1, min(int(args.min_steps), cfg.max_steps)) - if args.bptt_window: - cfg.bptt_window = int(args.bptt_window) - if args.ponder_beta: - cfg.ponder_beta = float(args.ponder_beta) - if args.halt_prior: - cfg.halt_prior = float(args.halt_prior) - if args.halt_thresh: - cfg.halt_thresh = float(args.halt_thresh) - # The settings above are counted in ROWS, and the CHECKPOINT decides how - # many rows a step is - n_recur + n_coda. So a config written for one - # architecture is silently wrong on another, and the failure is not always - # loud: bptt_window 16 against max_steps 6 makes the detach condition - # n < -10, which never fires, so NOTHING is detached and the graph holds - # every block instead of the last few. That is how a config meant for a - # 24-row model put a 6-row model out of memory. - warn = [] - if cfg.bptt_window > cfg.max_steps: - warn.append(f"bptt_window {cfg.bptt_window} exceeds max_steps " - f"{cfg.max_steps}, so nothing would be detached and the " - f"whole chain would be held. Clamping to {cfg.max_steps}.") - cfg.bptt_window = cfg.max_steps - if cfg.train_steps_mean and cfg.train_steps_mean >= cfg.max_steps: - warn.append(f"train_steps_mean {cfg.train_steps_mean:g} is at or above " - f"max_steps {cfg.max_steps}, so depth sampling saves " - f"nothing - every step runs the ceiling.") - if cfg.halt_prior and 1.0 / cfg.halt_prior > cfg.max_steps * 1.5: - warn.append(f"halt_prior {cfg.halt_prior:g} pulls toward " - f"{1/cfg.halt_prior:.1f} rows but the model only has " - f"{cfg.max_steps}.") - if warn: - print(f" [config] this checkpoint has {cfg.n_recur + cfg.n_coda} " - f"block(s) per row and {cfg.max_steps} rows; config.yaml looks " - f"written for a different shape:", flush=True) - for w in warn: - print(f" - {w}", flush=True) - print(f" gradient reaches back {cfg.bptt_window} of {cfg.max_steps} rows") - print(f" depth: up to {cfg.max_steps} rows, halting prior pulls toward " - f"{1/max(cfg.halt_prior,1e-9):.1f} rows, inference stops at " - f"{cfg.halt_thresh:g} cumulative") - if cfg.train_steps_mean > 0: - import torch as _t - _n = (_t.poisson(_t.full((20000,), cfg.train_steps_mean)).int() + 1 - ).clamp(cfg.min_steps, cfg.max_steps).float().mean() - print(f" recurrence depth is SAMPLED while training: mean {_n:.2f} of " - f"{cfg.max_steps} ({100*_n/cfg.max_steps:.0f}% of the recurrent " - f"compute). Inference still runs all {cfg.max_steps} and lets " - f"halting decide.") + cfg.euler_steps = int(args.euler_steps) + cfg.lambda_anchor = float(args.lambda_anchor) + pool.dying_at = args.dying_at _PLOTS.update(on=bool(args.plots), since=args.plot_since, log=args.sample_log, weights=args.weights_dir) @@ -1885,47 +1835,6 @@ def build_paged(wdir, device, resident=None, ram_capacity=256, ceiling=None, return model, cfg, pool, man -def cmd_ponder_probe(args): - """ - Does the model actually spend more computation on harder problems? - - This is the falsifiable claim behind adaptive depth. If mean halting steps - are flat across difficulty, the halting head learned nothing useful and the - mechanism is decoration. Reported per digit count, which is the cleanest - difficulty axis available. - """ - import corpora.arithmetic as math_data - from minagi.tokenizer import load_tokenizer - device = torch.device(args.device) - model, ck = load_recur(args.ckpt, device) - tok = load_tokenizer(args.data) - import random - rng = random.Random(0) - print(f"{'digits':>7} {'mean steps':>11} {'max':>5} {'example':<34}") - rows = [] - for d in range(1, args.max_digits + 1): - steps = [] - example = "" - for _ in range(args.n): - fn, cap, _ = math_data.TASKS[args.task] - line = fn(rng, min(d, cap), False) - prompt = line.rpartition("=")[0] + "=" - example = example or prompt - ids = torch.tensor([tok.encode(prompt).ids], device=device) - with torch.no_grad(): - _, extra = model(ids, collect=True) - steps.append(float(extra["steps"][0, -1])) - rows.append((d, float(np.mean(steps)), max(steps))) - print(f"{d:>7} {np.mean(steps):>11.2f} {max(steps):>5.0f} {example:<34}") - lo = rows[0][1] - hi = rows[-1][1] - print(f"\n1-digit {lo:.2f} steps -> {args.max_digits}-digit {hi:.2f} steps " - f"({hi-lo:+.2f})") - print("adaptive compute is working" if hi - lo > 0.15 else - "FLAT - the halting head is not responding to difficulty") - return 0 - - def main(): ap = argparse.ArgumentParser( description="train mini-AGI: read files continually, or stream a " @@ -2141,40 +2050,18 @@ def main(): "held-out says the ground moved; this is the manual " "override for when you know it did and it has not " "noticed yet") - rd.add_argument("--bptt-window", type=int, - default=_cfg(_c, "model.bptt_window", None), - help="how many of the last ROWS carry gradient back " - "through the recurrence. Counted in rows, so it must " - "be rescaled whenever a row changes size") - rd.add_argument("--ponder-beta", type=float, - default=_cfg(_c, "model.ponder_beta", None), - help="weight on the KL that holds halting to its prior") - rd.add_argument("--halt-prior", type=float, - default=_cfg(_c, "model.halt_prior", None), - help="geometric prior on depth: the KL pulls the halting " - "head toward a mean of 1/this ROWS. Must be rescaled " - "whenever a row changes size") - rd.add_argument("--halt-thresh", type=float, - default=_cfg(_c, "model.halt_thresh", None), - help="at inference, stop at the first row whose cumulative " - "halting mass passes this") - rd.add_argument("--min-steps", type=int, - default=_cfg(_c, "model.min_steps", 1), - help="rows a character must run before halting may stop " - "it; config.yaml had this but only the old `train` " - "command ever read it") + rd.add_argument("--euler-steps", type=int, + default=_cfg(_c, "model.euler_steps", 10), + help="Number of Euler integration steps during inference") + rd.add_argument("--lambda-anchor", type=float, + default=_cfg(_c, "model.lambda_anchor", 1.0), + help="Weight for the anchor cross-entropy loss") rd.add_argument("--no-pool-checkpoint", action="store_true", help="stop recomputing the expert pass during backward. " "It trades about 30%% more compute for most of the " "pool's activation memory, which is only worth it " "while depth makes that memory the binding " "constraint") - rd.add_argument("--train-steps-mean", type=float, - default=_cfg(_c, "model.train_steps_mean", 0.0), - help="sample the recurrence depth while training instead " - "of always running max_steps; 0 keeps the old " - "behaviour. NOT the mean depth - the draw is " - "poisson(this)+1, so 2.0 averages 3.0 of 6") rd.add_argument("--dying-at", type=float, default=_cfg(_c, "prune.dying_at", 0.75), help="share of prune.survival_chars an expert may go " @@ -2259,14 +2146,6 @@ def main(): help="give up after this many reverts rather than thrash") st.set_defaults(fn=cmd_stream) - p = sub.add_parser("ponder-probe") - p.add_argument("--ckpt", default="weights", - help="the weights directory, or a .pt checkpoint") - p.add_argument("--data", default="data_math_char") - p.add_argument("--task", default="add") - p.add_argument("--n", type=int, default=20) - p.add_argument("--max-digits", type=int, default=8) - p.set_defaults(fn=cmd_ponder_probe) args = ap.parse_args() sys.exit(args.fn(args) or 0)