From 8d6ad2204d49dc6dda1e4637d3e3b536e178f8a4 Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sat, 17 Oct 2026 09:33:58 +0000 Subject: [PATCH 1/2] feat(chat): summarise long chats at the model's real context window A conversation was folded into its summary early only once it filled half of a fixed 16000-token guess, so a local model started with 4096 overflowed first. The window is now the conversation's model's: ai.context_tokens when set, Lens's own local model's context, what the model server lists (or llama.cpp's /props), else 16000. ai.compact_at sets the share (50 %). A long stretch folded at once is written in runs that fit the summary model, each call in the activity ledger as the conversation's. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XZAK1YALs8mQRwgGnyju4F --- README.md | 3 +- docs/assistant.md | 15 +++- docs/requirements.md | 2 +- fastapi_backend/app/domain/chat.py | 85 +++++++++++++----- fastapi_backend/app/domain/llm.py | 92 ++++++++++++++++---- fastapi_backend/app/domain/local_llm.py | 11 +++ fastapi_backend/app/domain/settings.py | 4 + fastapi_backend/app/domain/store.py | 3 +- fastapi_backend/tests/api/_assist.py | 1 + fastapi_backend/tests/api/test_compaction.py | 91 ++++++++++++++++++- fastapi_backend/tests/fake_llm.py | 12 ++- nextjs-frontend/components/settings/model.ts | 14 ++- 12 files changed, 283 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 3532f9807..a620f1395 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,8 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p of each conversation's own. Regenerating an answer or editing a question makes a new version beside the old one, and nothing is lost. Each answer keeps a record of what produced it (the model, its tools, how long it took and what it cost), opened from its header. It reads answers aloud with a speech provider, macOS `say` or StyleTTS2 on this machine - (opt-in), and `/clear` and `/compact` keep long conversations short. Each namespace can have its own assistant with + (opt-in). Long conversations are summarised before they fill the model's context, and `/clear` and `/compact` start + afresh or summarise now. Each namespace can have its own assistant with its own name, instructions and memory, answering in Telegram and Slack (built in, with voice notes and files), in Discord, Matrix, WhatsApp and other chat rooms through Matterbridge, and by email at an address of its own that you email or forward things to. ([Chat](docs/assistant.md), [web search](docs/assistant.md#web-search-and-reading-pages), diff --git a/docs/assistant.md b/docs/assistant.md index 4ab1e52b0..c183c7365 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -37,11 +37,18 @@ then. every answer gets that summary plus the messages it doesn't cover yet. Room conversations (chat-rooms.md) go on for good, so they rely on it most. A summary covers the version it was written on: on a version that leaves what it covers, the model reads that version's messages, and the next summary is made from them. A conversation that fills - half of the model's context window (**Chat model's context**, `ai.context_tokens`, 16000 when empty; about four - characters a token) is folded in early, before the next answer, whatever the count. + half of its model's context window (**Summarise when a conversation fills**, `ai.compact_at`, 50 %; about four + characters a token) is folded in early, before the next answer, whatever the count, so the excerpts, tools and the + answer fit in the rest. The context window is the conversation's model's: **Chat model's context** + (`ai.context_tokens`) when it's set; otherwise the context Lens's own [local model](local-models.md) runs with + (`local_llm.context`) when it's the provider; otherwise what the model server says (its model list's + `context_length`, `max_model_len`, `max_context_length` or `context_window`, as OpenRouter, vLLM, Mistral, Groq and + others give it, or llama.cpp's `/props`), asked once every ten minutes; otherwise 16000 tokens. **Model for summaries** (`ai.compact_model`) writes them with a smaller, cheaper model on the same provider; empty, - the chat's model does. Settings › AI assistant › Summarise long conversations (`ai.compact`, on) turns it off; off, - or with no model, only the last six messages are read. + the chat's model does. A long stretch folded at once (`/compact`, or a conversation from before summaries) is + written in as many calls as it takes to fit that model's context, each in the [activity ledger](activity.md) as the + conversation's. Settings › AI assistant › Summarise long conversations (`ai.compact`, on) turns it off; off, or with + no model, only the last six messages are read. - **/clear and /compact:** typed as a message, in the app or in a chat room (`Lens /clear`). `/clear` makes the assistant start afresh: the messages stay to read, but the model no longer sees them or their summary. `/compact` folds the whole conversation into its summary now, and `/compact keep the dates and owners` tells the diff --git a/docs/requirements.md b/docs/requirements.md index 02a380d46..62a0e2eaa 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -83,7 +83,7 @@ request, to another check that proves the same thing; loosening a requirement or | Regenerating an answer or editing a question makes a new version beside the old one | [test_chat_versions.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_chat_versions.py) `test_regenerate_edit_switch_and_go_on`; [chat-versions.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-versions.test.tsx) `regenerating an answer`, `the version switch` | | Each answer keeps a record of what produced it: the model, its tools, how long it took and what it cost | [test_chat_records.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_chat_records.py) `test_an_answer_records_what_produced_it`, `test_a_regenerated_answer_has_its_own_record`; [chat-details.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/chat-details.test.tsx) `says what produced it: model and host, scope, tools, use, tokens, cost, time and how it ended` | | Answers read aloud by a speech provider, macOS `say` or StyleTTS2 (opt-in) | [test_voice.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_voice.py) `test_answers_are_read_aloud_by_a_speech_model`; [test_speech.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_speech.py) `test_macos_say_reads_aloud_on_a_mac`, `test_styletts2_reads_aloud_from_its_own_service`; [styletts2.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/styletts2.yml) `Read a sentence aloud` | -| `/clear` and `/compact` | [test_compaction.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_compaction.py) `test_clear_starts_afresh_and_compact_folds_everything_now`, `test_commands_in_the_app` | +| Long conversations summarised before they fill the model's context, and `/clear` and `/compact` | [test_compaction.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_compaction.py) `test_older_messages_are_folded_into_a_summary`, `test_the_context_window_is_the_model_servers`, `test_a_long_conversation_is_folded_in_runs_that_fit_the_summary_model`, `test_clear_starts_afresh_and_compact_folds_everything_now`, `test_commands_in_the_app` | | A namespace's own assistant, with its own name, instructions and memory | [test_ns_assistant.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_ns_assistant.py) `test_off_by_default_and_owners_turn_it_on`, `test_remembers_across_conversations_with_citations`; [ns-assistant.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/ns-assistant.test.tsx) `saving a name and forgetting a memory` | | Telegram and Slack built in, with voice notes and files | [test_telegram.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_telegram.py) `test_it_answers_private_chats_and_what_is_said_to_it_in_groups`, `test_voice_notes_are_heard`, `test_files_are_held_for_the_assistant_to_import`; [test_slack.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_slack.py) `test_it_answers_direct_messages_mentions_and_the_slash_command`, `test_voice_clips_are_heard_and_files_held` | | Other chat rooms through Matterbridge | [test_bridge.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_bridge.py) `test_it_answers_what_is_said_to_it_and_nothing_else`; [test_matterbridge_run.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_matterbridge_run.py) `test_chat_networks_are_set_in_the_app_and_become_matterbridges_config`, `test_the_bridge_talks_to_the_matterbridge_lens_runs`; [matterbridge.yml](https://github.com/adeelahmad/lense/blob/main/.github/workflows/matterbridge.yml) `Lens writes a config, and Matterbridge starts with it` | diff --git a/fastapi_backend/app/domain/chat.py b/fastapi_backend/app/domain/chat.py index 3d72a9d52..406805d60 100644 --- a/fastapi_backend/app/domain/chat.py +++ b/fastapi_backend/app/domain/chat.py @@ -14,7 +14,7 @@ from collections import Counter, defaultdict from urllib.parse import urlsplit -from . import __version__, activity, llm, recsets, render, semantic, store, textindex, untrusted +from . import __version__, activity, llm, local_llm, recsets, render, semantic, store, textindex, untrusted R = store.R STOP = set( @@ -553,8 +553,9 @@ def migrate_tree(db): RECENT = 6 # the latest messages, always given word for word COMPACT_AFTER = 8 # how many more build up before they're folded into the summary (one model call per that many) TURN_CHARS = 1500 # of each message, for the summary -CONTEXT_TOKENS = 16000 # the model's context window when ai.context_tokens isn't set -FULL = 0.5 # the share of it the conversation may take before it's folded in early (the rest: excerpts, tools, answer) +CONTEXT_TOKENS = 16000 # a model's context window when neither ai.context_tokens nor its server says (window) +COMPACT_AT = 50 # the % of it a conversation may fill before it's folded in early (ai.compact_at; the rest: excerpts, tools, answer) +SUMMARY_TOKENS = 800 # the most a summary is written in COMPACT_SYSTEM = ( "You keep the running summary of a conversation between a person and their assistant, so the assistant still " "knows what was said once the messages themselves are out of view. Fold the new messages into the summary so far: " @@ -610,18 +611,46 @@ def tokens(messages): return sum(len(m.get("content") or "") + len(attached_note(m.get("attachments"))) for m in messages) // 4 -def full(cfg, messages): - """True when the messages fill more than FULL of the model's context window (ai.context_tokens).""" - return tokens(messages) > (cfg["ai"].get("context_tokens") or CONTEXT_TOKENS) * FULL +def window(cfg, model=None): + """A model's context window in tokens (by default the chat's): ai.context_tokens when it's set (for the chat's + models); the local model's (local_llm.context) when this machine's llama-server is the provider; what the model + server says (llm.context_window); else CONTEXT_TOKENS.""" + if cfg["ai"].get("context_tokens"): + return cfg["ai"]["context_tokens"] + if local_llm.serves(cfg, model): + return local_llm.conf(cfg)["context"] + return llm.context_window(cfg, model) or CONTEXT_TOKENS + + +def full(cfg, messages, model=None): + """True when the messages fill more than ai.compact_at % (COMPACT_AT) of the model's context window.""" + return tokens(messages) > window(cfg, model) * (cfg["ai"].get("compact_at") or COMPACT_AT) / 100 + + +def _batches(cfg, model, had, old): + """`old` in runs that each fit the summary's model with the summary so far and room to write the next one, so a + long conversation folded at once (/compact, or one from before summaries) never overflows a small model.""" + room = window(cfg, model) * 3 // 4 - SUMMARY_TOKENS - (len(COMPACT_SYSTEM) + len(had.get("text") or "") + 2000) // 4 + run, size = [], 0 + for m in old: + n = len(_said(m)) // 4 + if run and size + n > room: + yield run + run, size = [], 0 + run.append(m) + size += n + if run: + yield run def compact(db, cfg, cid, force=False, focus=None, full_only=False, line=None): """Fold the messages older than the latest RECENT into the conversation's summary, once COMPACT_AFTER of them have - built up or they fill FULL of the model's context window (`full_only`: only then; ai.compact, on by default; needs a - model). `force` (/compact) folds every message in now, whatever the setting, with the person's `focus` (what to - keep) if any. The summary is written by ai.compact_model when set, a cheaper model than the chat's. It's of the - version `line` is (by default the head's), afresh when the summary was another version's. Returns the new summary - (with "folded": how many messages), or None when nothing changed.""" + built up or they fill ai.compact_at % of the chat model's context window (window; `full_only`: only then; + ai.compact, on by default; needs a model). `force` (/compact) folds every message in now, whatever the setting, + with the person's `focus` (what to keep) if any. The summary is written by ai.compact_model when set, a cheaper + model than the chat's, in as many calls as it takes to fit that model's window (_batches), each in the ledger as + the conversation's. It's of the version `line` is (by default the head's), afresh when the summary was another + version's. Returns the new summary (with "folded": how many messages), or None when nothing changed.""" if not llm.configured(cfg) or not (force or cfg["ai"].get("compact", True)): return None line = path(db, cid) if line is None else line @@ -630,18 +659,32 @@ def compact(db, cfg, cid, force=False, focus=None, full_only=False, line=None): old = fresh if force else fresh[:-RECENT] if not old: return None - if not force and not full(cfg, fresh) and (full_only or len(fresh) < RECENT + COMPACT_AFTER): - return None - ask = f"Summary so far:\n{had.get('text') or '(none yet)'}\n\nNew messages:\n\n" + "\n\n".join(_said(m) for m in old) - if focus: - ask += f"\n\nThe person asked the summary to keep: {focus[:500]}" - msgs = [{"role": "system", "content": COMPACT_SYSTEM}, {"role": "user", "content": ask}] - text = llm.chat(cfg, msgs, model=cfg["ai"].get("compact_model") or None, max_tokens=800).strip() - if not text: + if not force and (full_only or len(fresh) < RECENT + COMPACT_AFTER): + model = (db.one("SELECT model FROM $r", r=R("chat", cid)) or {}).get("model") # the conversation's, if picked + if not full(cfg, fresh, model): + return None + writer = cfg["ai"].get("compact_model") or None + out, folded = None, 0 + with activity.scope(db, f"chat:{cid}", cfg=cfg): + for run in _batches(cfg, writer, had, old): + ask = f"Summary so far:\n{had.get('text') or '(none yet)'}\n\nNew messages:\n\n" + "\n\n".join(_said(m) for m in run) + if focus: + ask += f"\n\nThe person asked the summary to keep: {focus[:500]}" + msgs = [{"role": "system", "content": COMPACT_SYSTEM}, {"role": "user", "content": ask}] + try: + text = llm.chat(cfg, msgs, model=writer, max_tokens=SUMMARY_TOKENS).strip() + except (llm.LLMError, activity.OverBudget): + if force or out is None: + raise + break # what's folded so far is kept; the rest is folded in next time + if not text: + break + had = out = {"text": text, "upto": run[-1]["id"], "at": store.now()} + folded += len(run) + if out is None or (force and folded < len(old)): # /compact folds everything or changes nothing return None - out = {"text": text, "upto": old[-1]["id"], "at": store.now()} db.q("UPDATE $r SET summary = $s", r=R("chat", cid), s=out) - return {**out, "folded": len(old)} + return {**out, "folded": folded} # ---------- commands: /clear and /compact, typed in the app or in a chat room ---------- diff --git a/fastapi_backend/app/domain/llm.py b/fastapi_backend/app/domain/llm.py index 888e5185a..6738a0ee3 100644 --- a/fastapi_backend/app/domain/llm.py +++ b/fastapi_backend/app/domain/llm.py @@ -26,10 +26,7 @@ def _post(cfg, payload): l = cfg["llm"] if not configured(cfg): raise LLMError("no language model is configured (Settings → LLM provider)") - headers = {"Content-Type": "application/json"} - key = l.get("api_key") or (os.environ.get(l["api_key_env"]) if l.get("api_key_env") else None) - if key: - headers["Authorization"] = f"Bearer {key}" + headers = {"Content-Type": "application/json", **_headers(cfg)} req = urllib.request.Request( l["base_url"].rstrip("/") + "/chat/completions", data=json.dumps(payload).encode(), headers=headers, method="POST" ) @@ -41,29 +38,90 @@ def _post(cfg, payload): raise LLMError(f"can't reach the LLM server at {l['base_url']}: {e}") from None -_MODELS = {} # base_url -> (when, names): the server's list, kept a minute +_MODELS = {} # base_url -> (when, {name: what the server says of it}): the server's list, kept a minute +_WINDOWS = {} # (base_url, model) -> (when, tokens or None): each model's context window, kept ten minutes -def list_models(cfg, ttl=60): - """The models the server offers (GET /models), kept for `ttl` seconds.""" +def _headers(cfg): + l = cfg["llm"] + key = l.get("api_key") or (os.environ.get(l["api_key_env"]) if l.get("api_key_env") else None) + return {"Authorization": f"Bearer {key}"} if key else {} + + +def _listing(cfg, ttl=60): + """The models the server offers (GET /models), each name with what the server says of it, kept for `ttl` seconds.""" l = cfg["llm"] - if not configured(cfg): - return [] url = l["base_url"].rstrip("/") hit = _MODELS.get(url) if hit and time.monotonic() - hit[0] < ttl: return hit[1] - headers = {} - key = l.get("api_key") or (os.environ.get(l["api_key_env"]) if l.get("api_key_env") else None) - if key: - headers["Authorization"] = f"Bearer {key}" try: - with urllib.request.urlopen(urllib.request.Request(url + "/models", headers=headers), timeout=10) as r: - names = sorted({str(m["id"]) for m in json.load(r).get("data") or [] if isinstance(m, dict) and m.get("id")}) + with urllib.request.urlopen(urllib.request.Request(url + "/models", headers=_headers(cfg)), timeout=10) as r: + listed = {str(m["id"]): m for m in json.load(r).get("data") or [] if isinstance(m, dict) and m.get("id")} except (urllib.error.URLError, TimeoutError, OSError, ValueError, AttributeError) as e: raise LLMError(f"can't list the models at {l['base_url']}: {e}") from None - _MODELS[url] = (time.monotonic(), names) - return names + _MODELS[url] = (time.monotonic(), listed) + return listed + + +def list_models(cfg, ttl=60): + """The models the server offers (GET /models), kept for `ttl` seconds.""" + return sorted(_listing(cfg, ttl)) if configured(cfg) else [] + + +# what a GET /models entry calls the model's context window: LM Studio's loaded context first (what it runs with), then +# OpenRouter's, Together's and others' context_length, vLLM's max_model_len, Mistral's max_context_length, Groq's +# context_window and max_input_tokens +WINDOW_KEYS = ("loaded_context_length", "context_length", "max_model_len", "max_context_length", "context_window", "max_input_tokens") + + +def _tokens(v): + if isinstance(v, str) and v.strip().isdigit(): + v = int(v) + return int(v) if isinstance(v, (int, float)) and not isinstance(v, bool) and v >= 256 else None + + +def _window_of(entry): + for k in WINDOW_KEYS: + if n := _tokens(entry.get(k)): + return n + return _tokens((entry.get("top_provider") or {}).get("context_length")) if isinstance(entry.get("top_provider"), dict) else None + + +def _llamacpp_window(cfg): + """The context llama.cpp's server runs each request with (GET /props at its root), which its model list doesn't + give (only the context the model was trained for).""" + root = re.sub(r"/v1/?$", "", cfg["llm"]["base_url"].rstrip("/")) + try: + with urllib.request.urlopen(urllib.request.Request(root + "/props", headers=_headers(cfg)), timeout=10) as r: + j = json.load(r) + except (urllib.error.URLError, TimeoutError, OSError, ValueError) as e: + raise LLMError(f"can't ask llama.cpp's server for its context: {e}") from None + if not isinstance(j, dict): + return None + gen = j.get("default_generation_settings") + return _tokens(gen.get("n_ctx")) if isinstance(gen, dict) else _tokens(j.get("n_ctx")) + + +def context_window(cfg, model=None, ttl=600): + """The model's context window in tokens as its server says: its GET /models entry (WINDOW_KEYS; a server listing + one model speaks for whichever is asked for), or llama.cpp's /props. None when the server doesn't say or can't be + asked. Kept `ttl` seconds, a miss too, so a server that doesn't say isn't asked before every answer.""" + if not configured(cfg): + return None + model = model or cfg["llm"]["model"] + key = (cfg["llm"]["base_url"].rstrip("/"), model) + hit = _WINDOWS.get(key) + if hit and time.monotonic() - hit[0] < ttl: + return hit[1] + try: + listed = _listing(cfg) + entry = listed.get(model) or (next(iter(listed.values())) if len(listed) == 1 else {}) + n = _window_of(entry) or (_llamacpp_window(cfg) if entry.get("owned_by") == "llamacpp" else None) + except LLMError: + n = None + _WINDOWS[key] = (time.monotonic(), n) + return n def _load(r): diff --git a/fastapi_backend/app/domain/local_llm.py b/fastapi_backend/app/domain/local_llm.py index 5293ce345..ebd18d419 100644 --- a/fastapi_backend/app/domain/local_llm.py +++ b/fastapi_backend/app/domain/local_llm.py @@ -126,6 +126,17 @@ def conf(cfg): return {**store.DEFAULTS["local_llm"], **(cfg.get("local_llm") or {})} +def serves(cfg, model=None): + """True when the LLM provider is this machine's llama-server running the local model (or `model`, when that's it), + so its context is local_llm.context.""" + c, l = conf(cfg), cfg.get("llm") or {} + try: + port = urllib.parse.urlsplit(l.get("base_url") or "").port + except ValueError: + return False + return bool(c["enabled"] and c["model"] and (model or l.get("model")) == c["model"] and port == c["port"]) + + def resolve(ref): """A catalog entry for a catalog id or an "hf://.gguf" reference.""" if ref in BY_ID: diff --git a/fastapi_backend/app/domain/settings.py b/fastapi_backend/app/domain/settings.py index bba6483eb..904502ec6 100644 --- a/fastapi_backend/app/domain/settings.py +++ b/fastapi_backend/app/domain/settings.py @@ -448,6 +448,10 @@ def _check(section, key, value, default): if value is not None and not (isinstance(value, int) and not isinstance(value, bool) and 1024 <= value <= 2_000_000): raise ValueError("ai.context_tokens is a whole number of tokens from 1024 to 2000000") return value + if (section, key) == ("ai", "compact_at"): + if not (isinstance(value, int) and not isinstance(value, bool) and 10 <= value <= 90): + raise ValueError("ai.compact_at is a whole percentage of the model's context from 10 to 90") + return value if (section, key) == ("ai", "browser_steps"): if not (isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 100): raise ValueError("ai.browser_steps is a whole number of steps from 1 to 100") diff --git a/fastapi_backend/app/domain/store.py b/fastapi_backend/app/domain/store.py index 5d74ee50c..a405563a8 100644 --- a/fastapi_backend/app/domain/store.py +++ b/fastapi_backend/app/domain/store.py @@ -298,7 +298,8 @@ def labels(): "refine_notes": True, # the model keeps notes' titles and one-line summaries true to the whole note (notebook.py) "compact": True, # a long conversation keeps a summary of the messages the model no longer sees (chat.compact) "compact_model": None, # a cheaper model to write those summaries (None: the chat's model) - "context_tokens": None, # the chat model's context window; summarised early before it fills (None: 16000) + "context_tokens": None, # the chat model's context window (None: what its server says, else 16000; chat.window) + "compact_at": 50, # a conversation filling this % of that window is summarised early, before the next answer "organise_notes": True, # notes nobody filed are filed in PARA, or a suggestion waits on them (notebook.py) "untrusted_marking": True, # third-party text reaches the model marked as data, not instructions (untrusted.py) "untrusted_asks": True, # after reading it, tools that send data out or change the server ask first (ai_tools.py) diff --git a/fastapi_backend/tests/api/_assist.py b/fastapi_backend/tests/api/_assist.py index 0eacfb47f..4db187df3 100644 --- a/fastapi_backend/tests/api/_assist.py +++ b/fastapi_backend/tests/api/_assist.py @@ -26,6 +26,7 @@ def sse(text): def start_llm(cfg): fake_llm.Handler.tool_script, fake_llm.Handler.reject_tools, fake_llm.Handler.decision = [], False, None fake_llm.Handler.ignore_tool_choice = False + fake_llm.Handler.about, fake_llm.Handler.props = {}, None fake_llm.Handler.names, fake_llm.Handler.cypher_script = None, [] fake_llm.Handler.seen = [] # what earlier tests on this worker sent isn't this test's srv, url = fake_llm.start() diff --git a/fastapi_backend/tests/api/test_compaction.py b/fastapi_backend/tests/api/test_compaction.py index 5c2d1e531..32a4bccaf 100644 --- a/fastapi_backend/tests/api/test_compaction.py +++ b/fastapi_backend/tests/api/test_compaction.py @@ -2,9 +2,12 @@ from __future__ import annotations +from urllib.parse import urlsplit + import pytest from app.domain import chat +from app.domain import llm as llm_mod from tests import fake_llm from tests.api._assist import sse, start_llm from tests.helpers import login, make_user, seed @@ -103,6 +106,84 @@ def test_a_conversation_that_fills_the_context_is_folded_in_early(db, cfg, llm): assert chat.compact(db, cfg, cid, full_only=True) is None # short ones don't fill it +def long_talk(db, cid, n): + for i in range(n): + chat.add(db, cid, "user" if i % 2 == 0 else "assistant", f"long message {i} " + "word " * 100) + + +def forget_windows(): + llm_mod._MODELS.clear() + llm_mod._WINDOWS.clear() + + +def test_the_context_window_is_the_model_servers(db, cfg, llm): + """Without ai.context_tokens, a conversation is folded in early at the context the model really has: what the + model server lists, llama.cpp's /props, or Lens's own local model's.""" + cid = chat.create(db, 1) + long_talk(db, cid, chat.RECENT + 1) # seven of these fill half of 1700 tokens, not half of 16000 + forget_windows() + assert chat.window(cfg) == chat.CONTEXT_TOKENS and chat.compact(db, cfg, cid, full_only=True) is None + + llm.about = {"fake": {"max_model_len": 1700}} # vLLM + forget_windows() + assert chat.window(cfg) == 1700 and chat.compact(db, cfg, cid, full_only=True)["folded"] == 1 + + cid = chat.create(db, 1) + long_talk(db, cid, chat.RECENT + 1) + llm.about, llm.props = ( + {"fake": {"owned_by": "llamacpp", "meta": {"n_ctx_train": 131072}}}, + {"default_generation_settings": {"n_ctx": 1700}}, + ) + forget_windows() + assert chat.window(cfg) == 1700 # what the server runs with, not what the model was trained for + assert chat.compact(db, cfg, cid, full_only=True)["folded"] == 1 + + cid = chat.create(db, 1) + long_talk(db, cid, chat.RECENT + 1) + llm.about, llm.props = {}, None + port = urlsplit(cfg["llm"]["base_url"]).port + cfg["local_llm"] = {**cfg.get("local_llm", {}), "enabled": True, "model": "fake", "port": port, "context": 1700} + forget_windows() + assert chat.window(cfg) == 1700 and chat.compact(db, cfg, cid, full_only=True)["folded"] == 1 + cfg["local_llm"]["enabled"] = False + + # ai.context_tokens, when set, is what counts; ai.compact_at is how much of it a conversation may fill + llm.about = {"fake": {"context_length": 1700}, "fake-large": {"context_length": 1700}} + forget_windows() + cid = chat.create(db, 1) + long_talk(db, cid, chat.RECENT + 1) + cfg["ai"]["context_tokens"] = 100000 + assert chat.window(cfg) == 100000 and chat.compact(db, cfg, cid, full_only=True) is None + cfg["ai"]["context_tokens"], cfg["ai"]["compact_at"] = None, 90 + assert chat.compact(db, cfg, cid, full_only=True) is None + cfg["ai"]["compact_at"] = 50 + assert chat.compact(db, cfg, cid, full_only=True)["folded"] == 1 + + # the conversation's own model's context counts, when it has one + llm.about = {"fake-large": {"context_length": 1700}} + forget_windows() + cid = chat.create(db, 1, model="fake-large") + long_talk(db, cid, chat.RECENT + 1) + assert chat.window(cfg) == chat.CONTEXT_TOKENS and chat.compact(db, cfg, cid, full_only=True)["folded"] == 1 + + +def test_a_long_conversation_is_folded_in_runs_that_fit_the_summary_model(db, cfg, llm): + cfg["ai"]["compact_model"] = "fake-large" + llm.about = {"fake-large": {"context_length": 4096}} # about a dozen of these long messages at a time + forget_windows() + cid = chat.create(db, 1) + long_talk(db, cid, 30) + reply, mid = chat.run_command(db, cfg, cid, "compact") + asked = summaries(llm.seen) + assert reply.startswith("Compacted 30 messages") and len(asked) == 3 + assert all(b["model"] == "fake-large" and b["max_tokens"] == chat.SUMMARY_TOKENS for b in asked) + assert "(none yet)" in asked[0]["messages"][1]["content"] and "Summary so far:\nOK" in asked[2]["messages"][1]["content"] + assert "long message 29 " in asked[2]["messages"][1]["content"] and chat.memory(db, cfg, cid)["upto"] == mid + # each call is in the activity ledger as the conversation's + rows = db.rows("SELECT resources FROM activity WHERE action = 'model.chat' AND model = 'fake-large'") + assert len(rows) == 3 and all(f"chat:{cid}" in r["resources"] for r in rows) + + def test_commands_are_read_from_the_message(): assert chat.command("/clear") == ("clear", "") assert chat.command(" /Compact keep the dates\n") == ("compact", "keep the dates") @@ -159,5 +240,13 @@ def test_the_summary_settings_are_checked(client, db): h = login(client, "root@x.io", "root password 1") ok = client.put("/api/v1/settings/ai", headers=h, json={"compact_model": " small ", "context_tokens": 32000}) assert ok.status_code == 200, ok.text - for bad in ({"context_tokens": 10}, {"context_tokens": "big"}, {"compact_model": 3}): + assert client.put("/api/v1/settings/ai", headers=h, json={"compact_at": 70}).status_code == 200 + for bad in ( + {"context_tokens": 10}, + {"context_tokens": "big"}, + {"compact_model": 3}, + {"compact_at": 5}, + {"compact_at": 95}, + {"compact_at": "half"}, + ): assert client.put("/api/v1/settings/ai", headers=h, json=bad).status_code == 400, bad diff --git a/fastapi_backend/tests/fake_llm.py b/fastapi_backend/tests/fake_llm.py index fed2051af..1dd312543 100644 --- a/fastapi_backend/tests/fake_llm.py +++ b/fastapi_backend/tests/fake_llm.py @@ -9,6 +9,8 @@ class Handler(http.server.BaseHTTPRequestHandler): seen = [] models = ["fake", "fake-large"] # what GET /models lists + about = {} # what GET /models says of a model besides its id, like {"fake": {"context_length": 4096}} + props = None # GET /props, as llama.cpp's server answers it ({"default_generation_settings": {"n_ctx": ...}}) tool_script = [] # assistant messages to return, in order, when a request offers tools (or functions of the request) reject_tools = False # behave like a server whose model can't call tools ignore_tool_choice = False # behave like a server that calls tools even when asked not to (tool_choice "none") @@ -28,8 +30,14 @@ def _json(self, obj): self.end_headers() self.wfile.write(data) - def do_GET(self): # GET /models - return self._json({"object": "list", "data": [{"id": m, "object": "model"} for m in Handler.models]}) + def do_GET(self): # GET /models, or llama.cpp's GET /props + if self.path == "/props" and Handler.props is not None: + return self._json(Handler.props) + if self.path == "/props": + self.send_error(404) + return None + listed = [{"id": m, "object": "model", **Handler.about.get(m, {})} for m in Handler.models] + return self._json({"object": "list", "data": listed}) def do_POST(self): body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) diff --git a/nextjs-frontend/components/settings/model.ts b/nextjs-frontend/components/settings/model.ts index b9419a77b..58fce1580 100644 --- a/nextjs-frontend/components/settings/model.ts +++ b/nextjs-frontend/components/settings/model.ts @@ -924,8 +924,18 @@ export const FIELDS: FieldSpec[] = [ max: 2000000, nullable: true, unit: "tokens", - placeholder: "16000", - hint: "A conversation is summarised early before it fills half of this.", + placeholder: "what the model server says", + hint: "Empty: what the model server says (Lens's own local model: its context), else 16000.", + }, + { + section: "ai", + key: "compact_at", + label: "Summarise when a conversation fills", + kind: "int", + min: 10, + max: 90, + unit: "%", + hint: "Of the model's context, before the next answer, so the excerpts, tools and the answer fit in the rest.", }, { section: "ai", From 77e5d03f51949af4f227a0081403af1e7811b8dd Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 01:04:01 +0000 Subject: [PATCH 2/2] fix(web): show the summarise-at setting in the AI fold The settings test's AI section lacked the new ai.compact_at, so Save did nothing; the field now sits with the other long-conversation options. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01XZAK1YALs8mQRwgGnyju4F --- nextjs-frontend/__tests__/settings-web-search.test.tsx | 1 + nextjs-frontend/components/settings/bodies.tsx | 2 ++ 2 files changed, 3 insertions(+) diff --git a/nextjs-frontend/__tests__/settings-web-search.test.tsx b/nextjs-frontend/__tests__/settings-web-search.test.tsx index 9e164a653..d432193bc 100644 --- a/nextjs-frontend/__tests__/settings-web-search.test.tsx +++ b/nextjs-frontend/__tests__/settings-web-search.test.tsx @@ -41,6 +41,7 @@ const web = (values: Record) => price_out: null, compact_model: null, context_tokens: null, + compact_at: 50, }, }, decisions: { diff --git a/nextjs-frontend/components/settings/bodies.tsx b/nextjs-frontend/components/settings/bodies.tsx index eab5e9c80..5379b1fbd 100644 --- a/nextjs-frontend/components/settings/bodies.tsx +++ b/nextjs-frontend/components/settings/bodies.tsx @@ -2599,6 +2599,7 @@ const LIMITS = [ "ai.max_transcript_reads", "ai.compact_model", "ai.context_tokens", + "ai.compact_at", ]; function AiBody({ ctx }: { ctx: BodyCtx }) { @@ -2667,6 +2668,7 @@ function AiBody({ ctx }: { ctx: BodyCtx }) { +