From 182030fc17dcef82b4263dbb698f849fbbd27d6d Mon Sep 17 00:00:00 2001 From: Adeel Ahmad Date: Sun, 18 Oct 2026 11:18:16 +0000 Subject: [PATCH] feat(timeline): ask the assistant about a feed item from the item An opened timeline item has "Ask about this", and `a` on a line does the same: the chat beside the page opens with the item's id, line, time, link and fields as what the question is about (sent as the highlighted part, so the model reads it as data). The page chat listens for a lens:ask-about event that any page's own buttons can send. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q5D9UGwJBfue5FU8tL27aF --- README.md | 3 +- changelog.d/ask-about-timeline-item.md | 4 +++ docs/assistant.md | 4 ++- docs/requirements.md | 1 + docs/timeline.md | 19 ++++++++++-- nextjs-frontend/__tests__/page-chat.test.tsx | 21 +++++++++++++ .../__tests__/timeline-ui.test.tsx | 29 +++++++++++++++++ .../components/page-chat/page-chat-model.ts | 9 ++++++ .../components/page-chat/page-chat.tsx | 11 +++++++ nextjs-frontend/components/timeline/item.tsx | 31 +++++++++++++------ nextjs-frontend/components/timeline/model.ts | 22 +++++++++++++ .../components/timeline/timeline-page.tsx | 9 +++++- 12 files changed, 148 insertions(+), 15 deletions(-) create mode 100644 changelog.d/ask-about-timeline-item.md diff --git a/README.md b/README.md index a5eb642ca..dfb8d329f 100644 --- a/README.md +++ b/README.md @@ -158,7 +158,8 @@ listed under [In progress](#in-progress), and what comes next under [Planned](#p notifications to webhooks, Slack, Discord and Matterbridge. Everything that happens in Lens (new assets, emails, calendar events, files, feed posts, what people say to Lens, sensor readings, notes, graph changes, runs, budget warnings) is kept as a stream of activities in W3C Activity Streams form, which agents read through the API, MCP and - the assistant, and people review on the Timeline page (and Latest on Home), marking what they've seen. Automations act + the assistant, and people review on the Timeline page (and Latest on Home), marking what they've seen and asking the + assistant about any item from the item. Automations act on that stream, on a schedule, at a time or when their own webhook is called, alone or with their triggers joined (all of them, a number of times, one that doesn't come in time, or a batch): they test conditions (text, patterns, numbers, lists, times, days, whether a value changed and whether something like it happened before in their diff --git a/changelog.d/ask-about-timeline-item.md b/changelog.d/ask-about-timeline-item.md new file mode 100644 index 000000000..6d735667b --- /dev/null +++ b/changelog.d/ask-about-timeline-item.md @@ -0,0 +1,4 @@ +- **Talk to the assistant from a timeline item.** An opened item on the Timeline has **Ask about this**, and `a` on + a line does the same: the chat beside the page opens with the item (its id, line, time, where it opens and its + fields) as what your question is about, read as data, so you type the question and press Enter. Nothing changes + until you use it ([timeline](docs/timeline.md#asking-about-an-item)). diff --git a/docs/assistant.md b/docs/assistant.md index cf2e71869..62f63a717 100644 --- a/docs/assistant.md +++ b/docs/assistant.md @@ -83,7 +83,9 @@ then. - **Chat on any page:** every page except Chat itself has an Ask button that opens a chat panel beside the page (full screen on phones). Its conversation follows you between pages, and across visits in the same browser, until New chat; it is an ordinary conversation, also listed in Chat. Each question can share the page's text (the page chip, - on by default) and text highlighted on the page (Ask about this, above any selection). The model reads them with the + on by default) and text highlighted on the page (Ask about this, above any selection). A timeline item's **Ask about + this** (or `a` on its line) hands the panel the item instead: its id, line, time, where it opens and its fields + ([timeline](timeline.md#asking-about-an-item)). The model reads them with the question (up to 12,000 characters of page text and 4,000 of the highlight) as what the person is looking at, not as citable excerpts. The question keeps the page's address, title and highlight, shown above it; the page's text isn't stored. An asset's text and pages keep their own selection toolbar, whose Ask in chat goes to the diff --git a/docs/requirements.md b/docs/requirements.md index 6fa07c2e5..86adb5327 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -193,6 +193,7 @@ request, to another check that proves the same thing; loosening a requirement or | Batch runs over a collection | [test_batches.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_batches.py) `test_collections_and_batches` | | Notifications to webhooks, Slack, Discord and Matterbridge | [test_notifications.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_notifications.py) `test_owners_manage_targets_and_a_test_is_signed`, `test_runs_that_end_are_sent_once_to_matterbridge` | | Everything that happens is kept as W3C Activity Streams activities, read by agents through the API, MCP and the assistant | [test_events.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_events.py) `test_activities_have_one_shape`; [test_activities.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_activities.py) `test_the_timeline_lists_new_assets_newest_first`, `test_agents_read_the_timeline_too`; [test_news_feed_source.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_news_feed_source.py) `test_a_news_feed_as_a_source` | +| Ask the assistant about a timeline item from the item (its button, or `a` on its line), in the chat beside the page, with the item's id, line and fields as what the question is about | [timeline-ui.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/timeline-ui.test.tsx) `asking about an item hands the page chat its id, line and fields, by button or the a key`; [page-chat.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/page-chat.test.tsx) `opens with what a page's own Ask about this hands it, and sends it as the highlighted part` | | The Timeline page and Latest on Home, marking what you've seen | [test_timeline_marks.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_timeline_marks.py) `test_marks_are_set_cleared_and_filter_the_timeline`; [timeline-ui.test.tsx](https://github.com/adeelahmad/lense/blob/main/nextjs-frontend/__tests__/timeline-ui.test.tsx) `Home's Latest card lists what you haven't reviewed` | | Automations act on the stream, on a schedule, at a time or when their own webhook is called | [test_automation_runs.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_automation_runs.py) `test_each_new_activity_runs_each_automation_it_matches_once`, `test_a_schedule_runs_its_automation_when_its_timer_fires`, `test_a_call_to_a_webhook_runs_its_automation`; [test_timer_triggers.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_timer_triggers.py) `test_an_at_trigger_fires_once_then_reads_done` | | They test conditions, then tell you, ask the model or the agent, use a tool, call a web address with sealed credentials, run Lens work, or change assets and notes | [test_automation_runs.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/domain/test_automation_runs.py) `test_the_if_limits_and_dry_runs`; [test_actions_messages.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_actions_messages.py) `test_notify_tells_me_in_my_chat_app`; [test_actions_ai.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_actions_ai.py) `test_ask_model_answers_in_text_or_as_the_fields_a_schema_names`, `test_ask_agent_works_on_what_happened_and_saves_the_conversation`, `test_call_tool_reads_as_the_person_and_changes_only_by_asking_them`; [test_actions_web.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_actions_web.py) `test_credentials_in_headers_are_sealed_to_the_automation_and_the_host_and_never_shown`; [test_actions_run.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_actions_run.py) `test_run_workflow_queues_it_on_the_asset_as_the_automation`, `test_run_pipeline_processes_the_asset_again`; [test_actions_lens.py](https://github.com/adeelahmad/lense/blob/main/fastapi_backend/tests/api/test_actions_lens.py) `test_set_field_is_kept_in_the_assets_history_by_the_automation`, `test_write_note_writes_a_page_adds_to_a_things_page_or_notes_an_asset` | diff --git a/docs/timeline.md b/docs/timeline.md index c1dca1d76..7ae52734d 100644 --- a/docs/timeline.md +++ b/docs/timeline.md @@ -12,6 +12,7 @@ Status: in progress. The studio's timeline is built in steps; only what is merge | Review marks: reviewed, important and hidden, per person; the filters on them; `mark_activities` for agents | built | | The Timeline page under Explore: filters, grouping, review with keys, new items as they come | built | | A Latest card on Home with the newest unreviewed items | built | +| Ask about an item: talk to the assistant about it from the item, in the chat beside the page | built | | What each item caused (automation runs, approvals), and Approve and Decline on waiting approvals | planned | ## What shows @@ -30,8 +31,8 @@ unless you ask for them by kind. - A run of the same source or person and the same kind, each within 30 minutes of the one before, is one line ("Work Gmail received 37 emails"), so a first mailbox sync reads as one line. A click opens it to its items. - Each line has an icon by kind, the summary (a link to what it is about), who did it, the namespace, the kind and - how long ago. Opened, an item shows its object's fields, the automation run that caused it if one did, **Open** and - **Hide**. + how long ago. Opened, an item shows its object's fields, the automation run that caused it if one did, **Open**, + **Ask about this** and **Hide**. - Filters: words in the summary (Enter applies), **Unreviewed only**, **Important**, kinds by family (Email, Calendar, Files, Notes, Sensors, Chat, Automations, System; noisy kinds like sensor readings only when picked), source or person (from those on the timeline), namespace, entity (as activities tag it), a date range, and an @@ -47,11 +48,22 @@ unless you ask for them by kind. | `Enter` | open or close the line | | `r` | mark it reviewed, or not | | `i` | mark it important, or not | +| `a` | ask the assistant about it (every item of a line of several) | Hovering or focusing a line shows its **Important** (star) and **Reviewed** (circle) toggles; once set they stay in sight. On a line of several items they mark every item. **Mark all shown as reviewed** marks everything on the page you haven't reviewed, with **Undo** in the message rather than a confirm; **Hide** works the same way. +### Asking about an item + +**Ask about this** on an opened item, or `a` on the line you are on, opens the chat beside the page +([chat on any page](assistant.md)) with the item as what your question is about, the way highlighted text is: its +id, kind and time, who did it, its line, where it opens and its fields (up to 4,000 characters; on a line of several, +as many whole items as fit). Type the question and press Enter. The model reads the item as data, not as +instructions, since its text may come from outside Lens, and it has the id, so "mark it important" or "what else came +from this sender?" works with its `timeline` and `mark_activities` tools. Agents outside the web app read the same +items through `timeline` over MCP and the API ([Agents](#agents)). + ### Home The overview on Home has a **Latest** card with the five newest activities you haven't reviewed and a link to the @@ -98,6 +110,9 @@ go when the person is removed. ## Refine later +- Ask about an item in the full Chat as well as beside the page, and keep which item a question was about as a link + back to it. + - Server-sent updates instead of the page polling for new items. - Saved filter views. - A daily digest of unreviewed items, as an automation starter. diff --git a/nextjs-frontend/__tests__/page-chat.test.tsx b/nextjs-frontend/__tests__/page-chat.test.tsx index 23aaebbef..2fe83333d 100644 --- a/nextjs-frontend/__tests__/page-chat.test.tsx +++ b/nextjs-frontend/__tests__/page-chat.test.tsx @@ -5,6 +5,7 @@ import "@testing-library/jest-dom"; import { Chats } from "@/app/openapi-client"; import { PageChat, SharedFrom } from "@/components/page-chat/page-chat"; import { + askAbout, hiddenOn, PAGE_CHAT_KEY, pageContext, @@ -211,6 +212,26 @@ describe("PageChat", () => { expect(sent[0]).toEqual({ content: "More?", context: { url: "/library?ns=calls", title: "Library" } }); }); + it("opens with what a page's own Ask about this hands it, and sends it as the highlighted part", async () => { + m(Chats.createChat).mockReturnValue(ok({ id: 9 })); + const sent: unknown[] = []; + m(streamSSE).mockImplementation(async function* (_url: string, opts: { body: unknown }) { + sent.push(opts.body); + yield { event: "done", data: JSON.stringify({ message: 2 }) }; + }); + renderChat(); + act(() => askAbout("Timeline item u1 (asset.added)\nAdeel: Adeel added a file")); + expect(await screen.findByRole("complementary", { name: "Page chat" })).toBeInTheDocument(); + expect(screen.getByText(/Timeline item u1/)).toBeInTheDocument(); + fireEvent.change(screen.getByRole("textbox", { name: "Ask a question" }), { target: { value: "What is it?" } }); + fireEvent.click(screen.getByRole("button", { name: "Send" })); + await waitFor(() => expect(sent).toHaveLength(1)); + expect(sent[0]).toMatchObject({ + content: "What is it?", + context: { url: "/library?ns=calls", selection: "Timeline item u1 (asset.added)\nAdeel: Adeel added a file" }, + }); + }); + it("isn't on the Chat page", () => { path = "/chat/3"; renderChat(); diff --git a/nextjs-frontend/__tests__/timeline-ui.test.tsx b/nextjs-frontend/__tests__/timeline-ui.test.tsx index d4b6d128d..44a0c2dd7 100644 --- a/nextjs-frontend/__tests__/timeline-ui.test.tsx +++ b/nextjs-frontend/__tests__/timeline-ui.test.tsx @@ -5,6 +5,7 @@ import type { ReactNode } from "react"; import { Activities } from "@/app/openapi-client"; import type { ActivityPage, StreamActivity } from "@/app/openapi-client/types.gen"; +import { ASK_ABOUT_EVENT } from "@/components/page-chat/page-chat-model"; import { LatestCard } from "@/components/timeline/latest-card"; import { TimelinePage } from "@/components/timeline/timeline-page"; import { ToastProvider } from "@/components/ui/toast"; @@ -188,6 +189,34 @@ test("keys: j and k move, r marks reviewed, i marks important", async () => { expect(m(Activities.markActivities)).not.toHaveBeenCalled(); }); +test("asking about an item hands the page chat its id, line and fields, by button or the a key", async () => { + const asked: string[] = []; + const listen = (e: Event) => asked.push((e as CustomEvent).detail); + window.addEventListener(ASK_ABOUT_EVENT, listen); + try { + wrap(); + const rows = within(await screen.findByRole("list", { name: "Timeline" })).getAllByTestId("timeline-row"); + fireEvent.click(within(rows[1]).getByRole("button", { name: "Show details" })); + fireEvent.click(within(rows[1]).getByRole("button", { name: "Ask about this" })); + expect(asked).toHaveLength(1); + expect(asked[0]).toContain("Timeline item u1 (asset.added, "); + expect(asked[0]).toContain("Adeel: Adeel received “Mail u1”"); + expect(asked[0]).toContain("Opens at /resources/u1"); + expect(asked[0]).toContain("Subject: Mail u1"); + // a on a line of several asks about all of them + fireEvent.keyDown(window, { key: "j" }); + fireEvent.keyDown(window, { key: "a" }); + expect(asked).toHaveLength(2); + expect(asked[1].match(/^Timeline item /gm)).toHaveLength(3); + expect(asked[1]).toContain("Timeline item e1 "); + // typing a in a field isn't a key for the page + fireEvent.keyDown(screen.getByRole("searchbox", { name: "Words in the summary" }), { key: "a" }); + expect(asked).toHaveLength(2); + } finally { + window.removeEventListener(ASK_ABOUT_EVENT, listen); + } +}); + test("hiding takes an item out of sight, with an undo", async () => { wrap(); const rows = within(await screen.findByRole("list", { name: "Timeline" })).getAllByTestId("timeline-row"); diff --git a/nextjs-frontend/components/page-chat/page-chat-model.ts b/nextjs-frontend/components/page-chat/page-chat-model.ts index 052c5165c..d87288730 100644 --- a/nextjs-frontend/components/page-chat/page-chat-model.ts +++ b/nextjs-frontend/components/page-chat/page-chat-model.ts @@ -35,6 +35,15 @@ export function readState(raw: string | null | undefined): PageChatState { } } +/** The event a page's own buttons send to open the page chat with a quote (askAbout). */ +export const ASK_ABOUT_EVENT = "lens:ask-about"; + +/** Open the page chat with `quote` as what the question is about, as "Ask about this" on highlighted text does: for + * buttons that hand it something of the page, like a timeline item. */ +export function askAbout(quote: string): void { + window.dispatchEvent(new CustomEvent(ASK_ABOUT_EVENT, { detail: quote })); +} + /** Pages where the page chat stays out of the way: the Chat page is the full chat already. */ export function hiddenOn(pathname: string): boolean { return pathname === "/chat" || pathname.startsWith("/chat/"); diff --git a/nextjs-frontend/components/page-chat/page-chat.tsx b/nextjs-frontend/components/page-chat/page-chat.tsx index c51c2ed03..9d9d25e2a 100644 --- a/nextjs-frontend/components/page-chat/page-chat.tsx +++ b/nextjs-frontend/components/page-chat/page-chat.tsx @@ -18,6 +18,7 @@ import { ToolSteps } from "@/components/chat/steps"; import { applyEvent, newTurn, savedSteps, sharedFiles, type ToolStep, type TurnState } from "@/components/chat/stream"; import { useAskPlace } from "@/components/page-chat/ask-place"; import { + ASK_ABOUT_EVENT, hiddenOn, INITIAL, PAGE_CHAT_KEY, @@ -101,6 +102,16 @@ export function PageChat() { [update], ); + // A page's own "Ask about this" (askAbout): a timeline item, say. + useEffect(() => { + const onAsk = (e: Event) => { + const quote = (e as CustomEvent).detail; + if (typeof quote === "string" && quote.trim()) open(quote.trim()); + }; + window.addEventListener(ASK_ABOUT_EVENT, onAsk); + return () => window.removeEventListener(ASK_ABOUT_EVENT, onAsk); + }, [open]); + // On a phone the panel covers the page: following a link (a citation, the page a question was asked on) shows it. const lastPath = useRef(pathname); useEffect(() => { diff --git a/nextjs-frontend/components/timeline/item.tsx b/nextjs-frontend/components/timeline/item.tsx index e9ead4d38..49acd5891 100644 --- a/nextjs-frontend/components/timeline/item.tsx +++ b/nextjs-frontend/components/timeline/item.tsx @@ -20,7 +20,9 @@ import { import Link from "next/link"; import { forwardRef, type KeyboardEvent, type ReactNode } from "react"; +import { askAbout } from "@/components/page-chat/page-chat-model"; import { + askQuote, familyOf, fieldLabel, groupSummary, @@ -104,6 +106,15 @@ export function MarkToggles({ ); } +/** Ask the assistant about items, in the chat at the side of the page: its line, fields and id go with the question. */ +export function AskButton({ items }: { items: Activity[] }) { + return ( + + ); +} + /** What an activity is about, on expand: its object's fields, what caused it, and room for what it caused. */ export function ActivityDetail({ a, @@ -148,6 +159,7 @@ export function ActivityDetail({ Open )} + {onHide && ( - )} - - )} +
+ {caused} + + {onHide && ( + + )} +
); } diff --git a/nextjs-frontend/components/timeline/model.ts b/nextjs-frontend/components/timeline/model.ts index ab8816349..1e66d8d28 100644 --- a/nextjs-frontend/components/timeline/model.ts +++ b/nextjs-frontend/components/timeline/model.ts @@ -299,6 +299,28 @@ export function fieldLabel(k: string): string { return s.charAt(0).toUpperCase() + s.slice(1); } +// ---------- asking about items ---------- +/** What "Ask about this" hands the page chat for items: for each, its id (so the assistant can mark it), kind and + * time, who did it and its line, where it opens and its fields. Whole items only, up to `max` characters. */ +export function askQuote(items: Activity[], max = 4000): string { + const out: string[] = []; + let size = 0; + for (const a of items) { + const href = objectHref(a); + const lines = [ + `Timeline item ${a.id} (${a.kind}, ${a.published})`, + `${a.actor.name}: ${a.summary}`, + ...(href ? [`Opens at ${href}`] : []), + ...objectFields(a).map(([k, v]) => `${fieldLabel(k)}: ${v}`), + ]; + const text = lines.join("\n"); + if (out.length && size + text.length + 2 > max) break; + out.push(text); + size += text.length + 2; + } + return out.join("\n\n").slice(0, max); +} + // ---------- the last view, per browser ---------- export const LAST_KEY = "lens.timeline.filters"; diff --git a/nextjs-frontend/components/timeline/timeline-page.tsx b/nextjs-frontend/components/timeline/timeline-page.tsx index 2df7c2a13..65d494be8 100644 --- a/nextjs-frontend/components/timeline/timeline-page.tsx +++ b/nextjs-frontend/components/timeline/timeline-page.tsx @@ -5,9 +5,11 @@ import { usePathname, useRouter, useSearchParams } from "next/navigation"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import type { ActivityMark } from "@/app/openapi-client/types.gen"; +import { askAbout } from "@/components/page-chat/page-chat-model"; import { FiltersBar, type Actor } from "@/components/timeline/filters"; import { TimelineRow, groupMark } from "@/components/timeline/item"; import { + askQuote, filterCount, fromParams, group, @@ -192,6 +194,11 @@ export function TimelinePage() { return; } const g = groups[active]; + if (g && e.key === "a") { + e.preventDefault(); + askAbout(askQuote(g.items)); + return; + } if (!g || (e.key !== "r" && e.key !== "i")) return; e.preventDefault(); const m = groupMark(g, marks.of); @@ -307,7 +314,7 @@ export function TimelinePage() { )}

- Keys: j and k move, Enter opens, r marks reviewed, i marks important. + Keys: j and k move, Enter opens, r marks reviewed, i marks important, a asks about it.

) : (