Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions nextjs-frontend/__tests__/document-model.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {
facesOn,
marksOn,
pageAt,
pageAtOffset,
pageNumber,
pageRef,
pagesSummary,
Expand Down Expand Up @@ -92,6 +93,16 @@ describe("a document's pages", () => {
expect(pageStart(SEGS, 1)).toBeNull(); // a page without text
});

it("knows which page is being read in pages scrolled one under another", () => {
const tops = [0, 1000, 2000, 3000];
expect(pageAtOffset(tops, 0)).toBe(0);
expect(pageAtOffset(tops, 999)).toBe(0);
expect(pageAtOffset(tops, 1000)).toBe(1);
expect(pageAtOffset(tops, 2500)).toBe(2);
expect(pageAtOffset(tops, 99999)).toBe(3);
expect(pageAtOffset([], 50)).toBe(0);
});

it("names pages, reads ?page= and steps the zoom", () => {
const pages = [page(0), page(1, { label: "iv" })];
expect(pageNumber(pages, 0)).toBe("1");
Expand Down
31 changes: 23 additions & 8 deletions nextjs-frontend/__tests__/recording-document.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,9 @@ function show(
);
}

const shown = () => screen.getByRole("img", { name: /^Page / });
// every page is there, one under another: the one being read
const reading = () => document.querySelector<HTMLElement>("[data-page-at][aria-current=page]")!;
const shown = () => within(reading()).getByRole("img", { name: /^Page / });

beforeAll(() => {
Element.prototype.scrollIntoView = jest.fn();
Expand Down Expand Up @@ -210,6 +212,19 @@ describe("a document's page", () => {
expect(screen.getByText("The summary panel")).toBeInTheDocument();
});

it("shows every page one under another, and scrolling turns the page being read", async () => {
show();
const pages = [...document.querySelectorAll<HTMLElement>("[data-page-at]")];
expect(pages).toHaveLength(3);
pages.forEach((el, i) => Object.defineProperty(el, "offsetTop", { value: i * 1000, configurable: true }));
const stage = pages[0].parentElement!;
Object.defineProperty(stage, "clientHeight", { value: 600, configurable: true });
stage.scrollTop = 2100;
fireEvent.scroll(stage);
await waitFor(() => expect(screen.getByRole("textbox", { name: "Page, of 3" })).toHaveValue("3"));
expect(shown()).toHaveAttribute("alt", "Page iii");
});

it("says what a page shows where it was described: to screen readers on the page, and above its text", () => {
const text = "A page of typed text with a drawing of a lighthouse.";
const model = {
Expand Down Expand Up @@ -308,15 +323,15 @@ describe("a document's page", () => {
),
],
} as unknown as PlayerModel;
const { container } = show({ model });
expect(container.querySelectorAll("[data-face]")).toHaveLength(0); // none on page 1
show({ model });
expect(reading().querySelectorAll("[data-face]")).toHaveLength(0); // none on page 1
fireEvent.click(screen.getByRole("tab", { name: "People" }));
const people = screen.getByRole("list", { name: "People on its pages" });
expect(within(people).getByText("On p. 2–iii")).toBeInTheDocument();
expect(screen.getByText(/boxes and pages, no names/)).toBeInTheDocument();
fireEvent.click(within(people).getByRole("button", { name: "Turn to Face 1's first page, p. 2" }));
expect(shown()).toHaveAttribute("alt", "Page 2");
const box = container.querySelector<HTMLElement>("[data-face='1']");
const box = reading().querySelector<HTMLElement>("[data-face='1']");
expect(box?.style.left).toBe("40%");
expect(box).toHaveTextContent("Face 1");
});
Expand Down Expand Up @@ -357,16 +372,16 @@ describe("a document's page", () => {
expect(within(list).getByText("on 2 pages · found 3 times")).toBeInTheDocument();
fireEvent.click(within(list).getByRole("button", { name: "Dog: show its boxes, first seen p. iii" }));
expect(shown()).toHaveAttribute("alt", "Page iii");
const dog = container.querySelectorAll<HTMLElement>("[data-object='dog']");
const dog = reading().querySelectorAll<HTMLElement>("[data-object='dog']");
expect(dog).toHaveLength(1);
expect([dog[0].style.left, dog[0].style.top]).toEqual(["50%", "60%"]);
expect(dog[0]).toHaveTextContent("Dog");
fireEvent.click(within(list).getByRole("button", { name: "Person: show its boxes, first seen p. 1" }));
expect(shown()).toHaveAttribute("alt", "Page 1");
expect(container.querySelectorAll("[data-object='person']")).toHaveLength(1);
expect(container.querySelectorAll("[data-object='dog']")).toHaveLength(0);
expect(reading().querySelectorAll("[data-object='person']")).toHaveLength(1);
expect(reading().querySelectorAll("[data-object='dog']")).toHaveLength(0);
fireEvent.click(screen.getByRole("button", { name: "Next page" }));
expect(container.querySelectorAll("[data-object='person']")).toHaveLength(2); // two people on p. 2
expect(reading().querySelectorAll("[data-object='person']")).toHaveLength(2); // two people on p. 2
fireEvent.click(within(list).getByRole("button", { name: "Person: stop showing its boxes" }));
expect(container.querySelectorAll("[data-object]")).toHaveLength(0);
});
Expand Down
182 changes: 155 additions & 27 deletions nextjs-frontend/components/recording/document/document-layout.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {
clampPage,
facesOn,
marksOn,
pageAtOffset,
pageNumber,
pagesSummary,
pageStart,
Expand Down Expand Up @@ -56,12 +57,18 @@ import { EmptyState } from "@/components/ui/states";
import { needRole } from "@/lib/hooks/session";
import { cn } from "@/lib/utils";

/** The page shown, the block of text marked on it, and the zoom; turning pages and choosing blocks. */
/**
* The page being read, the block of text marked on it, and the zoom; turning pages and choosing blocks. `jump` counts
* the moves made elsewhere (a page turned, a block chosen, the clock moved), which bring the page into view; `see` is
* the reader scrolling to a page, which doesn't.
*/
export type DocView = {
page: number;
selected: number | null;
zoom: number;
jump: number;
turn: (page: number) => void;
see: (page: number) => void;
choose: (block: number) => void;
setZoom: (z: number) => void;
};
Expand All @@ -78,6 +85,7 @@ export function useDocView(): DocView {
const [page, setPage] = useState(() => clampPage(startPage ?? 0, count));
const [selected, setSelected] = useState<number | null>(null);
const [zoom, setZoom] = useState(1);
const [jump, setJump] = useState(0);
const quiet = useRef(false);

useEffect(() => {
Expand All @@ -95,6 +103,7 @@ export function useDocView(): DocView {
if (i < 0 || !segs[i]) return;
setPage(segs[i].page ?? 0);
setSelected(i);
setJump((n) => n + 1);
});
}, [api, segs]);

Expand All @@ -103,6 +112,7 @@ export function useDocView(): DocView {
const to = clampPage(p, count);
setPage(to);
setSelected(null);
setJump((n) => n + 1);
const t0 = pageStart(segs, to);
if (t0 != null) {
quiet.current = true;
Expand All @@ -127,7 +137,9 @@ export function useDocView(): DocView {
turn(startPage);
}, [startPage, turn]);

return { page, selected, zoom, turn, choose, setZoom };
const see = useCallback((p: number) => setPage(clampPage(p, count)), [count]);

return { page, selected, zoom, jump, turn, see, choose, setZoom };
}

const MARK: Record<Mark["kind"], string> = {
Expand Down Expand Up @@ -276,25 +288,129 @@ function PageField({ view, count }: { view: DocView; count: number }) {
);
}

/** The page shown, with a toolbar to turn pages and zoom; `object` is the kind of object whose boxes to draw. */
function PageStage({ view, compact, object = null }: { view: DocView; compact?: boolean; object?: string | null }) {
const { model, state, find } = useRec();
const count = model.pages.length;
const p = model.pages[view.page];
/** A page in the pages scrolled one under another: drawn once it's near the view, a blank its size until then. */
function StagePage({
page,
view,
near,
object,
}: {
page: PageInfo;
view: DocView;
near: boolean;
object: string | null;
}) {
const { model, find } = useRec();
const idx = page.idx;
const marks = useMemo(
() => marksOn(model.segments, view.page, view.selected, find.hits, find.index),
[model.segments, view.page, view.selected, find.hits, find.index],
() => (near ? marksOn(model.segments, idx, view.selected, find.hits, find.index) : []),
[near, model.segments, idx, view.selected, find.hits, find.index],
);
const faces = useMemo(
() => (model.facesMode === "off" ? [] : facesOn(model.faces, view.page)),
[model.faces, model.facesMode, view.page],
() => (near && model.facesMode !== "off" ? facesOn(model.faces, idx) : []),
[near, model.faces, model.facesMode, idx],
);
const things = useMemo(() => {
const t = object ? model.objects.find((o) => o.label === object) : undefined;
return t ? boxesOnPage(t, view.page) : [];
}, [model.objects, object, view.page]);
const name = `Page ${pageNumber(model.pages, view.page)}`;
const shows = descriptionOf(model.descriptions, view.page, true); // what it shows, for screen readers
const t = near && object ? model.objects.find((o) => o.label === object) : undefined;
return t ? boxesOnPage(t, idx) : [];
}, [near, model.objects, object, idx]);
if (!near)
return (
<div
aria-hidden
className="mx-auto rounded-[2px] bg-surface shadow-2"
style={{
width: `${view.zoom * 100}%`,
maxWidth: page.width ? `${Math.round(page.width * view.zoom)}px` : undefined,
aspectRatio: page.width && page.height ? `${page.width} / ${page.height}` : "3 / 4",
}}
/>
);
const name = `Page ${pageNumber(model.pages, idx)}`;
const shows = descriptionOf(model.descriptions, idx, true); // what it shows, for screen readers
return (
<PageImage
page={page}
marks={marks}
faces={faces}
things={things}
thing={object ?? ""}
zoom={view.zoom}
name={shows ? `${name}: ${shows.text}` : name}
/>
);
}

/**
* Every page, one under another, with a toolbar to turn pages and zoom; `object` is the kind of object whose boxes to
* draw. Scrolling turns the page being read; turning a page, or choosing a block, scrolls to it. Pages are drawn as
* they near the view, and kept once drawn.
*/
function PageStage({ view, compact, object = null }: { view: DocView; compact?: boolean; object?: string | null }) {
const { model, state } = useRec();
const count = model.pages.length;
const p = model.pages[view.page];
const box = useRef<HTMLDivElement>(null);
const els = useRef(new Map<number, HTMLElement>());
const [drawn, setDrawn] = useState<ReadonlySet<number>>(() => new Set());
const quiet = useRef(false); // the next scroll is ours: not the reader turning the page
const frame = useRef(0);

// pages near the view are drawn (where the browser can tell; the page being read and its neighbours always are)
useEffect(() => {
const root = box.current;
if (!root || typeof IntersectionObserver === "undefined") return;
const io = new IntersectionObserver(
(entries) => {
const add = entries
.filter((e) => e.isIntersecting)
.map((e) => Number((e.target as HTMLElement).dataset.pageAt));
if (add.length) setDrawn((was) => (add.every((i) => was.has(i)) ? was : new Set([...was, ...add])));
},
{ root, rootMargin: "100% 0px" },
);
for (const el of els.current.values()) io.observe(el);
return () => io.disconnect();
}, [count]);

// a page turned, a block chosen or the zoom changed: the page, or the block marked on it, comes into view
const { jump, zoom, page, selected, see } = view;
const first = useRef(true);
useEffect(() => {
if (first.current) {
first.current = false;
if (!jump) return;
}
const root = box.current;
const el = els.current.get(page);
if (!root || !el) return;
const mark = selected != null ? el.querySelector<HTMLElement>("[data-mark=selected]") : null;
const r = root.getBoundingClientRect();
const t = (mark ?? el).getBoundingClientRect();
const top = mark
? root.scrollTop + t.top - r.top - Math.max(16, (root.clientHeight - t.height) / 2)
: root.scrollTop + t.top - r.top - 12;
quiet.current = true;
root.scrollTo?.({ top: Math.max(0, top) });
}, [jump, zoom]); // only on a move, not as the reader scrolls

const onScroll = () => {
if (quiet.current) {
quiet.current = false;
return;
}
if (frame.current) return;
frame.current = requestAnimationFrame(() => {
frame.current = 0;
const root = box.current;
if (!root) return;
const tops = model.pages.map((x) => els.current.get(x.idx)?.offsetTop ?? 0);
const at = model.pages[pageAtOffset(tops, root.scrollTop + root.clientHeight / 3)]?.idx;
if (at != null && at !== page) see(at);
});
};
useEffect(() => () => cancelAnimationFrame(frame.current), []);

return (
<section aria-label="Pages" className="flex min-h-0 min-w-0 flex-col bg-surface-neutral">
<div className="flex flex-wrap items-center gap-1.5 border-b border-border bg-background px-3 py-2">
Expand Down Expand Up @@ -338,17 +454,29 @@ function PageStage({ view, compact, object = null }: { view: DocView; compact?:
<Plus />
</IconButton>
</div>
<div className={cn("min-h-0 flex-1 overflow-auto p-4", compact && "max-h-[58dvh] min-h-[320px] p-3")}>
{p ? (
<PageImage
page={p}
marks={marks}
faces={faces}
things={things}
thing={object ?? ""}
zoom={view.zoom}
name={shows ? `${name}: ${shows.text}` : name}
/>
<div
ref={box}
onScroll={onScroll}
className={cn(
"relative flex min-h-0 flex-1 flex-col gap-4 overflow-auto p-4",
compact && "max-h-[58dvh] min-h-[320px] gap-3 p-3",
)}
>
{count > 0 ? (
model.pages.map((x) => (
<div
key={x.idx}
data-page-at={x.idx}
aria-current={x.idx === page ? "page" : undefined}
ref={(el) => {
if (el) els.current.set(x.idx, el);
else els.current.delete(x.idx);
}}
className="shrink-0"
>
<StagePage page={x} view={view} near={Math.abs(x.idx - page) <= 1 || drawn.has(x.idx)} object={object} />
</div>
))
) : state.phase === "processing" || state.phase === "analyzing" ? (
<EmptyState icon={<Loader2 className="animate-spin" />} title="Drawing its pages" className="py-16">
Its pages appear here as soon as they&apos;re drawn and read.
Expand Down
18 changes: 18 additions & 0 deletions nextjs-frontend/components/recording/document/model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,24 @@ export function clampPage(idx: number, count: number): number {
return Math.max(0, Math.min(Math.max(0, count - 1), Math.round(idx)));
}

/**
* The page being read in pages scrolled one under another: the last whose top (`tops`, in order) is at or above `y`,
* the line a reader's eye is on.
*/
export function pageAtOffset(tops: number[], y: number): number {
let lo = 0;
let hi = tops.length - 1;
let at = 0;
while (lo <= hi) {
const mid = (lo + hi) >> 1;
if (tops[mid] <= y) {
at = mid;
lo = mid + 1;
} else hi = mid - 1;
}
return at;
}

/** The next zoom in (dir 1) or out (-1); the fit (1) when it's in between. */
export function zoomStep(z: number, dir: 1 | -1): number {
if (dir > 0) return ZOOMS.find((x) => x > z + 1e-6) ?? ZOOMS[ZOOMS.length - 1];
Expand Down
Loading