Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,9 @@ The setup wizard's model provider step and Settings → LLM provider offer the m
Studio, llama.cpp, vLLM, LocalAI) and these cloud providers, each through its OpenAI-compatible API. Picking one fills
in the address and a model, and a cloud provider's API key field takes the focus, with a link to where the provider
hands out keys: paste the key and save. The model stays editable, with the provider's other models suggested.
**Test** tries the address, model and key on screen, saved or not; the saved key goes only to the saved address, so
a new address is tested with the key typed for it. Over the API, `POST /api/v1/settings/llm/test` takes the same
`base_url`, `model` and `api_key`, and with no body tests what is saved.

| Provider | Address | Model filled in |
|---|---|---|
Expand Down
8 changes: 6 additions & 2 deletions fastapi_backend/app/api/v1/routes/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
HttpsStatus,
IndexQueued,
LlmTestResult,
LlmTry,
LocalLlmStatus,
LogLine,
LogStatus,
Expand Down Expand Up @@ -138,8 +139,11 @@ def get_encryption(user: AdminReader) -> EncryptionProgress:


@router.post("/settings/llm/test")
def test_llm(user: AdminWriter, cfg: Cfg) -> LlmTestResult:
"""Ask the configured model for one word, to check the address, key and model name."""
def test_llm(user: AdminWriter, cfg: Cfg, body: LlmTry | None = None) -> LlmTestResult:
"""Ask the model for one word, to check the address, key and model name: what is given (typed, not saved yet),
else what is saved."""
if body:
cfg = {**cfg, "llm": llm.trying(cfg["llm"], body.base_url, body.model, body.api_key)}
if not llm.configured(cfg):
return LlmTestResult(ok=False, error="set a base URL and a model first")
t0 = time.time()
Expand Down
16 changes: 16 additions & 0 deletions fastapi_backend/app/domain/llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,22 @@ def configured(cfg):
return bool(cfg["llm"].get("base_url") and cfg["llm"].get("model"))


def trying(saved, base_url=None, model=None, api_key=None):
"""The LLM settings with typed values over the saved ones, for a test before saving. The saved key (stored or
from the environment) goes only to the saved address, so a new address never receives it."""
l = dict(saved)
if base_url and base_url.strip():
url = base_url.strip()
if url.rstrip("/") != (saved.get("base_url") or "").rstrip("/"):
l.update(api_key=None, api_key_env=None)
l["base_url"] = url
if model and model.strip():
l["model"] = model.strip()
if api_key:
l["api_key"] = api_key
return l


def _post(cfg, payload):
l = cfg["llm"]
if not configured(cfg):
Expand Down
11 changes: 10 additions & 1 deletion fastapi_backend/app/domain/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -936,6 +936,7 @@ def __init__(self, cfg, pool_size=None):
self._all = []
# one connection now, the rest of the pool as concurrent queries need them: each server sign-in costs time
self._size, self._grow_lock = size, threading.Lock()
self._held = threading.local() # the connection the current thread is using, if any (conn)
try:
c = self._open()
self._all.append(c)
Expand Down Expand Up @@ -1002,17 +1003,25 @@ def closed(self):
def conn(self):
"""A connection from the pool. A server connection that has dropped (the server restarted, or a keepalive
ping went unanswered while the machine slept) is replaced: without that, every query sent on it fails until
the process restarts."""
the process restarts. A thread already using a connection gets that one again: the garbage collector closes an
abandoned streamed answer (its reader left) on whatever thread it runs on, mid-query, and that answer's cleanup
queries too; with the embedded engine's one connection held by that very thread, waiting for it would hang the
whole server."""
if (held := getattr(self._held, "c", None)) is not None:
yield held
return
c = self._take()
try:
if _closed(c):
c = self._replace(c)
self._held.c = c
yield c
except Exception as e:
if not self.embedded and _lost(e):
c = self._replace(c)
raise
finally:
self._held.c = None
self._pool.put(c)

def _take(self):
Expand Down
10 changes: 10 additions & 0 deletions fastapi_backend/app/schemas/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,16 @@ class EncryptionProgress(ResponseModel):
finished_at: float | None = None


class LlmTry(RequestModel):
"""What to test before saving it (Settings → LLM provider); anything left out is taken from what is saved."""

base_url: str | None = Field(None, max_length=500, description="the server's address, as typed")
model: str | None = Field(None, max_length=200, description="the model name, as typed")
api_key: str | None = Field(
None, max_length=500, description="a key typed but not saved; the saved key is used only at the saved address"
)


class LlmTestResult(ResponseModel):
ok: bool
error: str | None = None
Expand Down
22 changes: 22 additions & 0 deletions fastapi_backend/tests/api/test_admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,28 @@ def test_llm_test_without_a_model(client, db):
}


def test_llm_test_tries_what_is_typed(client, db, cfg):
make_user(db, "root@x.io", "root password 1", admin=True)
h = login(client, "root@x.io", "root password 1")
saved = cfg["llm"]["base_url"]
assert client.put("/api/v1/settings/llm", json={"base_url": "http://127.0.0.1:9/v1"}, headers=h).status_code == 200
assert not client.post("/api/v1/settings/llm/test", headers=h).json()["ok"] # nothing typed: the saved address
r = client.post("/api/v1/settings/llm/test", json={"base_url": saved, "model": "fake"}, headers=h).json()
assert r["ok"] and r["model"] == "fake" # the typed address, not saved yet
assert client.get("/api/v1/settings", headers=h).json()["llm"]["values"]["base_url"] == "http://127.0.0.1:9/v1"


def test_llm_trying_keeps_the_saved_key_at_the_saved_address():
from app.domain import llm

saved = {"base_url": "http://a/v1", "model": "m", "api_key": "k", "api_key_env": "E"}
assert llm.trying(saved, "http://a/v1/", "n")["api_key"] == "k"
other = llm.trying(saved, "http://b/v1")
assert other["api_key"] is None and other["api_key_env"] is None and other["model"] == "m"
assert llm.trying(saved, "http://b/v1", None, "typed")["api_key"] == "typed"
assert llm.trying(saved) == saved and saved["api_key"] == "k"


def test_admin_pages_need_an_admin(client, db):
make_user(db, "ed@x.io", "editor password 1", roles={"pods": "editor"})
h = login(client, "ed@x.io", "editor password 1")
Expand Down
2 changes: 1 addition & 1 deletion fastapi_backend/tests/domain/test_store_reconnect.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def close(self):
def _db(monkeypatch, first):
d = store.DB.__new__(store.DB)
d.url, d.embedded = "ws://test", False
d._pool = store.queue.LifoQueue()
d._pool, d._held = store.queue.LifoQueue(), store.threading.local()
d._all = [first]
d._pool.put(first)
fresh = _Conn()
Expand Down
57 changes: 57 additions & 0 deletions fastapi_backend/tests/domain/test_store_reentrant.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
"""A thread that holds a database connection and asks for one again gets the same one, instead of waiting forever for
itself. The garbage collector closes an abandoned streamed answer (its reader navigated away) on whatever thread it
runs on, in the middle of that thread's own query, and the answer's cleanup queries too; with the embedded engine's
single connection that was a self-deadlock that froze the whole server (seen in the clicks survey, PR #400)."""

from __future__ import annotations

import threading

import pytest


def _in_time(fn, seconds=15):
"""`fn` run on its own thread, so a deadlock fails the test instead of hanging it; its result."""
out = {}
t = threading.Thread(target=lambda: out.setdefault("value", fn()), daemon=True)
t.start()
t.join(seconds)
assert not t.is_alive(), "the query waited forever for the connection its own thread holds"
return out["value"]


def test_a_generator_closed_mid_query_can_still_query(db):
"""What the garbage collector does to a left-behind streamed answer: close it, on a thread that is itself inside a
query, so its cleanup's query runs while that thread holds the connection."""
if not db.embedded:
pytest.skip("a server gets a pool: a second connection answers")
seen = {}

def answer():
try:
yield "token"
finally:
seen["cleanup"] = db.values("RETURN 2") # on.end() in a streamed chat answer

def run():
g = answer()
next(g)
with db.conn(): # this thread is mid-query (the connection is taken) when the collector closes the generator
g.close()
return db.values("RETURN 3")

assert _in_time(run) == [3] and seen["cleanup"] == [2]


def test_the_connection_comes_back_to_the_pool_afterwards(db):
if not db.embedded:
pytest.skip("a server gets a pool: a second connection answers")

def run():
with db.conn():
inner = db.values("RETURN 1")
return inner, db.values("RETURN 4")

assert _in_time(run) == ([1], [4])
# another thread is not given this thread's connection: it takes it from the pool, after it is back
assert _in_time(lambda: db.values("RETURN 5")) == [5]
2 changes: 1 addition & 1 deletion fastapi_backend/tests/domain/test_store_retry.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def query(self, sql, v):
def _db(conn):
d = store.DB.__new__(store.DB)
d.url, d.embedded = "ws://test", False
d._pool = store.queue.LifoQueue()
d._pool, d._held = store.queue.LifoQueue(), store.threading.local()
d._all = [conn]
d._pool.put(conn)
return d
Expand Down
17 changes: 17 additions & 0 deletions nextjs-frontend/__tests__/settings-llm-presets.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,23 @@ describe("Settings › LLM provider presets", () => {
});
});

it("tests the address and model on screen before they are saved", async () => {
m(Admin.testLlm)
.mockReset()
.mockImplementation(() => ok({ ok: true, reply: "OK", ms: 50, model: "qwen3" }));
show(view(true));
fireEvent.change(screen.getByLabelText("Base URL (OpenAI-compatible)"), {
target: { value: "http://10.0.20.11:1434/v1" },
});
fireEvent.change(screen.getByLabelText("Model"), { target: { value: "qwen3" } });
fireEvent.click(screen.getByRole("button", { name: "Test" }));
await waitFor(() => expect(Admin.testLlm).toHaveBeenCalledTimes(1));
expect(m(Admin.testLlm).mock.calls[0][0]).toMatchObject({
body: { base_url: "http://10.0.20.11:1434/v1", model: "qwen3", api_key: null },
});
expect(Admin.updateSettings).not.toHaveBeenCalled();
});

it("opens the saved key for replacing when the provider changes", async () => {
show(view(true));
expect(screen.getByText(/^Set/)).toBeInTheDocument();
Expand Down
3 changes: 3 additions & 0 deletions nextjs-frontend/app/openapi-client/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2110,6 +2110,7 @@ export type {
LlmProvidersResponse,
LlmProvidersResponses,
LlmTestResult,
LlmTry,
LocalLlmProgress,
LocalLlmStatus,
LocalLlmStatusData,
Expand Down Expand Up @@ -3313,6 +3314,8 @@ export type {
TestFileStoreResponse,
TestFileStoreResponses,
TestLlmData,
TestLlmError,
TestLlmErrors,
TestLlmResponse,
TestLlmResponses,
TestLogsData,
Expand Down
12 changes: 9 additions & 3 deletions nextjs-frontend/app/openapi-client/sdk.gen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1577,6 +1577,7 @@ import type {
TestFileStoreErrors,
TestFileStoreResponses,
TestLlmData,
TestLlmErrors,
TestLlmResponses,
TestLogsData,
TestLogsResponses,
Expand Down Expand Up @@ -2881,14 +2882,19 @@ export class Admin {
/**
* Test Llm
*
* Ask the configured model for one word, to check the address, key and model name.
* Ask the model for one word, to check the address, key and model name: what is given (typed, not saved yet),
* else what is saved.
*/
public static testLlm<ThrowOnError extends boolean = false>(
options?: Options<TestLlmData, ThrowOnError>,
): RequestResult<TestLlmResponses, unknown, ThrowOnError> {
return (options?.client ?? client).post<TestLlmResponses, unknown, ThrowOnError>({
): RequestResult<TestLlmResponses, TestLlmErrors, ThrowOnError> {
return (options?.client ?? client).post<TestLlmResponses, TestLlmErrors, ThrowOnError>({
url: "/api/v1/settings/llm/test",
...options,
headers: {
"Content-Type": "application/json",
...options?.headers,
},
});
}

Expand Down
40 changes: 39 additions & 1 deletion nextjs-frontend/app/openapi-client/types.gen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10579,6 +10579,32 @@ export type LlmTestResult = {
[key: string]: unknown;
};

/**
* LlmTry
*
* What to test before saving it (Settings → LLM provider); anything left out is taken from what is saved.
*/
export type LlmTry = {
/**
* Base Url
*
* the server's address, as typed
*/
base_url?: string | null;
/**
* Model
*
* the model name, as typed
*/
model?: string | null;
/**
* Api Key
*
* a key typed but not saved; the saved key is used only at the saved address
*/
api_key?: string | null;
};

/**
* LocalLlmProgress
*/
Expand Down Expand Up @@ -26575,12 +26601,24 @@ export type GetEncryptionResponses = {
export type GetEncryptionResponse = GetEncryptionResponses[keyof GetEncryptionResponses];

export type TestLlmData = {
body?: never;
/**
* Body
*/
body?: LlmTry | null;
path?: never;
query?: never;
url: "/api/v1/settings/llm/test";
};

export type TestLlmErrors = {
/**
* Validation Error
*/
422: HttpValidationError;
};

export type TestLlmError = TestLlmErrors[keyof TestLlmErrors];

export type TestLlmResponses = {
/**
* Successful Response
Expand Down
13 changes: 12 additions & 1 deletion nextjs-frontend/components/settings/bodies.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -648,8 +648,19 @@ function LlmBody({ ctx }: { ctx: BodyCtx }) {
const key = ctx.state("llm.api_key");
const baseUrl = ctx.state("llm.base_url");
const model = ctx.state("llm.model");
// Test what is on screen, saved or not; the saved key goes only to the saved address
const test = useMutation({
mutationFn: () => data(Admin.testLlm({ client })),
mutationFn: () =>
data(
Admin.testLlm({
client,
body: {
base_url: String(baseUrl.value ?? "") || null,
model: String(model.value ?? "") || null,
api_key: (key.value as string | undefined) || null,
},
}),
),
});
// The setup wizard's provider step: a server found nearby or a cloud provider fills in the address and the model,
// and a cloud provider's key is asked for next
Expand Down
Loading
Loading