Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions fastapi_backend/app/domain/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -712,6 +712,33 @@ def cancel(db, jid):
raise ValueError(f"the job has already {j['status']}")


def migrate_document_steps(db):
"""Documents and images queued before they had a read step of their own: what their runs have left reads them
instead of transcribing them, so they stop waiting for a speech model, and a run that failed at transcribe runs
again (unless a newer run of the same file is queued)."""
rows = db.rows(
"SELECT record::id(id) AS id, recording, steps, step_index, status FROM job "
"WHERE status IN $a OR (status = 'failed' AND next_step = 'transcribe')",
a=ACTIVE,
)
for j in rows:
if _source(db, j["recording"]) not in ("document", "image"):
continue
i, steps = j.get("step_index") or 0, [_spec(s) for s in j.get("steps") or []]
left = for_source(steps[i:], "document")
if left == steps[i:] or not left:
continue
db.q(
"UPDATE $j SET steps = $s, next_step = $n, updated_at = $t",
j=R("job", j["id"]),
s=steps[:i] + left,
n=left[0]["type"],
t=store.now(),
)
if j["status"] == "failed" and not any(r["recording"] == j["recording"] and r["status"] in ACTIVE for r in rows):
retry(db, j["id"])


def retry(db, jid):
j = get(db, jid)
if not j:
Expand Down
4 changes: 3 additions & 1 deletion fastapi_backend/app/domain/migrations.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@

def steps():
"""[(name, fn(db))] in the order they run. Append only."""
from . import access, auth, hierarchy
from . import access, auth, hierarchy, jobs

return [
# IIIF-only access levels (public, transcript, signed-in, private) become the access setting
Expand All @@ -57,6 +57,8 @@ def steps():
("collection-homes", hierarchy.migrate_homes),
# people who signed in before the first-sign-in welcome tour existed don't get it
("tour-seen", auth.migrate_tours),
# documents and images queued to be transcribed are read instead, and those that failed there run again
("document-read-step", jobs.migrate_document_steps),
]


Expand Down
2 changes: 1 addition & 1 deletion fastapi_backend/tests/api/test_setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def make():
def test_fresh_database_without_namespaces_starts(folder):
db = store.connect(bare_cfg(folder))
try:
assert migrations.pending(db) == [] and [m["state"] for m in migrations.status(db)] == ["done", "done", "done"]
assert migrations.pending(db) == [] and [m["state"] for m in migrations.status(db)] == ["done"] * len(migrations.steps())
assert store.space_names(db) == {}
finally:
db.close()
Expand Down
33 changes: 29 additions & 4 deletions fastapi_backend/tests/domain/test_migrations.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

# Every step that has shipped, in order. A database remembers steps by name, so these may never be renamed, reordered
# or removed: add new steps to the end of migrations.steps() and here.
SHIPPED = ["access-levels", "collection-homes", "tour-seen"]
SHIPPED = ["access-levels", "collection-homes", "tour-seen", "document-read-step"]


def test_steps_are_append_only():
Expand Down Expand Up @@ -62,8 +62,8 @@ def test_steps_an_older_lens_counted_are_not_run_again(db):
db.q("DELETE migration")
db.q("UPSERT $r SET n = 1", r=R("seq", "migrations"))
rid = _old_shape(db)
assert migrations.pending(db) == ["collection-homes", "tour-seen"]
assert store.migrate(db) == ["collection-homes", "tour-seen"]
assert migrations.pending(db) == ["collection-homes", "tour-seen", "document-read-step"]
assert store.migrate(db) == ["collection-homes", "tour-seen", "document-read-step"]
row = db.one("SELECT meta_json, collection FROM $r", r=R("recording", rid))
assert json.loads(row["meta_json"]) == {"access": "signed-in"} # access-levels was counted as done, so not rerun
assert row["collection"] == store.default_collection(db, store.ns_id(db, "pods"))
Expand Down Expand Up @@ -139,7 +139,7 @@ def test_lens_migrations_lists_and_runs(tmp_path, monkeypatch, capsys):
assert "pending access-levels" in out and "pending collection-homes" in out
cli.main(["--config", str(cfg_path), "migrations", "--run"])
out = capsys.readouterr().out
assert "ran 3 upgrade(s): access-levels, collection-homes, tour-seen" in out
assert "ran 4 upgrade(s): access-levels, collection-homes, tour-seen, document-read-step" in out
assert "done access-levels" in out


Expand Down Expand Up @@ -240,3 +240,28 @@ def test_people_who_signed_in_before_the_tour_skip_it(db):
db.q("UPDATE $r SET last_login_at = $t", r=R("account", old), t=store.now())
dict(migrations.steps())["tour-seen"](db)
assert auth.toured_at(db, old) and auth.toured_at(db, new) is None


def test_documents_waiting_to_be_transcribed_are_read_instead(db):
sid = "pods"
ids = {}
for name, source in (("doc", "document"), ("failed", "document"), ("song", "audio")):
ids[name] = db.next_id("recording")
db.q("CREATE $r CONTENT {source: $s, status: 'new', space: $sp}", r=R("recording", ids[name]), s=source, sp=sid)
old = ["transcribe", "diarize", "analyze", "report"]
for name, status in (("doc", "queued"), ("failed", "failed"), ("song", "queued")):
db.q(
"CREATE $j CONTENT {recording: $r, space: $sp, steps: $st, step_index: 0, next_step: 'transcribe', status: $s}",
j=R("job", ids[name]),
r=ids[name],
sp=sid,
st=[{"type": t} for t in old],
s=status,
)
dict(migrations.steps())["document-read-step"](db)
dict(migrations.steps())["document-read-step"](db) # safe to run twice
job = {n: db.one("SELECT steps, next_step, status FROM $j", j=R("job", i)) for n, i in ids.items()}
for n in ("doc", "failed"):
assert [s["type"] for s in job[n]["steps"]] == ["read", "analyze", "report"]
assert (job[n]["next_step"], job[n]["status"]) == ("read", "queued")
assert job["song"]["next_step"] == "transcribe" and len(job["song"]["steps"]) == 4
Loading