Files
Ryan Vogel 23baa80aaa Measure on-device Android dictation across four languages
Replace the phase-one single-phrase evidence with a fixed 40-clip multilingual, partly music-mixed corpus. Record per-clip errors, load, warm inference, memory, two-core sensitivity and delivered platform-service results separately. Keep an independent foreground editor and media track for mic-to-insertion verification.

Verified on an API 35 ARM64 Studio emulator: Whisper tiny/base/small, Parakeet TDT v3 int8 and strictly on-device Android recognition. Small in its short window made 15/285 word errors; the platform service returned 33/40. Both native runners also decoded the notebook clip after tightening the PCM input checks.
2026-11-20 12:26:12 -05:00

46 lines
2.6 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Micro-average word error (sum edit distance / sum reference words); never omit a failed clip."""
import csv, re, statistics, unicodedata, sys
from pathlib import Path
def words(s):
s=unicodedata.normalize("NFC",s).casefold().replace("’","'")
return re.findall(r"[^\W_]+(?:'[^\W_]+)*",s,re.UNICODE)
def distance(a,b):
d=list(range(len(b)+1))
for i,x in enumerate(a,1):
n=[i]
for j,y in enumerate(b,1):
n.append(min(n[-1]+1,d[j]+1,d[j-1]+int(x!=y)))
d=n
return d[-1]
def summarize(rows):
def wer(rs):
return 100*sum(r["errors"] for r in rs)/sum(r["words"] for r in rs) if rs else None
result={k:wer([r for r in rows if fn(r)]) for k,fn in {
"all":lambda r:True,"completed":lambda r:r["status"]=="ok","clean":lambda r:r["snr_db"]=="clean","music":lambda r:r["snr_db"]!="clean",
**{l:(lambda r,l=l:r["lang"]==l) for l in ("en","es","de","no")}}.items()}
times=sorted(float(r["wall_ms"]) for r in rows if r["wall_ms"]!="" and r["status"]=="ok")
result.update(ok=len(times),p50_ms=statistics.median(times) if times else None,
p95_ms=times[(len(times)*95+99)//100-1] if times else None,
peak_mib=max((int(r["hwm_kib"])/1024 for r in rows if r["hwm_kib"]),default=None))
return {k:round(v,2) if v is not None else None for k,v in result.items()}
if __name__=="__main__":
base=Path(__file__).parent
with (base/"corpus.tsv").open() as f: corpus={r["id"]:r for r in csv.DictReader(f,delimiter="\t")}
data=Path(sys.argv[1])
fields="engine id lang snr_db words errors wall_ms cpu_ms load_ms hwm_kib ready_to_result_ms stop_to_result_ms hypothesis status".split()
out=csv.DictWriter(sys.stdout,fieldnames=fields,delimiter="\t",lineterminator="\n");out.writeheader()
for name in ("tiny","base","small","small-short","parakeet","android"):
if not (data/(name+".tsv")).exists():continue
rows=[]
for x in (data/(name+".tsv")).read_text().splitlines():
if name=="android":
id,wall,status,ready,stop,text=x.split("\t",5);cpu=load=hwm=""
else:
id,wall,cpu,load,hwm,text=x.split("\t",5); status="ok" if text.strip() else "empty";ready=stop=""
c=corpus[id]; ref=words(c["reference"])
r=dict(zip(fields,[name,id,c["lang"],c["snr_db"],len(ref),distance(ref,words(text)),wall,cpu,load,hwm,ready,stop,text.strip(),status]))
rows.append(r);out.writerow(r)
assert len(rows)==len(corpus) and len({r["id"] for r in rows})==len(corpus)
print(name,summarize(rows),file=sys.stderr)