mirror of
https://github.com/anomalyco/hex.git
synced 2026-11-21 17:56:51 +08:00
* iOS: fall back to DictationTranscriber when a speech model won't install, and never show Apple's error text A TestFlight tester (iPhone 15, iOS 26.2, build 25, Russian) saw the full keyboard's strip read "transcription.ru asset not found after attempted download." Apple Speech lists some languages whose asset never arrives (Apple's forum shows the same for Arabic), and Hex both stopped there and showed Apple's sentence: UserFacingError (#2035) kept any description without an error code in it, and this one has none. - SpeechModelInstall (HEXShared, used by the app and in-keyboard dictation): install the chosen module's model, then the same language on DictationTranscriber when SpeechTranscriber's won't install, and go straight to DictationTranscriber for three days. When neither installs while online, Automatic sets the language aside for three days and moves to the next one ("Russian isn't available on this iPhone yet. Using English."); a language picked by hand says so and offers that one ("Try English", with Use English in the app). Offline or stalled, it says the download failed and keeps the language. Every message fits the full keyboard's strip. - StallWatch: a download whose progress hasn't moved for 60 s is cancelled and treated as failed, instead of holding the progress bar forever. - AppleSpeechAssets.reserve makes room under maximumReservedLocales by releasing other reservations first, and a reservation Hex can't get no longer fails preparation: the keyboard reserved every language it used and never released one, so the app's reserve could throw for good. - UserFacingError is now an allowlist: only Hex's own errors (HexDescribedError) show their text, a lost connection while getting the model ready says so, and anything else gets the context's sentence. This replaces the marker filter, which let Apple's readable sentences through. - DictationController re-prepares in the language Automatic moved to (at most twice), keeps a notice, and offers a language after an explicit choice fails. The dictation bar shows a failure on two smaller lines. - Snapshot scenes: full-keyboard-language-unavailable[-dark], keyboard-language-unavailable, language-unavailable, language-switched. Verified: xcodebuild build-for-testing on an iPhone 15 / iOS 27.2 simulator, and the 44 DictationController, picker, SpeechModelInstall and StallWatch tests pass (fallback, set-aside, offline, unsupported, stall, explicit offer, Automatic switch, error text). Apple Speech itself can't run in the simulator, so the install steps are fakes there; which module fails for Russian on iOS 26.2 is unverified on hardware (no iOS 26.x simulator runtime). * iOS: dictate English with Apple on iPhone 11, and stop the Parakeet relaunch loop A TestFlight tester (iPhone 11, iOS 27.3, build 16) reported the model download sticking at 50% and the app then crashing at every launch, with no crash log. iPhone 11 (A13, 4 GB) has no SpeechTranscriber (SpeechTranscriber.isAvailable is false on every A13 iPhone), so Hex used Parakeet V2 for English. FluidAudio 0.15.5 reports downloads as 0-50% and Core ML's compile as 50-100%, so 50% is MLModel(contentsOf:) compiling the encoder, which can't be cancelled; iOS ends the app there. Once the files are on disk, init saw hasCachedModel and loaded Parakeet again at once, so every launch ended the same way. iOS ending an app for memory leaves no crash report, which fits the empty crash list. - AppleSpeechTranscriber.shouldUse: English without SpeechTranscriber uses DictationTranscriber where Parakeet doesn't fit (under 5 GiB of RAM: every A13 iPhone) on iOS 26+. Parakeet stays where it fits and before iOS 26. - ParakeetLoadGuard: a load still marked as running at launch belonged to a process that ended, so the next launches don't load Parakeet by themselves until a load works; Hex says it closed while loading the model and waits for Set up. If Automatic now picks Apple Speech, it prepares that instead. - Parakeet's download half runs under StallWatch (90 s without progress ends it); the compile half isn't watched since it can't be stopped. FluidAudio's download errors (Hugging Face paths and HTML in their text) become "Parakeet V2 didn't download. Check your connection and try again." Verified: xcodebuild test on an iPhone 15 / iOS 27.2 simulator, 247 tests, 0 failures, 1 skipped (AppleSpeechClipTests, needs clips), including new tests for shouldUse on a 4 GB iPhone and the load guard across launches. Parakeet's memory use, the compile at 50% and the ended launches need a real iPhone 11; the simulator neither limits memory nor runs SpeechTranscriber.
209 lines
8.8 KiB
Swift
209 lines
8.8 KiB
Swift
import AVFAudio
|
||
import Foundation
|
||
import Speech
|
||
|
||
/// The languages Apple Speech transcribes on this iPhone, shared by the app and in-keyboard
|
||
/// dictation so both pick the same language and the same downloaded model.
|
||
@available(iOS 26.0, *)
|
||
nonisolated enum AppleSpeechLanguages {
|
||
/// The language dictation uses: Settings › Dictation language, else Automatic. Records
|
||
/// the result in the App Group for the keyboard layout and translation.
|
||
static func resolve(settings: LanguageSettings = LanguageSettings()) async -> SpeechLanguage? {
|
||
let explicit = settings.dictationLanguage
|
||
let picked = await pick(explicit: explicit, settings: settings)
|
||
if let picked, settings.resolvedDictationLanguage != picked.identifier {
|
||
settings.resolvedDictationLanguage = picked.identifier
|
||
}
|
||
if explicit == nil, let picked, settings.automaticDictationLanguage != picked.identifier {
|
||
settings.automaticDictationLanguage = picked.identifier
|
||
}
|
||
return picked
|
||
}
|
||
|
||
/// What Automatic picks on this iPhone, whatever the setting says. Records it for the
|
||
/// keyboard's language switch.
|
||
static func automatic(settings: LanguageSettings = LanguageSettings()) async -> SpeechLanguage? {
|
||
let picked = await pick(explicit: nil, settings: settings)
|
||
if let picked, settings.automaticDictationLanguage != picked.identifier {
|
||
settings.automaticDictationLanguage = picked.identifier
|
||
}
|
||
return picked
|
||
}
|
||
|
||
private static func pick(explicit: String?, settings: LanguageSettings) async -> SpeechLanguage? {
|
||
await SpeechLanguagePicker.pick(
|
||
explicit: explicit,
|
||
candidates: SpeechLanguagePicker.automaticCandidates(
|
||
preferred: Locale.preferredLanguages,
|
||
current: Locale.current.identifier
|
||
),
|
||
speech: { locale in
|
||
guard SpeechTranscriber.isAvailable,
|
||
let found = await SpeechTranscriber.supportedLocale(equivalentTo: locale) else { return nil }
|
||
// Its model wouldn't install lately: `DictationTranscriber`'s, if it has one.
|
||
guard !settings.speechModuleFailed(found.identifier(.bcp47)) else { return nil }
|
||
return found
|
||
},
|
||
dictation: { locale in
|
||
await DictationTranscriber.supportedLocale(equivalentTo: locale)
|
||
},
|
||
skip: { settings.isUnavailable($0.identifier(.bcp47)) }
|
||
)
|
||
}
|
||
|
||
/// Every language either module transcribes, sorted by name. Records the list for the
|
||
/// keyboard's language switch.
|
||
static func options(
|
||
display: Locale = .current,
|
||
settings: LanguageSettings = LanguageSettings()
|
||
) async -> [SpeechLanguageOption] {
|
||
var identifiers: [String] = []
|
||
var seen = Set<String>()
|
||
let speech = SpeechTranscriber.isAvailable ? await SpeechTranscriber.supportedLocales : []
|
||
for locale in speech + (await DictationTranscriber.supportedLocales) {
|
||
let identifier = locale.identifier(.bcp47)
|
||
if seen.insert(identifier).inserted {
|
||
identifiers.append(identifier)
|
||
}
|
||
}
|
||
if !identifiers.isEmpty, settings.supportedDictationLanguages != identifiers {
|
||
settings.supportedDictationLanguages = identifiers
|
||
}
|
||
return identifiers
|
||
.map { SpeechLanguageOption(id: $0, name: SpeechLanguagePicker.displayName(for: $0, in: display)) }
|
||
.sorted { $0.name.localizedStandardCompare($1.name) == .orderedAscending }
|
||
}
|
||
}
|
||
|
||
/// Apple Speech's model downloads, for the app and in-keyboard dictation alike.
|
||
@available(iOS 26.0, *)
|
||
nonisolated enum AppleSpeechAssets {
|
||
/// Installs `language`'s model (`SpeechModelInstall`: `SpeechTranscriber`, then
|
||
/// `DictationTranscriber`) and returns the language it installed. When neither installs,
|
||
/// the error names what Automatic picks instead, to offer or to say it switched.
|
||
static func install(
|
||
_ language: SpeechLanguage,
|
||
settings: LanguageSettings = LanguageSettings(),
|
||
log: @escaping @Sendable (String) -> Void = { _ in },
|
||
progress: @escaping @Sendable (Double) -> Void = { _ in }
|
||
) async throws -> SpeechLanguage {
|
||
do {
|
||
let installed = try await SpeechModelInstall.run(
|
||
language,
|
||
steps: steps,
|
||
settings: settings,
|
||
stallLimit: .seconds(60),
|
||
log: log,
|
||
progress: progress
|
||
)
|
||
settings.clearUnavailable(language.identifier)
|
||
return installed
|
||
} catch var unavailable as SpeechModelUnavailable {
|
||
unavailable.automatic = settings.dictationLanguage == nil
|
||
if unavailable.reason == .notOnThisIPhone,
|
||
let next = await AppleSpeechLanguages.automatic(settings: settings),
|
||
next.languageCode != language.languageCode {
|
||
unavailable.fallback = next.identifier
|
||
}
|
||
throw unavailable
|
||
}
|
||
}
|
||
|
||
static let steps = SpeechModelInstall.Steps(
|
||
dictationEquivalent: { locale in
|
||
await DictationTranscriber.supportedLocale(equivalentTo: locale)
|
||
},
|
||
status: { language in
|
||
switch await AssetInventory.status(forModules: [AppleSpeechModule(language).module]) {
|
||
case .installed: .installed
|
||
case .unsupported: .unsupported
|
||
default: .downloadable
|
||
}
|
||
},
|
||
download: { language, progress in
|
||
let module = AppleSpeechModule(language)
|
||
await reserve(language.locale)
|
||
guard let request = try await AssetInventory.assetInstallationRequest(supporting: [module.module]) else {
|
||
return
|
||
}
|
||
let watched = request.progress
|
||
let monitor = Task {
|
||
while !Task.isCancelled {
|
||
progress(watched.fractionCompleted)
|
||
try? await Task.sleep(for: .milliseconds(200))
|
||
}
|
||
}
|
||
defer { monitor.cancel() }
|
||
try await withTaskCancellationHandler {
|
||
try await request.downloadAndInstall()
|
||
} onCancel: {
|
||
watched.cancel()
|
||
}
|
||
},
|
||
isOnline: { await SpeechModelInstall.isOnline() }
|
||
)
|
||
|
||
/// Reserves `locale`'s assets, first releasing others when Hex already holds
|
||
/// `AssetInventory.maximumReservedLocales` (each language picked in the keyboard reserves
|
||
/// one, and the limit shrinks with free storage). Best effort: without a reservation the
|
||
/// download still runs, or fails with its own reason.
|
||
static func reserve(_ locale: Locale) async {
|
||
let key = locale.identifier(.bcp47)
|
||
var reserved = await AssetInventory.reservedLocales
|
||
guard !reserved.contains(where: { $0.identifier(.bcp47) == key }) else { return }
|
||
let limit = AssetInventory.maximumReservedLocales
|
||
while reserved.count >= max(1, limit), let oldest = reserved.first {
|
||
reserved.removeFirst()
|
||
_ = await AssetInventory.release(reservedLocale: oldest)
|
||
}
|
||
_ = try? await AssetInventory.reserve(locale: locale)
|
||
}
|
||
}
|
||
|
||
/// One dictation's Apple Speech module: `SpeechTranscriber` or `DictationTranscriber`.
|
||
@available(iOS 26.0, *)
|
||
nonisolated enum AppleSpeechModule {
|
||
case speech(SpeechTranscriber)
|
||
case dictation(DictationTranscriber)
|
||
|
||
init(_ language: SpeechLanguage) {
|
||
switch language.module {
|
||
case .speech:
|
||
self = .speech(SpeechTranscriber(locale: language.locale, preset: .transcription))
|
||
case .dictation:
|
||
// Punctuated, without the short-form hint: dictations run up to a minute.
|
||
self = .dictation(DictationTranscriber(locale: language.locale, preset: .longDictation))
|
||
}
|
||
}
|
||
|
||
var module: any SpeechModule {
|
||
switch self {
|
||
case .speech(let transcriber): transcriber
|
||
case .dictation(let transcriber): transcriber
|
||
}
|
||
}
|
||
|
||
var name: String {
|
||
switch self {
|
||
case .speech: "SpeechTranscriber"
|
||
case .dictation: "DictationTranscriber"
|
||
}
|
||
}
|
||
|
||
/// The final results joined, once the analyzer has finished the file.
|
||
func finalText() async throws -> String {
|
||
var transcript = AttributedString()
|
||
switch self {
|
||
case .speech(let transcriber):
|
||
for try await result in transcriber.results where result.isFinal {
|
||
transcript += result.text
|
||
}
|
||
case .dictation(let transcriber):
|
||
for try await result in transcriber.results where result.isFinal {
|
||
transcript += result.text
|
||
}
|
||
}
|
||
return String(transcript.characters).trimmingCharacters(in: .whitespacesAndNewlines)
|
||
}
|
||
}
|