Merge pull request #2034 from anomalyco/ios-norwegian

iOS: dictation in every Apple language, a keyboard language switch, and Norwegian, Spanish and German layouts
This commit is contained in:
Ryan Vogel
2026-11-20 07:55:31 -05:00
committed by GitHub
38 changed files with 1969 additions and 167 deletions
+23 -27
View File
@@ -191,19 +191,25 @@ final class KeyboardLocalDictation {
transcription = Task { [weak self] in
let started = Date()
do {
let (heard, locale) = try await KeyboardSpeech.transcribe(url, hints: hints)
let (heard, language) = try await KeyboardSpeech.transcribe(url, hints: hints)
try? FileManager.default.removeItem(at: url)
let locale = language.identifier
let corrected = DictationCorrections.apply(fixes, to: heard)
let style = KeyboardBridge().style
let cleaned = await CleanupTagger.cleanIfEnabled(corrected, style: style)
let translated = await DictationTranslator.translateIfEnabled(cleaned)
let cleaned = await CleanupTagger.cleanIfEnabled(corrected, style: style, language: locale)
let translated = await DictationTranslator.translateIfEnabled(
cleaned,
source: language.locale.language
)
let text = translated ?? cleaned
KeyboardProbeLog.shared.write("transcribed in \(String(format: "%.2f", Date().timeIntervalSince(started))) s: \(text.count) chars \(KeyboardProbeLog.memory())")
guard let self, !Task.isCancelled else { return }
// Record typing logs keeps every step; nothing is kept while it is off.
let stages: DictationStages? = TypingLogSettings().isEnabled
? DictationStages(
engine: "Apple Speech (keyboard)",
engine: language.module == .dictation
? "Apple Speech (keyboard, DictationTranscriber)"
: "Apple Speech (keyboard)",
locale: locale,
style: style.rawValue,
recordedMS: Int((seconds * 1000).rounded()),
@@ -473,36 +479,30 @@ private enum KeyboardSpeech {
}
}
/// The transcript and the speech locale's identifier.
static func transcribe(_ url: URL, hints: [String]) async throws -> (String, String) {
/// The transcript and the language it was transcribed in.
static func transcribe(_ url: URL, hints: [String]) async throws -> (String, SpeechLanguage) {
let log = KeyboardProbeLog.shared
// Same order as the app's AppleSpeechTranscriber.preferredLocale(), so the
// The same rule as the app's (Settings › Dictation language, else Automatic), so the
// keyboard uses the assets Hex downloaded.
var locale: Locale?
for identifier in Locale.preferredLanguages + [Locale.current.identifier, "en-US"] {
if let supported = await SpeechTranscriber.supportedLocale(equivalentTo: Locale(identifier: identifier)) {
locale = supported
break
}
}
guard let locale else { throw Failure.noLocale }
let transcriber = SpeechTranscriber(locale: locale, preset: .transcription)
let status = await AssetInventory.status(forModules: [transcriber])
log.write("speech locale=\(locale.identifier) assets=\(status) available=\(SpeechTranscriber.isAvailable)")
guard let language = await AppleSpeechLanguages.resolve() else { throw Failure.noLocale }
let locale = language.locale
let speech = AppleSpeechModule(language)
let status = await AssetInventory.status(forModules: [speech.module])
log.write("speech locale=\(locale.identifier) module=\(speech.name) assets=\(status) available=\(SpeechTranscriber.isAvailable)")
if status != .installed {
let started = Date()
_ = try? await AssetInventory.reserve(locale: locale)
if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
if let request = try await AssetInventory.assetInstallationRequest(supporting: [speech.module]) {
log.write("downloading speech assets")
try await request.downloadAndInstall()
}
let after = await AssetInventory.status(forModules: [transcriber])
let after = await AssetInventory.status(forModules: [speech.module])
log.write("speech assets \(after) after \(String(format: "%.1f", Date().timeIntervalSince(started))) s \(KeyboardProbeLog.memory())")
guard after == .installed else { throw Failure.assetsMissing("\(after)") }
}
let analyzer = SpeechAnalyzer(
modules: [transcriber],
modules: [speech.module],
options: SpeechAnalyzer.Options(priority: .userInitiated, modelRetention: .whileInUse)
)
let file = try AVAudioFile(forReading: url)
@@ -515,13 +515,9 @@ private enum KeyboardSpeech {
try await analyzer.start(inputAudioFile: file, finishAfterFile: true)
log.write("analyzer started \(KeyboardProbeLog.memory())")
var transcript = AttributedString()
for try await result in transcriber.results where result.isFinal {
transcript += result.text
}
let text = String(transcript.characters).trimmingCharacters(in: .whitespacesAndNewlines)
let text = try await speech.finalText()
guard !text.isEmpty else { throw Failure.empty }
return (text, locale.identifier)
return (text, language)
}
}
+57 -1
View File
@@ -52,6 +52,7 @@ final class KeyboardViewController: UIInputViewController {
override func viewDidAppear(_ animated: Bool) {
super.viewDidAppear(animated)
warmFullKeyboardLexiconIfNeeded()
listDictationLanguagesIfNeeded()
}
override func viewWillDisappear(_ animated: Bool) {
@@ -212,6 +213,8 @@ final class KeyboardViewController: UIInputViewController {
TypingLogField("learnHowIType", .bool(TypingJournalSettings().isEnabled)),
TypingLogField("cleanupTagger", .bool(CleanupTaggerSettings().isEnabled)),
TypingLogField("hexKeyPlacement", .string(HexKeyPlacement.current.rawValue)),
TypingLogField("keyboardLayout", .string(fullKeyboard?.engine.language.rawValue ?? "")),
TypingLogField("dictationLanguage", .string(LanguageSettings().dictationLanguage ?? "automatic")),
TypingLogField("style", .string(bridge.style.rawValue)),
]
)
@@ -299,7 +302,10 @@ final class KeyboardViewController: UIInputViewController {
self.localDictation.stop()
// Loading the cleanup model while recording starts ran older iPhones out of
// keyboard memory, so it loads after the microphone closes.
CleanupTagger.prepareIfEnabled(style: self.bridge.style)
CleanupTagger.prepareIfEnabled(
style: self.bridge.style,
language: LanguageSettings().effectiveDictationLanguage
)
} else {
self.stopWarmDictation()
}
@@ -322,6 +328,11 @@ final class KeyboardViewController: UIInputViewController {
}
self.refreshColdStartURL(now: Date().timeIntervalSince1970, force: true)
}
viewModel.onLanguageSelected = { [weak self] language in
LanguageSettings().chooseDictationLanguage(language)
self?.languageSignature = nil
self?.syncLanguageState()
}
viewModel.onReturnAppSelected = { [weak self] appID in
guard let self else { return }
self.bridge.returnAppID = appID
@@ -535,6 +546,7 @@ final class KeyboardViewController: UIInputViewController {
if viewModel.quickReturnApps != quickReturnApps {
viewModel.quickReturnApps = quickReturnApps
}
syncLanguageState()
viewModel.applyBridgeState(
hasFullAccess: true,
@@ -551,6 +563,50 @@ final class KeyboardViewController: UIInputViewController {
schedulePoll(force: false)
}
// MARK: - Dictation language
/// What `syncLanguageState` last read, so the 25 Hz poll only works out the choices on a change.
private var languageSignature: String?
/// Apple Speech's languages are listed once per keyboard process.
private static var didListDictationLanguages = false
/// Settings › Dictation language, what Automatic picks, and the switch's choices, from
/// the App Group (iOS 26, where dictation is Apple Speech).
private func syncLanguageState() {
guard KeyboardLocalDictation.isSupported else { return }
let settings = LanguageSettings()
let explicit = settings.dictationLanguage
let automatic = settings.automaticDictationLanguage
let recent = settings.recentDictationLanguages
let supported = settings.supportedDictationLanguages
let signature = [explicit ?? "", automatic ?? "", recent.joined(separator: ","), "\(supported.count)"]
.joined(separator: "|")
guard signature != languageSignature else { return }
languageSignature = signature
let choices = DictationLanguageSwitch.choices(
current: explicit,
recent: recent,
preferred: Locale.preferredLanguages,
supported: supported
)
if viewModel.dictationLanguage != explicit { viewModel.dictationLanguage = explicit }
if viewModel.automaticLanguage != automatic { viewModel.automaticLanguage = automatic }
if viewModel.languageChoices != choices { viewModel.languageChoices = choices }
}
/// Asks Apple Speech which languages it has and what Automatic picks, so the switch can
/// offer them before Hex has listed them.
private func listDictationLanguagesIfNeeded() {
guard #available(iOS 26.0, *), hasFullAccess, !Self.didListDictationLanguages else { return }
Self.didListDictationLanguages = true
Task { [weak self] in
_ = await AppleSpeechLanguages.options()
_ = await AppleSpeechLanguages.automatic()
self?.languageSignature = nil
self?.syncLanguageState()
}
}
private func refreshColdStartURL(now: TimeInterval, force: Bool) {
let bucket = Int(now / 2.0)
guard force || bucket != lastURLRefreshBucket else { return }
+46 -56
View File
@@ -100,73 +100,69 @@ actor AppleSpeechTranscriber: LocalTranscribing {
case .emptyTranscript:
"Apple Speech did not detect any speech."
case .unavailable:
"Apple Speech is not available for your preferred languages on this iPhone."
"Apple Speech can't transcribe this language on this iPhone."
}
}
}
nonisolated let displayName = "Apple Speech"
private let requestedLocale: Locale
private let language: SpeechLanguage
private var preparedAnalyzer: SpeechAnalyzer?
private var preparedTranscriber: SpeechTranscriber?
private var locale: Locale?
private var preparedModule: AppleSpeechModule?
private var isPrepared = false
private var reservedLocale: Locale?
var languageName: String {
Self.displayName(for: locale ?? requestedLocale)
Self.displayName(for: language.locale)
}
init(locale: Locale) {
requestedLocale = locale
var languageIdentifier: String {
language.identifier
}
nonisolated static var isAvailable: Bool {
SpeechTranscriber.isAvailable
/// `requestsAuthorization` is false only in tests, which can't answer the Speech
/// Recognition prompt; `SpeechAnalyzer` itself runs without it, as in-keyboard dictation does.
private let requestsAuthorization: Bool
init(language: SpeechLanguage, requestsAuthorization: Bool = true) {
self.language = language
self.requestsAuthorization = requestsAuthorization
}
static func preferredLocale() async -> Locale? {
var identifiers = Locale.preferredLanguages
identifiers.append(Locale.current.identifier)
identifiers.append("en-US")
/// Settings › Dictation language, else the first preferred language either module has.
static func preferredLanguage() async -> SpeechLanguage? {
await AppleSpeechLanguages.resolve()
}
var checked = Set<String>()
for identifier in identifiers {
let candidate = Locale(identifier: identifier)
let normalized = candidate.identifier(.bcp47)
guard checked.insert(normalized).inserted else { continue }
if let supported = await SpeechTranscriber.supportedLocale(
equivalentTo: candidate
) {
return supported
}
}
return nil
/// Whether Hex dictates this language with Apple Speech rather than Parakeet. English
/// without `SpeechTranscriber` (an iPhone that lacks it) stays on Parakeet, as before;
/// every other language `DictationTranscriber` has beats English-only Parakeet.
nonisolated static func shouldUse(_ language: SpeechLanguage) -> Bool {
language.module == .speech || !SpeechLanguagePicker.isEnglish(language.identifier)
}
func prepare(
progressHandler: @escaping @Sendable (ModelPreparationProgress) -> Void
) async throws {
guard locale == nil else { return }
guard await Self.requestAuthorization() == .authorized else {
throw TranscriberError.authorizationDenied
}
guard let locale = await SpeechTranscriber.supportedLocale(
equivalentTo: requestedLocale
) else {
throw TranscriberError.unavailable
guard !isPrepared else { return }
if requestsAuthorization {
guard await Self.requestAuthorization() == .authorized else {
throw TranscriberError.authorizationDenied
}
}
let locale = language.locale
try await AppleSpeechAssetReservations.shared.acquire(locale)
reservedLocale = locale
do {
let transcriber = SpeechTranscriber(locale: locale, preset: .transcription)
let status = await AssetInventory.status(forModules: [transcriber])
let speech = AppleSpeechModule(language)
let status = await AssetInventory.status(forModules: [speech.module])
if status != .installed,
let request = try await AssetInventory.assetInstallationRequest(
supporting: [transcriber]
supporting: [speech.module]
) {
let progress = request.progress
let monitor = Task {
@@ -183,7 +179,7 @@ actor AppleSpeechTranscriber: LocalTranscribing {
defer { monitor.cancel() }
try await request.downloadAndInstall()
}
guard await AssetInventory.status(forModules: [transcriber]) == .installed else {
guard await AssetInventory.status(forModules: [speech.module]) == .installed else {
throw TranscriberError.unavailable
}
progressHandler(
@@ -193,11 +189,11 @@ actor AppleSpeechTranscriber: LocalTranscribing {
)
)
let analyzer = makeAnalyzer(transcriber: transcriber)
let analyzer = makeAnalyzer(module: speech)
try await analyzer.prepareToAnalyze(in: nil)
self.locale = locale
preparedTranscriber = transcriber
isPrepared = true
preparedModule = speech
preparedAnalyzer = analyzer
progressHandler(
ModelPreparationProgress(
@@ -216,20 +212,20 @@ actor AppleSpeechTranscriber: LocalTranscribing {
}
func transcribe(_ audioURL: URL, vocabulary: [String]) async throws -> String {
guard let locale else {
guard isPrepared else {
throw TranscriberError.unavailable
}
let transcriber: SpeechTranscriber
let speech: AppleSpeechModule
let analyzer: SpeechAnalyzer
if let preparedTranscriber, let preparedAnalyzer {
transcriber = preparedTranscriber
if let preparedModule, let preparedAnalyzer {
speech = preparedModule
analyzer = preparedAnalyzer
self.preparedTranscriber = nil
self.preparedModule = nil
self.preparedAnalyzer = nil
} else {
transcriber = SpeechTranscriber(locale: locale, preset: .transcription)
analyzer = makeAnalyzer(transcriber: transcriber)
speech = AppleSpeechModule(language)
analyzer = makeAnalyzer(module: speech)
}
let audioFile = try AVAudioFile(forReading: audioURL)
@@ -244,22 +240,16 @@ actor AppleSpeechTranscriber: LocalTranscribing {
try await analyzer.start(inputAudioFile: audioFile, finishAfterFile: true)
var transcript = AttributedString()
for try await result in transcriber.results where result.isFinal {
transcript += result.text
}
let text = String(transcript.characters)
.trimmingCharacters(in: .whitespacesAndNewlines)
let text = try await speech.finalText()
guard !text.isEmpty else {
throw TranscriberError.emptyTranscript
}
return text
}
private func makeAnalyzer(transcriber: SpeechTranscriber) -> SpeechAnalyzer {
private func makeAnalyzer(module: AppleSpeechModule) -> SpeechAnalyzer {
SpeechAnalyzer(
modules: [transcriber],
modules: [module.module],
options: SpeechAnalyzer.Options(
priority: .userInitiated,
modelRetention: .lingering
+2
View File
@@ -457,6 +457,8 @@ struct ContentView: View {
.padding(.horizontal, 16)
.frame(minHeight: 48)
DictationLanguageRow(dictation: dictation)
rowDivider
HStack(spacing: 12) {
+86 -12
View File
@@ -57,6 +57,8 @@ final class DictationController {
/// Whether the keyboard gets the raw transcript to learn from.
let sharesHeard: Bool
let copyToPasteboard: Bool
/// The transcriber's language (BCP 47) when the capture started.
let language: String
}
private struct PendingKeyboardHandoff {
@@ -144,6 +146,13 @@ final class DictationController {
private(set) var handoffNotice: String?
private(set) var engineName = "Local speech"
private(set) var transcriptionLanguage = "English"
/// The BCP 47 identifier of `transcriptionLanguage`, for the steps that depend on it.
private(set) var transcriptionLanguageID = "en-US"
/// Settings › Dictation language: a BCP 47 identifier, or nil for Automatic.
private(set) var dictationLanguage: String?
/// A language change made while busy, applied once the dictation in flight is done.
private var pendingLanguagePreparation = false
private let languageSettings: LanguageSettings
private(set) var modelPreparation = ModelPreparationProgress(
fractionCompleted: 0,
detail: "Checking model files"
@@ -162,7 +171,8 @@ final class DictationController {
cleanupSettings: AppleIntelligenceCleanupSettings = AppleIntelligenceCleanupSettings(),
customVocabularySettings: CustomVocabularySettings = CustomVocabularySettings(),
typingJournalSettings: TypingJournalSettings = TypingJournalSettings(),
learnedDictationCorrections: (@MainActor () -> [DictationCorrection])? = nil
learnedDictationCorrections: (@MainActor () -> [DictationCorrection])? = nil,
languageSettings: LanguageSettings = LanguageSettings()
) {
let resolvedBridge = keyboardBridge ?? KeyboardBridge()
let resolvedAudio = audio ?? SystemAudioManager()
@@ -181,6 +191,8 @@ final class DictationController {
}
self.fileManager = fileManager
self.now = now
self.languageSettings = languageSettings
self.dictationLanguage = languageSettings.dictationLanguage
self.warmDuration = resolvedBridge.warmDuration
self.style = resolvedBridge.style
self.returnAppID = resolvedBridge.returnAppID
@@ -205,10 +217,11 @@ final class DictationController {
if hasCachedModel {
engineName = "Parakeet V2"
prepareModel()
} else if #available(iOS 26.0, *), AppleSpeechTranscriber.isAvailable {
} else if #available(iOS 26.0, *) {
Task { [weak self] in
guard let self else { return }
if await AppleSpeechTranscriber.preferredLocale() != nil {
if let language = await AppleSpeechTranscriber.preferredLanguage(),
AppleSpeechTranscriber.shouldUse(language) {
self.engineName = "Apple Speech"
self.prepareModel()
} else {
@@ -293,6 +306,44 @@ final class DictationController {
}
}
/// Settings › Dictation language. Prepares the new language's model now, or once the
/// dictation in flight is done.
func setDictationLanguage(_ identifier: String?) {
let normalized = (identifier?.isEmpty == true) ? nil : identifier
guard normalized != dictationLanguage else { return }
languageSettings.chooseDictationLanguage(normalized)
applyDictationLanguage(normalized)
}
/// Picks up a language the keyboard switched to in the App Group.
func followDictationLanguageSetting() {
let stored = languageSettings.dictationLanguage
guard stored != dictationLanguage else { return }
applyDictationLanguage(stored)
}
private func applyDictationLanguage(_ identifier: String?) {
dictationLanguage = identifier
switch phase {
case .recording, .transcribing, .requestingPermission:
pendingLanguagePreparation = true
case .modelRequired, .preparingModel, .ready, .failed:
prepareModel()
}
}
/// Runs a language change that waited for the dictation in flight.
private func prepareDeferredLanguageIfNeeded() {
guard pendingLanguagePreparation else { return }
switch phase {
case .ready, .failed, .modelRequired:
pendingLanguagePreparation = false
prepareModel()
case .recording, .transcribing, .requestingPermission, .preparingModel:
break
}
}
func setAllowBluetoothMic(_ allow: Bool) {
guard allowBluetoothMic != allow else { return }
allowBluetoothMic = allow
@@ -342,6 +393,7 @@ final class DictationController {
}
restoreAudioAfterCapture()
publishKeyboardSnapshot()
prepareDeferredLanguageIfNeeded()
}
func clearTranscript() {
@@ -418,6 +470,8 @@ final class DictationController {
if case .setup = request {
return
}
// A language picked in the keyboard: prepare it before the handoff records.
followDictationLanguageSetting()
if let customOpener = urlOpener {
defaultURLOpener = customOpener
}
@@ -677,6 +731,7 @@ final class DictationController {
guard self.modelPrepGeneration == prepGen else { return }
self.transcriber = candidate
self.transcriptionLanguage = await candidate.languageName
self.transcriptionLanguageID = await candidate.languageIdentifier
self.keyboardBridge.setModelInstalled(true)
self.isModelReady = true
self.phase = .ready
@@ -696,9 +751,9 @@ final class DictationController {
private static func defaultTranscriber() async -> any LocalTranscribing {
if #available(iOS 26.0, *),
AppleSpeechTranscriber.isAvailable,
let locale = await AppleSpeechTranscriber.preferredLocale() {
return AppleSpeechTranscriber(locale: locale)
let language = await AppleSpeechTranscriber.preferredLanguage(),
AppleSpeechTranscriber.shouldUse(language) {
return AppleSpeechTranscriber(language: language)
}
return ParakeetTranscriber()
}
@@ -728,7 +783,8 @@ final class DictationController {
transcriptionTask = nil
let effectiveStyle = styleOverride ?? style
let usesCleanup = cleanupSettings.isEnabled
CleanupTagger.prepareIfEnabled(style: effectiveStyle)
let language = transcriptionLanguageID
CleanupTagger.prepareIfEnabled(style: effectiveStyle, language: language)
let vocabularyTerms = customVocabularySettings.activeTerms
let personalCorrections = learnedDictationCorrections()
let recognitionHints = DictationCorrections.recognitionHints(
@@ -780,7 +836,8 @@ final class DictationController {
personalCorrections: personalCorrections,
recognitionHints: recognitionHints,
sharesHeard: sharesHeard,
copyToPasteboard: copyToPasteboard
copyToPasteboard: copyToPasteboard,
language: language
)
refreshWarmExpiry()
audioLevel = 0
@@ -828,6 +885,7 @@ final class DictationController {
liveActivity.endImmediately()
restoreAudioAfterCapture()
publishKeyboardSnapshot()
prepareDeferredLanguageIfNeeded()
return
}
@@ -859,16 +917,23 @@ final class DictationController {
// sentence start ("iPhone" -> "IPhone"); learned capitals get the same.
let corrected = DictationCorrections.apply(session.personalCorrections, to: rawText)
let spelled = VocabularySpelling.apply(session.vocabularyTerms, to: corrected)
let tagged = await CleanupTagger.cleanIfEnabled(spelled, style: session.style)
let tagged = await CleanupTagger.cleanIfEnabled(spelled, style: session.style, language: session.language)
let cleaned = await self.cleanedTranscriptIfEnabled(tagged, session: session)
let translated = await DictationTranslator.translateIfEnabled(cleaned)
let translated = await DictationTranslator.translateIfEnabled(
cleaned,
source: Locale(identifier: session.language).language
)
let text = translated ?? cleaned
guard self.jobGeneration == captureGeneration, !Task.isCancelled else {
return
}
// A translation is formatted in the language it is now in.
let formatLanguage = translated == nil
? session.language
: DictationTranslationSettings().targetLanguage
let formatted = VocabularySpelling.apply(
session.vocabularyTerms + DictationCorrections.spellings(session.personalCorrections),
to: TranscriptFormatter.format(text, style: session.style)
to: TranscriptFormatter.format(text, style: session.style, language: formatLanguage)
)
self.recordingDuration = 0
self.transcript = formatted
@@ -908,6 +973,7 @@ final class DictationController {
self.refreshWarmExpiry()
self.phase = .ready
self.publishKeyboardSnapshot()
self.prepareDeferredLanguageIfNeeded()
} catch {
guard !Task.isCancelled, let self, self.jobGeneration == captureGeneration else {
return
@@ -918,6 +984,7 @@ final class DictationController {
self.keyboardMessage = keyboardJobID == nil ? nil : error.localizedDescription
self.phase = .failed(error.localizedDescription)
self.publishKeyboardSnapshot()
self.prepareDeferredLanguageIfNeeded()
}
}
}
@@ -931,7 +998,8 @@ final class DictationController {
session.style != .verbatim,
!rawText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty,
let cleaner,
cleaner.isAvailable else {
cleaner.isAvailable,
cleaner.supports(language: session.language) else {
return rawText
}
return await cleaner.clean(rawText) ?? rawText
@@ -1013,6 +1081,8 @@ final class DictationController {
// Never expire an in-flight recording or accepted transcription mid-work.
if self.phase != .recording && self.phase != .transcribing {
// The keyboard's language switch writes the App Group; follow it.
self.followDictationLanguageSetting()
let remaining = self.keyboardSessionExpiresAt - self.now()
if remaining <= 0 {
self.endWarmSession()
@@ -1264,8 +1334,11 @@ final class DictationController {
}
return transcript
}
followDictationLanguageSetting()
if phase == .modelRequired || isFailedPhase {
prepareModel()
}
if phase == .preparingModel {
try? await waitUntilReady(timeoutSeconds: 10)
}
guard phase == .ready else { return nil }
@@ -1318,6 +1391,7 @@ final class DictationController {
let scene = args[flagIndex + 1]
controller.engineName = "Apple Speech"
controller.transcriptionLanguage = "English"
controller.transcriptionLanguageID = "en-US"
controller.keyboardBridge.setModelInstalled(true)
controller.isModelReady = true
+2
View File
@@ -103,6 +103,8 @@ struct ExperimentsCard: View {
.onChange(of: haptics) { _, enabled in
FullKeyboardSettings().haptics = enabled
}
KeyboardLayoutRow()
}
Divider()
+61 -2
View File
@@ -4,9 +4,31 @@ import UIKit
/// Renders the experimental `FullKeyboardView` under a composer when
/// `--snap-state full-keyboard-<state>[-dark|-dark-field]` is passed on Simulator.
/// `-dark` is a dark app; `-dark-field` is a light app whose field asks for a dark keyboard.
/// The layout follows `--dictation-language` (Automatic, English, without it), or
/// `--keyboard-layout nb|es|de|en` pins it; drafts are in the layout's language.
/// `--accent-key ø` picks the key the `accents` state holds.
struct FullKeyboardSnapshotStage: View {
let scene: String
static var layout: KeyboardLanguage {
argument(after: "--keyboard-layout").flatMap(KeyboardLanguage.init(rawValue:))
?? KeyboardLanguage.automatic(
dictationLanguage: DictationLanguageFixture.explicit ?? "en-US",
inputModes: [],
preferredLanguages: []
)
}
static var accentKey: String {
argument(after: "--accent-key") ?? "e"
}
private static func argument(after flag: String) -> String? {
let args = ProcessInfo.processInfo.arguments
guard let idx = args.firstIndex(of: flag), idx + 1 < args.count else { return nil }
return args[idx + 1]
}
static var requestedScene: String? {
let args = ProcessInfo.processInfo.arguments
guard let idx = args.firstIndex(of: "--snap-state"), idx + 1 < args.count else {
@@ -33,7 +55,38 @@ struct FullKeyboardSnapshotStage: View {
}
private var draft: String {
switch state {
switch Self.layout {
case .norwegian:
return switch state {
case "inserted": "Kommer ti min for sent, er hjemme 19.30. Begynn uten meg."
case "numbers": "Kommer ti min for sent, hjemme 19.3"
case "shift": "Kommer ti min for sent. "
case "typing": "Kommer ti min for sen"
case "accents": Self.accentKey == "e" ? "Kaf" : "Bl"
default: "Kommer ti min for sent, "
}
case .spanish:
return switch state {
case "inserted": "Llego diez minutos tarde, estaré en casa a las 7:30."
case "numbers": "Llego diez minutos tarde, en casa a las 7:3"
case "shift": "Llego diez minutos tarde. "
case "typing": "Llego diez minutos tard"
case "accents": "Caf"
default: "Llego diez minutos tarde, "
}
case .german:
return switch state {
case "inserted": "Bin zehn Minuten zu spät, um 7:30 zu Hause. Fangt ohne mich an."
case "numbers": "Bin zehn Minuten zu spät, um 7:3"
case "shift": "Bin zehn Minuten zu spät. "
case "typing": "Bin zehn Minuten zu spä"
case "accents": Self.accentKey == "s" ? "Stra" : "Caf"
default: "Bin zehn Minuten zu spät, "
}
case .english:
break
}
return switch state {
case "inserted": "Running 10 min late, should be home by 7:30. Start without me."
case "numbers": "Running 10 min late, home by 7:3"
case "shift": "Running 10 min late. "
@@ -123,7 +176,9 @@ private struct FullKeyboardFixture: UIViewRepresentable {
model.configureFixture(snapshot: warm, now: now)
}
DictationLanguageFixture.apply(to: model)
let view = FullKeyboardView(model: model, target: context.coordinator.target)
view.languageOverride = FullKeyboardSnapshotStage.layout
view.appearanceOverride = darkKeyboard
view.orientationOverride = .portrait
view.drawsBackdrop = true
@@ -135,7 +190,9 @@ private struct FullKeyboardFixture: UIViewRepresentable {
case "pressed":
view.debugPress("g")
case "accents":
view.debugHoldAccent("e", selectIndex: 2)
let key = FullKeyboardSnapshotStage.accentKey
let pick = ["e": "é", "s": "ß", "ø": "ö", "æ": "ä", "a": "á", "o": "ó", "n": "ñ", "u": "ú"][key]
view.debugHoldAccent(key, selecting: pick ?? key)
case "numbers":
view.engine.touchBegan(id: 900, at: center(of: .layerSwitch(.numbers), in: view))
view.engine.touchEnded(id: 900, at: center(of: .layerSwitch(.numbers), in: view))
@@ -154,6 +211,8 @@ private struct FullKeyboardFixture: UIViewRepresentable {
view.didInsertDictation("should be home by 7:30. Start without me.")
case "style":
view.debugShowStyleMenu()
case "languages":
view.debugShowStyleMenu(page: .languages)
case "morph":
// Loops idle → recording → idle so the strip morph can be screen-recorded.
func cycle(_ start: TimeInterval) {
+32
View File
@@ -66,6 +66,7 @@ struct KeyboardSnapshotStage: View {
private func configureScene() {
let now: TimeInterval = 10_000
viewModel.needsGlobeKey = true
DictationLanguageFixture.apply(to: viewModel)
switch scene {
case "keyboard-cold":
@@ -132,12 +133,43 @@ struct KeyboardSnapshotStage: View {
activeTray: .style
)
case "keyboard-language":
viewModel.configureFixture(
snapshot: KeyboardSnapshot(
state: .ready,
heartbeat: now - 0.1,
expiresAt: now + 14 * 60
),
now: now,
style: .polished,
activeTray: .language
)
default:
break
}
}
}
/// The dictation languages simulator scenes show: `--dictation-language nb-NO` picks one,
/// otherwise Automatic, which resolves to English.
enum DictationLanguageFixture {
static let choices = ["nb-NO", "en-US", "es-ES", "de-DE", "he-IL"]
static var explicit: String? {
let args = ProcessInfo.processInfo.arguments
guard let idx = args.firstIndex(of: "--dictation-language"), idx + 1 < args.count else { return nil }
return args[idx + 1]
}
@MainActor
static func apply(to model: KeyboardViewModel) {
model.languageChoices = choices
model.dictationLanguage = explicit
model.automaticLanguage = "en-US"
}
}
/// Hosts a clean conversation thread and an auto-focused real `UITextField`
/// so iOS Simulator docks the actual `HEXKeyboard.appex` `UIInputViewController`
/// extension process at the bottom of the screen.
+8 -1
View File
@@ -177,7 +177,14 @@ struct KeyboardSetupCard: View {
private var modelStatus: String {
if dictation.isModelReady {
return dictation.engineName
// "Norwegian Bokmål · Apple": the language it really transcribes, short enough
// for the row.
let code = Locale(identifier: dictation.transcriptionLanguageID).language.languageCode?.identifier ?? "en"
let language = Locale.current.localizedString(forLanguageCode: code).map {
$0.prefix(1).uppercased() + $0.dropFirst()
} ?? dictation.transcriptionLanguage
let engine = dictation.engineName == "Apple Speech" ? "Apple" : dictation.engineName
return "\(language) · \(engine)"
}
switch dictation.phase {
case .preparingModel:
+164
View File
@@ -0,0 +1,164 @@
import SwiftUI
import UIKit
/// Settings › Dictation language: Automatic, the first iPhone language Apple Speech
/// transcribes, or any language `SpeechTranscriber` or `DictationTranscriber` has. Shown
/// on iOS 26, where Hex dictates with Apple Speech; before it, Parakeet is English only.
struct DictationLanguageRow: View {
@Bindable var dictation: DictationController
@State private var options: [SpeechLanguageOption] = []
@State private var automaticName: String?
var body: some View {
if #available(iOS 26.0, *) {
VStack(spacing: 0) {
LanguageRowDivider()
HStack(spacing: 12) {
VStack(alignment: .leading, spacing: 2) {
Text("Dictation language")
.font(.system(size: 15, weight: .regular))
.foregroundStyle(HexTheme.textBase)
if dictation.dictationLanguage == nil, let automaticName {
Text(automaticName)
.font(.system(size: 12))
.foregroundStyle(HexTheme.textMuted)
.lineLimit(1)
}
}
Spacer(minLength: 8)
Menu {
Button {
dictation.setDictationLanguage(nil)
} label: {
if dictation.dictationLanguage == nil {
Label("Automatic", systemImage: "checkmark")
} else {
Text("Automatic")
}
}
Divider()
ForEach(options) { option in
Button {
dictation.setDictationLanguage(option.id)
} label: {
if dictation.dictationLanguage == option.id {
Label(option.name, systemImage: "checkmark")
} else {
Text(option.name)
}
}
}
} label: {
LanguageMenuValue(value: selectedName)
}
}
.padding(.horizontal, 16)
.frame(minHeight: 48)
}
.task {
// The keyboard may have switched the language while Hex was away.
dictation.followDictationLanguageSetting()
options = await AppleSpeechLanguages.options()
automaticName = await AppleSpeechLanguages.automatic().map {
SpeechLanguagePicker.displayName(for: $0.identifier)
}
}
}
}
private var selectedName: String {
guard let id = dictation.dictationLanguage else { return "Automatic" }
return SpeechLanguagePicker.shortNames(for: options.map(\.id))[id]
?? SpeechLanguagePicker.displayName(for: id)
}
}
/// Experiments › Full keyboard › Keyboard layout: Automatic (the dictation language's
/// layout), English, Norsk, Español, or Deutsch.
struct KeyboardLayoutRow: View {
@State private var setting = LanguageSettings().keyboardLayout
private var automatic: KeyboardLanguage {
KeyboardLanguage.automatic(
dictationLanguage: LanguageSettings().effectiveDictationLanguage,
inputModes: UITextInputMode.activeInputModes.compactMap(\.primaryLanguage),
preferredLanguages: Locale.preferredLanguages
)
}
var body: some View {
VStack(spacing: 0) {
Divider()
.overlay(HexTheme.borderWeak)
.padding(.leading, 16)
HStack(spacing: 12) {
VStack(alignment: .leading, spacing: 2) {
Text("Keyboard layout")
.font(.system(size: 15, weight: .regular))
.foregroundStyle(HexTheme.textBase)
if setting == nil {
Text("\(automatic.displayName), like your dictation language")
.font(.system(size: 12))
.foregroundStyle(HexTheme.textMuted)
}
}
Spacer(minLength: 8)
Menu {
choice(nil, title: "Automatic")
Divider()
ForEach(KeyboardLanguage.allCases, id: \.self) { language in
choice(language, title: language.displayName)
}
} label: {
LanguageMenuValue(value: setting?.displayName ?? "Automatic")
}
}
.padding(.horizontal, 16)
.padding(.vertical, 10)
}
.onAppear { setting = LanguageSettings().keyboardLayout }
}
private func choice(_ language: KeyboardLanguage?, title: String) -> some View {
Button {
setting = language
LanguageSettings().keyboardLayout = language
} label: {
if setting == language {
Label(title, systemImage: "checkmark")
} else {
Text(title)
}
}
}
}
/// The settings card's row divider.
private struct LanguageRowDivider: View {
var body: some View {
Rectangle()
.fill(HexTheme.borderWeak)
.frame(height: 0.5)
.padding(.leading, 16)
}
}
/// A menu's current value with the up-down chevron, as the settings card's other menus show it.
private struct LanguageMenuValue: View {
let value: String
var body: some View {
HStack(spacing: 5) {
Text(value)
.font(.system(size: 15, weight: .regular))
.foregroundStyle(HexTheme.textMuted)
.lineLimit(1)
Image(systemName: "chevron.up.chevron.down")
.font(.system(size: 11, weight: .medium))
.foregroundStyle(HexTheme.textMuted)
}
.padding(.vertical, 6)
.contentShape(Rectangle())
}
}
+5
View File
@@ -9,6 +9,8 @@ struct ModelPreparationProgress: Sendable {
protocol LocalTranscribing: Actor {
nonisolated var displayName: String { get }
var languageName: String { get }
/// The BCP 47 identifier of the language it transcribes.
var languageIdentifier: String { get }
func prepare(
progressHandler: @escaping @Sendable (ModelPreparationProgress) -> Void
@@ -19,6 +21,9 @@ protocol LocalTranscribing: Actor {
}
extension LocalTranscribing {
/// English unless the engine says otherwise.
var languageIdentifier: String { "en-US" }
/// Engines without a recognition hint ignore the list and transcribe exactly as before.
nonisolated func transcribe(_ audioURL: URL, vocabulary: [String]) async throws -> String {
try await transcribe(audioURL)
+17
View File
@@ -163,6 +163,12 @@ nonisolated enum TranscriptCleanupGuard {
protocol TranscriptCleaning: AnyObject {
var isAvailable: Bool { get }
func clean(_ text: String) async -> String?
/// Whether it can clean a dictation in this language (BCP 47).
func supports(language: String) -> Bool
}
extension TranscriptCleaning {
func supports(language: String) -> Bool { true }
}
/// One conservative on-device pass over a raw transcript.
@@ -185,6 +191,17 @@ final class AppleIntelligenceCleaner: TranscriptCleaning {
AppleIntelligenceCleanupAvailability.current.isAvailable
}
/// The on-device model cleans only languages it speaks; a Norwegian dictation on an
/// iPhone whose Apple Intelligence lacks Norwegian is left as heard.
func supports(language: String) -> Bool {
#if canImport(FoundationModels)
guard #available(iOS 26.0, *) else { return false }
return SystemLanguageModel.default.supportsLocale(Locale(identifier: language))
#else
return false
#endif
}
func clean(_ text: String) async -> String? {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty, isAvailable else { return nil }
+5 -1
View File
@@ -10,7 +10,11 @@ struct TranslateDictationRow: View {
@State private var status: LanguageAvailability.Status?
@State private var download: TranslationSession.Configuration?
private var source: Locale.Language { Locale.current.language }
/// What you dictate in: Settings › Dictation language, or the language it last resolved.
private var source: Locale.Language {
LanguageSettings().effectiveDictationLanguage.map { Locale(identifier: $0).language }
?? Locale.current.language
}
var body: some View {
VStack(alignment: .leading, spacing: 8) {
@@ -178,6 +178,39 @@ final class AutocorrectEngineTests: XCTestCase {
XCTAssertNil(AutocorrectEngine.singleEditKeyboardCost(from: "abcdef", to: "abxyef"), "two substitutions are rejected")
}
func testALayoutWithoutADictionaryNeverCorrectsIntoEnglish() {
engine.autocorrect.loadLexiconEntries([("omw", "On my way!")])
engine.autocorrect.setLanguage(.norwegian, available: ["en_US", "en"])
XCTAssertFalse(engine.autocorrect.hasDictionary)
typeWord("teh")
XCTAssertTrue(engine.suggestions.isEmpty, "no English suggestions for Norwegian")
tap(.space)
XCTAssertEqual(target.text, "teh ")
typeWord("omw")
tap(.space)
XCTAssertEqual(target.text, "teh On my way! ", "your own text replacements still apply")
}
func testEnglishFixesStayOffInOtherLanguages() {
engine.autocorrect.setLanguage(.norwegian, available: ["nb", "en_US"])
XCTAssertTrue(engine.autocorrect.hasDictionary)
typeWord("i")
tap(.space)
XCTAssertEqual(target.text, "i ", "Norwegian \"i\" means \"in\"")
engine.autocorrect.setLanguage(.english, available: ["en_US"])
typeWord("i")
tap(.space)
XCTAssertEqual(target.text, "i I ")
}
func testAdjacencyFollowsTheLayout() {
// German swaps Y and Z: on QWERTZ, Z sits beside T.
XCTAssertTrue(AutocorrectEngine.areAdjacentKeys("z", "t", language: .german))
XCTAssertFalse(AutocorrectEngine.areAdjacentKeys("z", "t", language: .english))
XCTAssertTrue(AutocorrectEngine.areAdjacentKeys("ø", "l", language: .norwegian))
XCTAssertTrue(AutocorrectEngine.areAdjacentKeys("ñ", "l", language: .spanish))
}
func testTurningAutocorrectSettingOffKeepsSuggestionsWithoutAutoReplacing() {
engine.autocorrect.isEnabled = false
typeWord("dont")
@@ -228,6 +228,27 @@ final class CleanupTaggerTests: XCTestCase {
XCTAssertEqual(result, "I mean what I say.", "a sentence with nothing to remove comes back as said")
}
func testTheEnglishTaggerLeavesOtherLanguagesAsHeard() async throws {
let suite = "hex.tests.\(UUID().uuidString)"
let defaults = try XCTUnwrap(UserDefaults(suiteName: suite))
defer { defaults.removePersistentDomain(forName: suite) }
let settings = CleanupTaggerSettings(defaults: defaults)
settings.isEnabled = true
let tagger = CleanupTagger()
let norwegian = "Vi sender den tirsdag, nei vent, torsdag."
CleanupTagger.prepareIfEnabled(style: .polished, language: "nb-NO", settings: settings, tagger: tagger)
XCTAssertFalse(tagger.isLoaded, "a Norwegian dictation loads nothing")
let result = await CleanupTagger.cleanIfEnabled(norwegian, style: .polished, language: "nb-NO", settings: settings, tagger: tagger)
XCTAssertEqual(result, norwegian)
XCTAssertFalse(tagger.isLoaded)
let english = await CleanupTagger.cleanIfEnabled(
"Let's ship it Tuesday, no wait, Thursday.", style: .polished, language: "en-GB", settings: settings, tagger: tagger
)
XCTAssertEqual(english, "Let's ship it Thursday.")
}
func testDictationStartPreparesOnlyWhenOnAndNotForVerbatim() throws {
let suite = "hex.tests.\(UUID().uuidString)"
let defaults = try XCTUnwrap(UserDefaults(suiteName: suite))
@@ -803,9 +803,198 @@ final class DictationControllerTests: XCTestCase {
}
}
func testDictationLanguageChangeWaitsForTheDictationInFlight() async throws {
let audio = FakeAudioManager()
let transcriber = ControlledTranscriber()
let languages = LanguageSettings(defaults: defaults)
var preparations = 0
let controller = DictationController(
audio: audio,
keyboardBridge: KeyboardBridge(defaults: defaults),
transcriber: transcriber,
transcriberFactory: {
preparations += 1
return transcriber
},
autoPrepareOnInit: false,
languageSettings: languages
)
controller.prepareModel()
try await settle()
XCTAssertEqual(preparations, 1)
XCTAssertNil(controller.dictationLanguage, "Automatic by default")
controller.setDictationLanguage("nb-NO")
try await settle()
XCTAssertEqual(languages.dictationLanguage, "nb-NO", "saved in the App Group for the keyboard")
XCTAssertEqual(preparations, 2, "an idle change prepares the new language at once")
XCTAssertEqual(controller.phase, .ready)
controller.startRecording(keyboardJobID: "language-job")
try await settle()
XCTAssertEqual(controller.phase, .recording)
controller.setDictationLanguage("de-DE")
try await settle()
XCTAssertEqual(preparations, 2, "never while recording")
XCTAssertEqual(controller.phase, .recording)
controller.finishRecording()
try await settle()
XCTAssertEqual(controller.transcript, "Ship the build.")
XCTAssertEqual(preparations, 3, "prepared once the dictation is done")
XCTAssertEqual(controller.phase, .ready)
controller.setDictationLanguage(nil)
try await settle()
XCTAssertNil(languages.dictationLanguage)
XCTAssertEqual(preparations, 4)
// The keyboard's switch writes the App Group; Hex follows without writing it back.
languages.dictationLanguage = "he-IL"
controller.followDictationLanguageSetting()
try await settle()
XCTAssertEqual(controller.dictationLanguage, "he-IL")
XCTAssertEqual(preparations, 5)
controller.followDictationLanguageSetting()
try await settle()
XCTAssertEqual(preparations, 5, "no change, no preparation")
XCTAssertEqual(languages.recentDictationLanguages, ["de-DE", "nb-NO"], "picks in Hex are recents")
}
private func settle() async throws {
for _ in 0..<8 {
try await Task.sleep(for: .milliseconds(25))
}
}
}
/// The dictation language rule: an explicit choice, else the first preferred language either
/// Apple Speech module has, `SpeechTranscriber` first.
final class SpeechLanguagePickerTests: XCTestCase {
/// SpeechTranscriber's and DictationTranscriber's locales as of iOS 26 (abridged).
private let speechLocales = ["en-US", "en-GB", "de-DE", "es-ES", "es-MX", "fr-FR", "hi-IN"]
private let dictationLocales = ["en-US", "en-GB", "de-DE", "es-ES", "es-MX", "fr-FR", "nb-NO", "da-DK", "sv-SE", "he-IL"]
private func lookup(_ list: [String]) -> (Locale) async -> Locale? {
{ wanted in
let code = wanted.language.languageCode?.identifier
let region = wanted.region?.identifier
let matches = list.map { Locale(identifier: $0) }.filter { $0.language.languageCode?.identifier == code }
return matches.first { $0.region?.identifier == region } ?? matches.first
}
}
private func pick(explicit: String? = nil, preferred: [String], current: String = "en_US") async -> SpeechLanguage? {
await SpeechLanguagePicker.pick(
explicit: explicit,
candidates: SpeechLanguagePicker.automaticCandidates(preferred: preferred, current: current),
speech: lookup(speechLocales),
dictation: lookup(dictationLocales)
)
}
func testANorwegianIPhoneGetsNorwegianFromDictationTranscriber() async {
let picked = await pick(preferred: ["nb-NO", "en-US"], current: "nb_NO")
XCTAssertEqual(picked?.identifier, "nb-NO")
XCTAssertEqual(picked?.module, .dictation)
}
func testSpeechTranscriberStaysWhereItAlreadyWorks() async {
let english = await pick(preferred: ["en-US"])
XCTAssertEqual(english?.identifier, "en-US")
XCTAssertEqual(english?.module, .speech)
let spanish = await pick(preferred: ["es-MX", "en-US"])
XCTAssertEqual(spanish?.identifier, "es-MX")
XCTAssertEqual(spanish?.module, .speech)
}
func testThePreferredOrderWinsOverTheModule() async {
// Hebrew first: DictationTranscriber's Hebrew beats SpeechTranscriber's English.
let picked = await pick(preferred: ["he-IL", "en-US"])
XCTAssertEqual(picked?.identifier, "he-IL")
XCTAssertEqual(picked?.module, .dictation)
}
func testNynorskFallsBackToBokmal() async {
XCTAssertEqual(
SpeechLanguagePicker.automaticCandidates(preferred: ["nn-NO", "en-US"], current: "nn_NO"),
["nn-NO", "nb-NO", "en-US"]
)
let picked = await pick(preferred: ["nn-NO", "en-US"], current: "nn_NO")
XCTAssertEqual(picked?.identifier, "nb-NO")
}
func testAnExplicitChoiceWinsAndFallsBackToAutomaticWhenUnsupported() async {
let danish = await pick(explicit: "da-DK", preferred: ["en-US"])
XCTAssertEqual(danish?.identifier, "da-DK")
XCTAssertEqual(danish?.module, .dictation)
let german = await pick(explicit: "de-DE", preferred: ["nb-NO"])
XCTAssertEqual(german?.identifier, "de-DE")
XCTAssertEqual(german?.module, .speech)
let klingon = await pick(explicit: "tlh", preferred: ["nb-NO", "en-US"])
XCTAssertEqual(klingon?.identifier, "nb-NO", "an unsupported choice falls back to Automatic")
}
func testUSEnglishIsTheLastResort() async {
let picked = await pick(preferred: ["xx-XX"], current: "xx_XX")
XCTAssertEqual(picked?.identifier, "en-US")
}
func testShortNamesDropTheRegionOnlyWhenItIsTheOnlyOne() {
let english = Locale(identifier: "en-US")
let names = SpeechLanguagePicker.shortNames(for: ["nb-NO", "en-US", "en-GB", "he-IL"], in: english)
XCTAssertEqual(names["nb-NO"], "Norwegian Bokmål")
XCTAssertEqual(names["he-IL"], "Hebrew")
XCTAssertEqual(names["en-US"], "English (United States)")
}
func testOnlyEnglishCountsAsEnglish() {
XCTAssertTrue(SpeechLanguagePicker.isEnglish(nil))
XCTAssertTrue(SpeechLanguagePicker.isEnglish("en-GB"))
XCTAssertFalse(SpeechLanguagePicker.isEnglish("nb-NO"))
XCTAssertFalse(SpeechLanguagePicker.isEnglish("he-IL"))
}
}
/// Real Apple Speech on real clips, through the app's `AppleSpeechTranscriber`. Skipped unless
/// `HEX_SPEECH_CLIPS` names a folder of `<language>.wav` files (`nb-NO.wav`, made with
/// `say -v Nora`); it downloads each language's model, so it needs the network. It skips the
/// Speech Recognition prompt, which a test can't answer (simctl can't grant it either).
final class AppleSpeechClipTests: XCTestCase {
func testTranscribesEachClipInItsLanguage() async throws {
guard let folder = ProcessInfo.processInfo.environment["HEX_SPEECH_CLIPS"] else {
throw XCTSkip("Set HEX_SPEECH_CLIPS to a folder of <language>.wav clips")
}
guard #available(iOS 26.0, *) else { throw XCTSkip("Apple Speech needs iOS 26") }
let clips = try FileManager.default.contentsOfDirectory(atPath: folder)
.filter { $0.hasSuffix(".wav") }
.sorted()
XCTAssertFalse(clips.isEmpty)
var report: [String] = []
for clip in clips {
let identifier = String(clip.dropLast(4))
let suite = "hex.tests.\(UUID().uuidString)"
let defaults = try XCTUnwrap(UserDefaults(suiteName: suite))
defer { defaults.removePersistentDomain(forName: suite) }
let settings = LanguageSettings(defaults: defaults)
settings.dictationLanguage = identifier
guard let language = await AppleSpeechLanguages.resolve(settings: settings),
Locale(identifier: language.identifier).language.languageCode
== Locale(identifier: identifier).language.languageCode else {
report.append("\(identifier): unsupported by SpeechTranscriber and DictationTranscriber")
continue
}
let transcriber = AppleSpeechTranscriber(language: language, requestsAuthorization: false)
do {
try await transcriber.prepare { _ in }
let text = try await transcriber.transcribe(URL(fileURLWithPath: folder).appendingPathComponent(clip))
report.append("\(identifier) [\(language.module.rawValue) \(language.identifier)]: \(text)")
} catch {
report.append("\(identifier) [\(language.module.rawValue) \(language.identifier)]: FAILED \(error.localizedDescription)")
}
}
let joined = report.joined(separator: "\n")
print("HEX-SPEECH-CLIPS\n\(joined)\nHEX-SPEECH-CLIPS-END")
try? joined.write(toFile: (folder as NSString).appendingPathComponent("transcripts.txt"), atomically: true, encoding: .utf8)
}
}
@@ -241,6 +241,47 @@ final class FullKeyboardEngineTests: XCTestCase {
XCTAssertFalse(view.hasAllocatedRecordingPanel, "Metal scope torn down once recording ends")
}
func testSpaceBarNamesTheDictationLanguage() {
let model = KeyboardViewModel()
let view = FullKeyboardView(model: model, target: target)
view.frame = CGRect(x: 0, y: 0, width: 393, height: 262)
view.layoutIfNeeded()
XCTAssertNil(view.plane.spaceLabel, "no label until the language is known")
model.automaticLanguage = "en-US"
model.languageChoices = ["nb-NO", "en-US"]
RunLoop.main.run(until: Date().addingTimeInterval(0.05))
XCTAssertEqual(view.plane.spaceLabel, "English")
model.dictationLanguage = "nb-NO"
RunLoop.main.run(until: Date().addingTimeInterval(0.05))
XCTAssertEqual(view.plane.spaceLabel, "Norsk")
model.hasFullAccess = false
RunLoop.main.run(until: Date().addingTimeInterval(0.05))
XCTAssertNil(view.plane.spaceLabel, "without Full Access the keyboard can't dictate")
}
func testTypesOnTheNorwegianLayout() {
engine.language = .norwegian
XCTAssertNotNil(engine.layout.key(.character("å")))
XCTAssertEqual(engine.autocorrect.language, .norwegian)
tap(.shift)
tap(.character("ø"))
tap(.character("l"))
XCTAssertEqual(target.text, "Øl")
let o = engine.layout.key(.character("ø"))!
engine.touchBegan(id: 7, at: CGPoint(x: o.frame.midX, y: o.frame.midY))
XCTAssertTrue(engine.openAccentMenuIfEligible(forTouch: 7))
XCTAssertEqual(Set(engine.activeAccentMenu?.options ?? []), ["ø", "ö"])
engine.touchCancelled(id: 7)
engine.language = .german
XCTAssertNil(engine.layout.key(.character("å")))
XCTAssertNotNil(engine.layout.key(.character("ü")))
}
func testEmailFieldShowsAtAndDot() {
target.keyboardType = .emailAddress
engine.configureForField()
@@ -117,6 +117,28 @@ final class KeyboardBridgeTests: XCTestCase {
)
}
func testFillerTrimmingFollowsTheDictationLanguage() {
// "er" is Norwegian for "is"; English trims it as a filler, Norwegian must not.
XCTAssertEqual(
TranscriptFormatter.format("det er bra, eh, vi ses", style: .polished, language: "nb-NO"),
"Det er bra, vi ses."
)
XCTAssertEqual(
TranscriptFormatter.format("er I think so", style: .polished, language: "en-US"),
"I think so."
)
// Portuguese "um" is "a"; a language without a filler list trims nothing.
XCTAssertEqual(
TranscriptFormatter.format("um café, por favor", style: .casual, language: "pt-BR"),
"um café, por favor"
)
XCTAssertEqual(
TranscriptFormatter.format("um let's go", style: .polished),
"Let's go.",
"no language is English, as before"
)
}
func testSmartInsertionSpacingRespectsSurroundingContext() {
XCTAssertEqual(
TranscriptFormatter.insertionText(for: "world.", beforeContext: "Hello", afterContext: nil),
@@ -239,3 +261,66 @@ final class KeyboardBridgeTests: XCTestCase {
)
}
}
/// The keyboard's quick dictation-language switch.
final class DictationLanguageSwitchTests: XCTestCase {
private let supported = ["en-US", "en-GB", "es-ES", "es-MX", "de-DE", "nb-NO", "he-IL", "fr-FR"]
func testOffersTheCurrentThenRecentThenPreferredLanguages() {
XCTAssertEqual(
DictationLanguageSwitch.choices(
current: "de-DE",
recent: ["he-IL", "de-DE"],
preferred: ["nb-NO", "en-US"],
supported: supported
),
["de-DE", "he-IL", "nb-NO", "en-US"]
)
}
func testMatchesPreferredLanguagesToOnesAppleTranscribes() {
let choices = DictationLanguageSwitch.choices(
current: nil,
recent: [],
preferred: ["nn-NO", "es-419", "es", "en-AU", "tlh", "fr-CA", "en-GB", "de-AT"],
supported: supported
)
// Nynorsk is Bokmål; "es-419" and "es" find Spain's Spanish; Klingon has none.
XCTAssertEqual(choices, ["nb-NO", "es-ES", "en-US", "fr-FR", "en-GB"])
XCTAssertEqual(choices.count, DictationLanguageSwitch.maxChoices)
}
func testWithoutAListEveryLanguageIsOffered() {
XCTAssertEqual(
DictationLanguageSwitch.choices(current: nil, recent: [], preferred: ["nb_NO", "en"], supported: []),
["nb-NO", "en"]
)
}
func testNamesEachLanguageInItself() {
XCTAssertEqual(DictationLanguageSwitch.nativeName(for: "nb-NO"), "Norsk")
XCTAssertEqual(DictationLanguageSwitch.nativeName(for: "en-US"), "English")
XCTAssertEqual(DictationLanguageSwitch.nativeName(for: "es-ES"), "Español")
XCTAssertEqual(DictationLanguageSwitch.nativeName(for: "de-DE"), "Deutsch")
XCTAssertEqual(DictationLanguageSwitch.nativeName(for: "he-IL"), "עברית")
XCTAssertEqual(
DictationLanguageSwitch.nativeNames(for: ["en-US", "en-GB", "nb-NO"]),
["en-US": "English (US)", "en-GB": "English (GB)", "nb-NO": "Norsk"]
)
}
func testAChoiceMovesToTheFrontOfTheRecents() throws {
let suite = "hex.tests.\(UUID().uuidString)"
let defaults = try XCTUnwrap(UserDefaults(suiteName: suite))
defer { defaults.removePersistentDomain(forName: suite) }
let settings = LanguageSettings(defaults: defaults)
for id in ["nb-NO", "es-ES", "de-DE", "he-IL", "fr-FR", "en-GB", "es-ES"] {
settings.chooseDictationLanguage(id)
}
XCTAssertEqual(settings.dictationLanguage, "es-ES")
XCTAssertEqual(settings.recentDictationLanguages, ["es-ES", "en-GB", "fr-FR", "he-IL", "de-DE"])
settings.chooseDictationLanguage(nil)
XCTAssertNil(settings.dictationLanguage, "Automatic")
XCTAssertEqual(settings.recentDictationLanguages.first, "es-ES", "Automatic isn't a recent")
}
}
+148 -2
View File
@@ -17,6 +17,13 @@ final class KeyboardLayoutTests: XCTestCase {
}
}
}
// Only the letters differ between languages.
for language in KeyboardLanguage.allCases where language != .english {
for globe in [false, true] {
layouts.append(KeyboardLayout(width: width, showsGlobe: globe, language: language))
}
layouts.append(KeyboardLayout(width: width, field: .email, language: language))
}
}
return layouts
}
@@ -189,6 +196,134 @@ final class KeyboardLayoutTests: XCTestCase {
XCTAssertEqual(layout.key(.character("q"))!.hitFrame.minX, 0)
}
// MARK: - Languages
func testLetterRowsMatchApplesLayouts() {
let rows: [KeyboardLanguage: [String]] = [
.english: ["qwertyuiop", "asdfghjkl", "zxcvbnm"],
.norwegian: ["qwertyuiopå", "asdfghjkløæ", "zxcvbnm"],
.spanish: ["qwertyuiop", "asdfghjklñ", "zxcvbnm"],
.german: ["qwertzuiopü", "asdfghjklöä", "yxcvbnm"],
]
for (language, expected) in rows {
let layout = KeyboardLayout(width: 402, language: language)
for row in 0..<3 {
XCTAssertEqual(letters(layout, row: row).joined(), expected[row], "\(language) row \(row + 1)")
}
}
}
func testElevenKeyRowsFitEveryIPhoneWidthAndLandscape() {
let cases: [(CGFloat, KeyboardOrientation, CGFloat)] = [
(375, .portrait, 0), (402, .portrait, 0), (440, .portrait, 0),
(667, .landscape, 0), (874, .landscape, 0), (956, .landscape, 60),
]
for language in [KeyboardLanguage.norwegian, .german] {
for (width, orientation, inset) in cases {
let layout = KeyboardLayout(width: width, orientation: orientation, horizontalInset: inset, language: language)
let english = KeyboardLayout(width: width, orientation: orientation, horizontalInset: inset)
let m = layout.metrics
let label = "\(language) \(width)pt \(orientation)"
let inner = width - 2 * inset
XCTAssertEqual(layout.characterWidth, (inner - 2 * m.side - 10 * m.gap) / 11, accuracy: 0.001, label)
XCTAssertGreaterThan(layout.characterWidth, 26, label)
for row in 0..<2 {
let keys = layout.keys.filter { $0.row == row }.sorted { $0.frame.minX < $1.frame.minX }
XCTAssertEqual(keys.count, 11, label)
// Rows 1 and 2 run edge to edge, like English row 1.
XCTAssertEqual(keys.first!.frame.minX, inset + m.side, accuracy: 0.001, label)
XCTAssertEqual(keys.last!.frame.maxX, width - inset - m.side, accuracy: 0.001, label)
}
// Row 3's letters are as wide as the others and centred between shift and delete,
// which keep their English size, as does the whole bottom row.
let row3 = layout.keys.filter { $0.row == 2 && $0.character != nil }.sorted { $0.frame.minX < $1.frame.minX }
XCTAssertEqual(row3.first!.frame.width, layout.characterWidth, accuracy: 0.001, label)
XCTAssertEqual(row3.first!.frame.minX - inset, width - inset - row3.last!.frame.maxX, accuracy: 0.001, label)
XCTAssertEqual(layout.key(.shift)!.frame, english.key(.shift)!.frame, label)
XCTAssertEqual(layout.key(.delete)!.frame, english.key(.delete)!.frame, label)
XCTAssertLessThan(layout.key(.shift)!.frame.maxX, row3.first!.frame.minX, label)
XCTAssertEqual(
layout.keys.filter { $0.row == 3 }.map(\.frame),
english.keys.filter { $0.row == 3 }.map(\.frame),
label
)
}
}
}
func testSpanishKeepsTenKeysWithNOnRowTwo() {
for width: CGFloat in [375, 402, 440] {
let layout = KeyboardLayout(width: width, language: .spanish)
XCTAssertEqual(layout.characterWidth, layout.letterWidth, accuracy: 0.001)
let n = layout.key(.character("ñ"))!
XCTAssertEqual(n.frame.maxX, width - layout.metrics.side, accuracy: 0.001)
XCTAssertEqual(n.hitFrame.maxX, width)
XCTAssertEqual(layout.key(.character("a"))!.frame.minX, layout.metrics.side, accuracy: 0.001)
}
}
func testAccentsMatchEachLanguage() {
XCTAssertEqual(AccentCatalog.options(for: "ø", uppercase: false, language: .norwegian), ["ø", "ö"])
XCTAssertEqual(AccentCatalog.options(for: "æ", uppercase: true, language: .norwegian), ["Æ", "Ä"])
XCTAssertNil(AccentCatalog.options(for: "å", uppercase: false, language: .norwegian))
let norwegianA = AccentCatalog.options(for: "a", uppercase: false, language: .norwegian)!
XCTAssertFalse(norwegianA.contains("å") || norwegianA.contains("æ"), "they have their own keys")
XCTAssertTrue(AccentCatalog.options(for: "s", uppercase: false, language: .german)!.contains("ß"))
XCTAssertFalse(AccentCatalog.options(for: "u", uppercase: false, language: .german)!.contains("ü"))
XCTAssertEqual(AccentCatalog.options(for: "e", uppercase: false, language: .spanish)?.dropFirst().first, "é")
XCTAssertEqual(AccentCatalog.options(for: "n", uppercase: true, language: .spanish), ["N", "Ñ", "Ń"])
XCTAssertEqual(
AccentCatalog.options(for: "a", uppercase: false),
AccentCatalog.options(for: "a", uppercase: false, language: .english),
"English is unchanged"
)
}
func testAccentMenusOnTheNewKeysStayInside() {
let keys: [KeyboardLanguage: [String]] = [
.norwegian: ["å", "ø", "æ", "a", "o"],
.spanish: ["ñ", "a", "o", "n"],
.german: ["ü", "ö", "ä", "s", "a", "u"],
]
for width in portraitWidths {
for (language, characters) in keys {
let layout = KeyboardLayout(width: width, language: language)
let bounds = CGRect(x: 0, y: 0, width: layout.width, height: layout.totalHeight)
for ch in characters {
let key = layout.key(.character(ch))!
guard let options = AccentCatalog.options(for: ch, uppercase: false, language: language) else { continue }
let menu = AccentMenu.build(for: key.frame, rawOptions: options, in: layout.width)
XCTAssertTrue(
bounds.insetBy(dx: -0.01, dy: -0.01).contains(menu.bubbleFrame),
"\(language) \(width) \(ch): \(menu.bubbleFrame)"
)
XCTAssertEqual(menu.selectedOption, ch)
}
}
}
}
func testTheLayoutFollowsTheDictationLanguage() {
func automatic(_ dictation: String?, modes: [String] = [], preferred: [String] = []) -> KeyboardLanguage {
KeyboardLanguage.automatic(dictationLanguage: dictation, inputModes: modes, preferredLanguages: preferred)
}
XCTAssertEqual(automatic("nb-NO"), .norwegian)
XCTAssertEqual(automatic("es-MX"), .spanish)
XCTAssertEqual(automatic("de-AT"), .german)
XCTAssertEqual(automatic("en-GB", modes: ["nb-NO"]), .english, "the dictation language wins")
XCTAssertEqual(automatic("he-IL", preferred: ["he-IL", "de-DE"]), .english, "Hebrew has no layout yet")
// Before Hex knows the dictation language: the first keyboard, then language, with a layout.
XCTAssertEqual(automatic(nil, modes: ["emoji", "nb-NO", "en-US"]), .norwegian)
XCTAssertEqual(automatic(nil, preferred: ["nn-NO"]), .norwegian)
XCTAssertEqual(automatic(nil, preferred: ["fr-FR"]), .english)
}
func testSpellCheckUsesTheLayoutsLanguageOrNone() {
XCTAssertEqual(KeyboardLanguage.norwegian.spellCheckLanguage(available: ["en_US", "nb"]), "nb")
XCTAssertNil(KeyboardLanguage.norwegian.spellCheckLanguage(available: ["en_US", "da"]))
XCTAssertEqual(KeyboardLanguage.german.spellCheckLanguage(available: ["de"]), "de")
}
// MARK: - Helpers
private func landscapeLayouts() -> [KeyboardLayout] {
@@ -196,7 +331,18 @@ final class KeyboardLayoutTests: XCTestCase {
KeyboardLayout(width: 667, orientation: .landscape),
KeyboardLayout(width: 852, orientation: .landscape, layer: .numbers),
KeyboardLayout(width: 956, orientation: .landscape, field: .email, horizontalInset: 60),
]
] + KeyboardLanguage.allCases.flatMap { language in
[
KeyboardLayout(width: 667, orientation: .landscape, language: language),
KeyboardLayout(width: 956, orientation: .landscape, showsGlobe: true, horizontalInset: 60, language: language),
]
}
}
private func letters(_ layout: KeyboardLayout, row: Int) -> [String] {
layout.keys.filter { $0.row == row && $0.character != nil }
.sorted { $0.frame.minX < $1.frame.minX }
.compactMap(\.character)
}
private func bottomRow(_ layout: KeyboardLayout) -> [KeyKind] {
@@ -204,7 +350,7 @@ final class KeyboardLayoutTests: XCTestCase {
}
private func describe(_ layout: KeyboardLayout) -> String {
"\(layout.width)pt \(layout.orientation) \(layout.layer) \(layout.field) globe=\(layout.showsGlobe)"
"\(layout.width)pt \(layout.orientation) \(layout.layer) \(layout.field) globe=\(layout.showsGlobe) \(layout.language)"
}
}
+2
View File
@@ -238,6 +238,8 @@ final class TypingLogTests: XCTestCase {
let defaults = UserDefaults(suiteName: "TypingLogTests-\(UUID().uuidString)")!
let autocorrect = AutocorrectEngine(checker: Checker(), learnedWords: LearnedWordsStore(defaults: defaults))
let view = FullKeyboardView(model: KeyboardViewModel(), target: target, autocorrect: autocorrect)
// English, whatever dictation language the simulator's App Group holds.
view.languageOverride = .english
view.engine.autocorrect.isEnabled = true
view.engine.clock = { [unowned self] in self.clock }
view.orientationOverride = .portrait
+119
View File
@@ -0,0 +1,119 @@
import AVFAudio
import Foundation
import Speech
/// The languages Apple Speech transcribes on this iPhone, shared by the app and in-keyboard
/// dictation so both pick the same language and the same downloaded model.
@available(iOS 26.0, *)
nonisolated enum AppleSpeechLanguages {
/// The language dictation uses: Settings › Dictation language, else Automatic. Records
/// the result in the App Group for the keyboard layout and translation.
static func resolve(settings: LanguageSettings = LanguageSettings()) async -> SpeechLanguage? {
let explicit = settings.dictationLanguage
let picked = await pick(explicit: explicit)
if let picked, settings.resolvedDictationLanguage != picked.identifier {
settings.resolvedDictationLanguage = picked.identifier
}
if explicit == nil, let picked, settings.automaticDictationLanguage != picked.identifier {
settings.automaticDictationLanguage = picked.identifier
}
return picked
}
/// What Automatic picks on this iPhone, whatever the setting says. Records it for the
/// keyboard's language switch.
static func automatic(settings: LanguageSettings = LanguageSettings()) async -> SpeechLanguage? {
let picked = await pick(explicit: nil)
if let picked, settings.automaticDictationLanguage != picked.identifier {
settings.automaticDictationLanguage = picked.identifier
}
return picked
}
private static func pick(explicit: String?) async -> SpeechLanguage? {
await SpeechLanguagePicker.pick(
explicit: explicit,
candidates: SpeechLanguagePicker.automaticCandidates(
preferred: Locale.preferredLanguages,
current: Locale.current.identifier
),
speech: { locale in
guard SpeechTranscriber.isAvailable else { return nil }
return await SpeechTranscriber.supportedLocale(equivalentTo: locale)
},
dictation: { locale in
await DictationTranscriber.supportedLocale(equivalentTo: locale)
}
)
}
/// Every language either module transcribes, sorted by name. Records the list for the
/// keyboard's language switch.
static func options(
display: Locale = .current,
settings: LanguageSettings = LanguageSettings()
) async -> [SpeechLanguageOption] {
var identifiers: [String] = []
var seen = Set<String>()
let speech = SpeechTranscriber.isAvailable ? await SpeechTranscriber.supportedLocales : []
for locale in speech + (await DictationTranscriber.supportedLocales) {
let identifier = locale.identifier(.bcp47)
if seen.insert(identifier).inserted {
identifiers.append(identifier)
}
}
if !identifiers.isEmpty, settings.supportedDictationLanguages != identifiers {
settings.supportedDictationLanguages = identifiers
}
return identifiers
.map { SpeechLanguageOption(id: $0, name: SpeechLanguagePicker.displayName(for: $0, in: display)) }
.sorted { $0.name.localizedStandardCompare($1.name) == .orderedAscending }
}
}
/// One dictation's Apple Speech module: `SpeechTranscriber` or `DictationTranscriber`.
@available(iOS 26.0, *)
nonisolated enum AppleSpeechModule {
case speech(SpeechTranscriber)
case dictation(DictationTranscriber)
init(_ language: SpeechLanguage) {
switch language.module {
case .speech:
self = .speech(SpeechTranscriber(locale: language.locale, preset: .transcription))
case .dictation:
// Punctuated, without the short-form hint: dictations run up to a minute.
self = .dictation(DictationTranscriber(locale: language.locale, preset: .longDictation))
}
}
var module: any SpeechModule {
switch self {
case .speech(let transcriber): transcriber
case .dictation(let transcriber): transcriber
}
}
var name: String {
switch self {
case .speech: "SpeechTranscriber"
case .dictation: "DictationTranscriber"
}
}
/// The final results joined, once the analyzer has finished the file.
func finalText() async throws -> String {
var transcript = AttributedString()
switch self {
case .speech(let transcriber):
for try await result in transcriber.results where result.isFinal {
transcript += result.text
}
case .dictation(let transcriber):
for try await result in transcriber.results where result.isFinal {
transcript += result.text
}
}
return String(transcript.characters).trimmingCharacters(in: .whitespacesAndNewlines)
}
}
+9 -3
View File
@@ -147,15 +147,19 @@ nonisolated final class CleanupTagger: @unchecked Sendable {
}
/// Clean up what I say, as dictation runs it: after transcription and learned fixes,
/// before translation and formatting. Off, Verbatim, or blank keeps `text` as is.
/// before translation and formatting. Off, Verbatim, blank, or a language other than
/// English (the tagger only knows English) keeps `text` as is. A nil `language` counts
/// as English.
static func cleanIfEnabled(
_ text: String,
style: DictationStyle,
language: String? = nil,
settings: CleanupTaggerSettings = CleanupTaggerSettings(),
tagger: CleanupTagger = .shared
) async -> String {
guard settings.isEnabled,
style != .verbatim,
SpeechLanguagePicker.isEnglish(language),
!text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
return text
}
@@ -163,13 +167,15 @@ nonisolated final class CleanupTagger: @unchecked Sendable {
}
/// Starts loading the model when a dictation stops recording, while it transcribes, so
/// cleanup is ready by the time the transcript is. Does nothing while the experiment is off or for Verbatim.
/// cleanup is ready by the time the transcript is. Does nothing while the experiment is
/// off, for Verbatim, or for a language other than English.
static func prepareIfEnabled(
style: DictationStyle,
language: String? = nil,
settings: CleanupTaggerSettings = CleanupTaggerSettings(),
tagger: CleanupTagger = .shared
) {
guard settings.isEnabled, style != .verbatim else { return }
guard settings.isEnabled, style != .verbatim, SpeechLanguagePicker.isEnglish(language) else { return }
tagger.prepare()
}
+193
View File
@@ -0,0 +1,193 @@
import Foundation
/// Settings › Dictation language and Experiments › Full keyboard › Keyboard layout, in the
/// App Group so the keyboard reads them and can switch the language.
nonisolated struct LanguageSettings {
private enum Key {
static let dictation = "settings.dictationLanguage"
static let resolvedDictation = "settings.dictationLanguage.resolved"
static let automaticDictation = "settings.dictationLanguage.automatic"
static let recent = "settings.dictationLanguage.recent"
static let supported = "settings.dictationLanguage.supported"
static let keyboardLayout = "settings.keyboardLayout"
}
static let maxRecent = 5
private let defaults: UserDefaults?
init(defaults: UserDefaults? = UserDefaults(suiteName: KeyboardBridge.appGroup)) {
self.defaults = defaults
}
/// A BCP 47 identifier such as `nb-NO`, or nil for Automatic.
var dictationLanguage: String? {
get { defaults?.string(forKey: Key.dictation) }
nonmutating set { defaults?.set(newValue, forKey: Key.dictation) }
}
/// The language Apple Speech last resolved for dictation (the app or the keyboard), so
/// translation can follow it without asking Speech again.
var resolvedDictationLanguage: String? {
get { defaults?.string(forKey: Key.resolvedDictation) }
nonmutating set { defaults?.set(newValue, forKey: Key.resolvedDictation) }
}
/// What Automatic picks on this iPhone, when the app or keyboard last worked it out.
var automaticDictationLanguage: String? {
get { defaults?.string(forKey: Key.automaticDictation) }
nonmutating set { defaults?.set(newValue, forKey: Key.automaticDictation) }
}
/// Languages picked lately, newest first, for the keyboard's quick switch.
var recentDictationLanguages: [String] {
get { defaults?.stringArray(forKey: Key.recent) ?? [] }
nonmutating set { defaults?.set(Array(newValue.prefix(Self.maxRecent)), forKey: Key.recent) }
}
/// Every language Apple Speech transcribes here (BCP 47), as last listed, so the
/// keyboard can offer only those without asking Speech.
var supportedDictationLanguages: [String] {
get { defaults?.stringArray(forKey: Key.supported) ?? [] }
nonmutating set { defaults?.set(newValue, forKey: Key.supported) }
}
/// The full keyboard's letters, or nil for Automatic (follow the dictation language).
var keyboardLayout: KeyboardLanguage? {
get { defaults?.string(forKey: Key.keyboardLayout).flatMap(KeyboardLanguage.init(rawValue:)) }
nonmutating set { defaults?.set(newValue?.rawValue, forKey: Key.keyboardLayout) }
}
/// The language dictation uses, when known: the one set, else what Automatic picks,
/// else the last one it ran in.
var effectiveDictationLanguage: String? {
dictationLanguage ?? automaticDictationLanguage ?? resolvedDictationLanguage
}
/// A choice made in Hex or the keyboard: the setting, and the front of the recents.
func chooseDictationLanguage(_ identifier: String?) {
dictationLanguage = identifier
if let identifier {
recentDictationLanguages = [identifier] + recentDictationLanguages.filter { $0 != identifier }
}
}
}
/// The Apple Speech module that transcribes a language. `SpeechTranscriber` is Apple's newer
/// model and Hex's default; `DictationTranscriber` (the model behind the iOS keyboard's
/// dictation) covers languages it lacks, such as Norwegian, Danish, and Swedish.
nonisolated enum SpeechModuleKind: String, Codable, Sendable {
case speech
case dictation
}
nonisolated struct SpeechLanguage: Equatable, Sendable {
let locale: Locale
let module: SpeechModuleKind
var identifier: String { locale.identifier(.bcp47) }
var languageCode: String? { locale.language.languageCode?.identifier }
}
/// One entry in Settings › Dictation language.
nonisolated struct SpeechLanguageOption: Equatable, Identifiable, Sendable {
/// BCP 47, such as `nb-NO`.
let id: String
let name: String
}
/// Which language and Apple Speech module a dictation uses.
nonisolated enum SpeechLanguagePicker {
/// What Automatic tries, in order: the iPhone's preferred languages, its own locale, then
/// US English. Norwegian Nynorsk also tries Bokmål, the only Norwegian Apple transcribes.
static func automaticCandidates(preferred: [String], current: String) -> [String] {
var result: [String] = []
var seen = Set<String>()
func add(_ identifier: String) {
let key = Locale(identifier: identifier).identifier(.bcp47)
if seen.insert(key).inserted {
result.append(identifier)
}
}
for identifier in preferred + [current, "en-US"] {
add(identifier)
if Locale(identifier: identifier).language.languageCode?.identifier == "nn" {
add("nb-NO")
}
}
return result
}
/// An explicit choice wins: `SpeechTranscriber` when it has that language, else
/// `DictationTranscriber`. Automatic takes the first candidate either module has,
/// preferring `SpeechTranscriber` for the same language, so a Norwegian iPhone gets
/// Norwegian from `DictationTranscriber` rather than English from `SpeechTranscriber`.
/// `speech` and `dictation` return each module's equivalent locale, or nil.
static func pick(
explicit: String?,
candidates: [String],
speech: (Locale) async -> Locale?,
dictation: (Locale) async -> Locale?
) async -> SpeechLanguage? {
if let explicit, !explicit.isEmpty {
let wanted = Locale(identifier: explicit)
if let found = await speech(wanted), sameLanguage(found, wanted) {
return SpeechLanguage(locale: found, module: .speech)
}
if let found = await dictation(wanted) {
return SpeechLanguage(locale: found, module: .dictation)
}
}
for identifier in candidates {
let candidate = Locale(identifier: identifier)
if let found = await speech(candidate) {
return SpeechLanguage(locale: found, module: .speech)
}
if let found = await dictation(candidate) {
return SpeechLanguage(locale: found, module: .dictation)
}
}
return nil
}
/// The same language and, when both name one, the same region.
private static func sameLanguage(_ a: Locale, _ b: Locale) -> Bool {
guard a.language.languageCode == b.language.languageCode else { return false }
guard let ra = a.region, let rb = b.region else { return true }
return ra == rb
}
/// "Norwegian Bokmål (Norway)" in the iPhone's own language.
static func displayName(for identifier: String, in display: Locale = .current) -> String {
display.localizedString(forIdentifier: identifier)
?? Locale(identifier: identifier).localizedString(forIdentifier: identifier)
?? identifier
}
/// Short names for a list of locales: the language alone where it has one region in the
/// list ("Norwegian Bokmål"), the full name where it has several ("English (Canada)").
static func shortNames(for identifiers: [String], in display: Locale = .current) -> [String: String] {
var regionsPerLanguage: [String: Int] = [:]
for identifier in identifiers {
let code = Locale(identifier: identifier).language.languageCode?.identifier ?? identifier
regionsPerLanguage[code, default: 0] += 1
}
var names: [String: String] = [:]
for identifier in identifiers {
let code = Locale(identifier: identifier).language.languageCode?.identifier ?? identifier
if regionsPerLanguage[code] == 1, let language = display.localizedString(forLanguageCode: code) {
names[identifier] = language.prefix(1).uppercased() + language.dropFirst()
} else {
names[identifier] = displayName(for: identifier, in: display)
}
}
return names
}
/// Whether a language identifier is English, for the English-only steps (the cleanup
/// tagger). nil (unknown) counts as English, as before languages existed.
static func isEnglish(_ identifier: String?) -> Bool {
guard let identifier else { return true }
return Locale(identifier: identifier).language.languageCode?.identifier == "en"
}
}
@@ -0,0 +1,70 @@
import Foundation
/// The keyboard's quick dictation-language switch: which languages it offers and the names
/// it shows for them. The choice is Settings › Dictation language (`LanguageSettings`).
nonisolated enum DictationLanguageSwitch {
static let maxChoices = 5
/// Up to five languages, best first: the one set now, the ones picked recently, then the
/// iPhone's preferred languages. Each is matched to a language Apple Speech has
/// (`supported`, BCP 47) when that list is known, and dropped when it has none.
static func choices(current: String?, recent: [String], preferred: [String], supported: [String]) -> [String] {
var result: [String] = []
for identifier in [current].compactMap({ $0 }) + recent + preferred {
guard let match = supportedMatch(for: identifier, in: supported),
!result.contains(match) else { continue }
result.append(match)
if result.count == maxChoices { break }
}
return result
}
/// The supported locale for an identifier: the same region, else the language's usual
/// region ("es" is Spain's), else any. Nynorsk maps to Bokmål. With no list, itself.
static func supportedMatch(for identifier: String, in supported: [String]) -> String? {
let wanted = Locale(identifier: identifier)
guard !supported.isEmpty else { return wanted.identifier(.bcp47) }
var code = wanted.language.languageCode?.identifier
if code == "nn" || code == "no" { code = "nb" }
let candidates = supported.filter { Locale(identifier: $0).language.languageCode?.identifier == code }
guard !candidates.isEmpty, let code else { return nil }
if let region = wanted.region?.identifier,
let exact = candidates.first(where: { Locale(identifier: $0).region?.identifier == region }) {
return exact
}
let usual = Locale.Language(identifier: Locale.Language(identifier: code).maximalIdentifier).region?.identifier
return candidates.first { Locale(identifier: $0).region?.identifier == usual } ?? candidates.first
}
/// The language's own short name: "English", "Norsk", "Español", "Deutsch", "עברית".
static func nativeName(for identifier: String) -> String {
let code = Locale(identifier: identifier).language.languageCode?.identifier ?? identifier
switch code {
case "nb", "no": return "Norsk"
case "nn": return "Nynorsk"
default: break
}
let own = Locale(identifier: code)
guard let name = own.localizedString(forLanguageCode: code) else { return identifier }
return name.prefix(1).uppercased(with: own) + name.dropFirst()
}
/// Native names for a list, with the region code only where two share a language:
/// "English (US)", "English (GB)", "Norsk".
static func nativeNames(for identifiers: [String]) -> [String: String] {
var perLanguage: [String: Int] = [:]
for identifier in identifiers {
perLanguage[nativeName(for: identifier), default: 0] += 1
}
var names: [String: String] = [:]
for identifier in identifiers {
let name = nativeName(for: identifier)
if perLanguage[name, default: 0] > 1, let region = Locale(identifier: identifier).region?.identifier {
names[identifier] = "\(name) (\(region))"
} else {
names[identifier] = name
}
}
return names
}
}
+4 -2
View File
@@ -31,15 +31,17 @@ struct DictationTranslationSettings {
enum DictationTranslator {
/// The translated text, or nil when the experiment is off, the speech is already in
/// the target language, or the pack isn't downloaded.
/// the target language, or the pack isn't downloaded. `source` is the dictation
/// language; without one the text's language is guessed.
@MainActor
static func translateIfEnabled(
_ text: String,
source: Locale.Language? = nil,
settings: DictationTranslationSettings = DictationTranslationSettings()
) async -> String? {
guard settings.isEnabled, #available(iOS 26.0, *) else { return nil }
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty, let source = detectedLanguage(of: trimmed) else { return nil }
guard !trimmed.isEmpty, let source = source ?? detectedLanguage(of: trimmed) else { return nil }
let target = Locale.Language(identifier: settings.targetLanguage)
guard source.languageCode != target.languageCode else { return nil }
guard await LanguageAvailability().status(from: source, to: target) == .installed else { return nil }
@@ -24,8 +24,10 @@ nonisolated enum AccentCatalog {
"/": ["/", "\\"],
]
static func options(for base: String, uppercase: Bool) -> [String]? {
guard let list = table[base.lowercased()] else { return nil }
/// The accents for a held key; `language` may replace a key's list (`KeyboardLanguage.accentOverrides`).
static func options(for base: String, uppercase: Bool, language: KeyboardLanguage = .english) -> [String]? {
let key = base.lowercased()
guard let list = language.accentOverrides[key] ?? table[key] else { return nil }
guard uppercase else { return list }
return list.map { option in
// German sharp s capitalizes to SS or ẞ; standard `.uppercased()` matches system expectations.
@@ -27,17 +27,27 @@ protocol TextChecking: AnyObject {
func isWordValid(_ word: String) -> Bool
func guesses(for word: String) -> [String]
func completions(for prefix: String) -> [String]
/// Switches the dictionary (a `UITextChecker` language code such as `nb`).
func useLanguage(_ code: String)
}
extension TextChecking {
func useLanguage(_ code: String) {}
}
@MainActor
final class SystemTextChecker: TextChecking {
private var checker: UITextChecker?
private let language: String
private var language: String
init(language: String = "en_US") {
self.language = language
}
func useLanguage(_ code: String) {
language = code
}
/// Warms `UITextChecker` after the keyboard's first frame commits so cold open stays < 12 ms.
func warmUp() {
guard checker == nil else { return }
@@ -165,6 +175,12 @@ final class AutocorrectEngine {
let checker: TextChecking
let learnedWords: LearnedWordsStore
var isEnabled = true
/// The keyboard layout's language: its dictionary, and whether English fixes apply.
private(set) var language: KeyboardLanguage = .english
/// False when the iPhone has no spelling dictionary for `language`: then nothing is
/// suggested or corrected but your own text replacements, rather than "correcting"
/// the language into another.
private(set) var hasDictionary = true
private var lexiconShortcuts: [String: String] = [:]
private var lexiconTerms: [String] = []
private var suppressedInSession: Set<String> = []
@@ -214,6 +230,18 @@ final class AutocorrectEngine {
return lastCorrection
}
/// Follows the keyboard layout. `available` is `UITextChecker.availableLanguages`.
func setLanguage(_ language: KeyboardLanguage, available: [String] = UITextChecker.availableLanguages) {
self.language = language
let code = language.spellCheckLanguage(available: available)
hasDictionary = code != nil
if let code {
checker.useLanguage(code)
}
suppressedInSession.removeAll()
lastCorrection = nil
}
func applyLearning(_ learning: TypingLearning) {
if self.learning != learning { self.learning = learning }
}
@@ -260,7 +288,7 @@ final class AutocorrectEngine {
field: KeyboardFieldKind,
autocorrectionType: UITextAutocorrectionType?
) -> [KeyboardSuggestion] {
guard field == .text, autocorrectionType != .no else { return [] }
guard field == .text, autocorrectionType != .no, hasDictionary else { return [] }
guard let token = Self.currentToken(in: beforeInput) else { return [] }
let lower = token.lowercased()
@@ -278,7 +306,7 @@ final class AutocorrectEngine {
} else if let learned = learning.typos[lower] {
// A learned fix space won't apply (the typo is a real word, or you kept it) is still offered.
pool.append(Self.matchCase(of: token, onto: learned))
} else if let suggested = Self.suggestOnlyContractions[lower] {
} else if language.usesEnglishFixes, let suggested = Self.suggestOnlyContractions[lower] {
pool.append(Self.matchCase(of: token, onto: suggested))
}
@@ -354,6 +382,9 @@ final class AutocorrectEngine {
!learning.rejected.contains(lower),
!learnedWords.contains(lower),
!Self.shorthandAllowlist.contains(lower) else { return nil }
guard hasDictionary else {
return lexiconShortcuts[lower].map { (token, $0) }
}
guard let fix = cautiousReplacement(for: token) else { return nil }
return (token, fix)
}
@@ -376,8 +407,9 @@ final class AutocorrectEngine {
let hasInternalCaps = token.dropFirst().contains(where: \.isUppercase) && !isAllCaps
if hasInternalCaps { return nil }
// 2. Deterministic contractions and common QWERTY transpositions.
if let fixed = Self.unambiguousFixes[lower] {
// 2. Deterministic contractions and common QWERTY transpositions (English only:
// Norwegian "i" means "in" and must not become "I").
if language.usesEnglishFixes, let fixed = Self.unambiguousFixes[lower] {
if isAllCaps && token.count < 3 { return nil }
return Self.matchCase(of: token, onto: fixed)
}
@@ -394,7 +426,7 @@ final class AutocorrectEngine {
// Skip multi-word splits (`alot` -> `a lot` is fine only if single space, keep v1 single-token + apostrophe).
guard !rawGuess.contains(" ") else { continue }
let normalizedGuess = rawGuess.replacingOccurrences(of: "'", with: "").lowercased()
guard let editCost = Self.singleEditKeyboardCost(from: lower, to: normalizedGuess) else { continue }
guard let editCost = Self.singleEditKeyboardCost(from: lower, to: normalizedGuess, language: language) else { continue }
let totalScore = editCost + Double(rank) * 0.08
if totalScore < bestScore {
bestScore = totalScore
@@ -407,17 +439,20 @@ final class AutocorrectEngine {
private func rankedGuesses(for token: String) -> [String] {
let lower = token.lowercased()
let language = language
return checker.guesses(for: token)
.prefix(8)
.enumerated()
.sorted { a, b in
let costA = Self.singleEditKeyboardCost(
from: lower,
to: a.element.replacingOccurrences(of: "'", with: "").lowercased()
to: a.element.replacingOccurrences(of: "'", with: "").lowercased(),
language: language
) ?? (2.0 + Double(abs(a.element.count - token.count)) * 0.3)
let costB = Self.singleEditKeyboardCost(
from: lower,
to: b.element.replacingOccurrences(of: "'", with: "").lowercased()
to: b.element.replacingOccurrences(of: "'", with: "").lowercased(),
language: language
) ?? (2.0 + Double(abs(b.element.count - token.count)) * 0.3)
return (costA + Double(a.offset) * 0.08) < (costB + Double(b.offset) * 0.08)
}
@@ -445,7 +480,11 @@ final class AutocorrectEngine {
}
/// Returns a weighted cost if `source` and `target` are within 1 Damerau-Levenshtein edit; `nil` otherwise.
static func singleEditKeyboardCost(from source: String, to target: String) -> Double? {
static func singleEditKeyboardCost(
from source: String,
to target: String,
language: KeyboardLanguage = .english
) -> Double? {
if source == target { return 0 }
let s = Array(source)
let t = Array(target)
@@ -460,7 +499,7 @@ final class AutocorrectEngine {
}
if diffs.count == 1 {
let i = diffs[0]
return areAdjacentKeys(s[i], t[i]) ? 0.65 : 1.15
return areAdjacentKeys(s[i], t[i], language: language) ? 0.65 : 1.15
}
if diffs.count == 2, diffs[1] == diffs[0] + 1,
s[diffs[0]] == t[diffs[1]], s[diffs[1]] == t[diffs[0]] {
@@ -487,21 +526,17 @@ final class AutocorrectEngine {
}
let extraIdx = skippedIndex ?? (longer.count - 1)
let extraChar = longer[extraIdx]
let leftAdj = extraIdx > 0 && areAdjacentKeys(extraChar, longer[extraIdx - 1])
let rightAdj = extraIdx + 1 < longer.count && areAdjacentKeys(extraChar, longer[extraIdx + 1])
let leftAdj = extraIdx > 0 && areAdjacentKeys(extraChar, longer[extraIdx - 1], language: language)
let rightAdj = extraIdx + 1 < longer.count && areAdjacentKeys(extraChar, longer[extraIdx + 1], language: language)
return (leftAdj || rightAdj) ? 0.70 : 0.90
}
private static let keyCoordinates: [Character: (Double, Double)] = {
var map: [Character: (Double, Double)] = [:]
for (col, ch) in "qwertyuiop".enumerated() { map[ch] = (0, Double(col)) }
for (col, ch) in "asdfghjkl".enumerated() { map[ch] = (1, Double(col) + 0.5) }
for (col, ch) in "zxcvbnm".enumerated() { map[ch] = (2, Double(col) + 1.5) }
return map
}()
private static let keyCoordinates: [KeyboardLanguage: [Character: (Double, Double)]] = Dictionary(
uniqueKeysWithValues: KeyboardLanguage.allCases.map { ($0, $0.keyCoordinates) }
)
static func areAdjacentKeys(_ a: Character, _ b: Character) -> Bool {
guard let pa = keyCoordinates[a], let pb = keyCoordinates[b] else { return false }
static func areAdjacentKeys(_ a: Character, _ b: Character, language: KeyboardLanguage = .english) -> Bool {
guard let keys = keyCoordinates[language], let pa = keys[a], let pb = keys[b] else { return false }
let dx = pa.1 - pb.1
let dy = pa.0 - pb.0
return (dx * dx + dy * dy).squareRoot() <= 1.6
@@ -103,6 +103,16 @@ final class FullKeyboardEngine {
var orientation: KeyboardOrientation = .portrait { didSet { if orientation != oldValue { rebuildLayout() } } }
var showsGlobe = false { didSet { if showsGlobe != oldValue { rebuildLayout() } } }
var horizontalInset: CGFloat = 0 { didSet { if horizontalInset != oldValue { rebuildLayout() } } }
/// The letters, accents and spelling dictionary (Settings › Keyboard layout).
var language: KeyboardLanguage = .english {
didSet {
guard language != oldValue else { return }
cancelAllTouches()
autocorrect.setLanguage(language)
rebuildLayout(cause: "language")
refreshSuggestions()
}
}
private(set) var layout = KeyboardLayout(width: 375)
var clock: () -> TimeInterval = { CACurrentMediaTime() }
@@ -308,7 +318,7 @@ final class FullKeyboardEngine {
let dx = touch.point.x - touch.startPoint.x
let dy = touch.point.y - touch.startPoint.y
guard (dx * dx + dy * dy).squareRoot() <= Self.accentSlop else { return false }
guard let options = AccentCatalog.options(for: base, uppercase: isUppercase),
guard let options = AccentCatalog.options(for: base, uppercase: isUppercase, language: language),
options.count > 1 else { return false }
activeAccentMenu = AccentMenu.build(
for: touch.key.frame,
@@ -687,7 +697,8 @@ final class FullKeyboardEngine {
layer: layerState.layer,
field: field,
showsGlobe: showsGlobe,
horizontalInset: horizontalInset
horizontalInset: horizontalInset,
language: language
)
logState(cause)
}
@@ -122,6 +122,8 @@ final class FullKeyboardView: UIView, UIInputViewAudioFeedback {
engine.autocorrect.isEnabled = FullKeyboardSettings().autocorrect
engine.showsKeyPopups = FullKeyboardSettings().keyPopups
refreshLanguageSignals()
syncLanguage()
engine.configureForField()
applyAppearance()
refreshFromModel()
@@ -183,7 +185,38 @@ final class FullKeyboardView: UIView, UIInputViewAudioFeedback {
engine.refreshSuggestions()
}
// MARK: - Language
/// Fixtures pin the layout; otherwise Settings › Keyboard layout, else Automatic, which
/// follows the dictation language.
var languageOverride: KeyboardLanguage? {
didSet { syncLanguage() }
}
/// The iPhone's keyboards and preferred languages, read when the keyboard appears, for
/// Automatic before any dictation language is known.
private var inputModeLanguages: [String] = []
private var preferredLanguages: [String] = []
private func refreshLanguageSignals() {
inputModeLanguages = UITextInputMode.activeInputModes.compactMap(\.primaryLanguage)
preferredLanguages = Locale.preferredLanguages
}
private func syncLanguage() {
let settings = LanguageSettings()
let next = languageOverride ?? settings.keyboardLayout ?? KeyboardLanguage.automatic(
dictationLanguage: settings.effectiveDictationLanguage,
inputModes: inputModeLanguages,
preferredLanguages: preferredLanguages
)
guard engine.language != next else { return }
engine.language = next
setNeedsLayout()
}
func syncSettings() {
syncLanguage()
let journal = TypingJournal.sharedIfEnabled()
if engine.journal !== journal {
engine.journal = journal
@@ -214,6 +247,7 @@ final class FullKeyboardView: UIView, UIInputViewAudioFeedback {
/// The keyboard came on screen for a (possibly new) field.
func prepareForField() {
refreshLanguageSignals()
syncSettings()
plane.cancelAllTouches()
engine.configureForField()
@@ -255,6 +289,16 @@ final class FullKeyboardView: UIView, UIInputViewAudioFeedback {
engine.touchBegan(id: -1, at: CGPoint(x: key.frame.midX, y: key.frame.midY))
}
/// Fixtures: show a key's accent callout with `option` selected.
func debugHoldAccent(_ character: String, selecting option: String) {
debugHoldAccent(character)
guard let menu = engine.activeAccentMenu,
let index = menu.options.firstIndex(of: option),
let key = engine.layout.key(.character(character)) else { return }
let cell = menu.cellFrames[index]
engine.touchMoved(id: -2, to: CGPoint(x: cell.midX, y: key.frame.midY))
}
/// Fixtures: show a key's long-press accent callout.
func debugHoldAccent(_ character: String, selectIndex: Int? = nil) {
guard let key = engine.layout.key(.character(character)) else { return }
@@ -374,14 +418,24 @@ final class FullKeyboardView: UIView, UIInputViewAudioFeedback {
setRecording(recording)
}
refreshRecordingPanel()
plane.spaceLabel = model.hasFullAccess ? model.languageLabel : nil
// A language picked in the switch changes the layout at once when it follows dictation.
syncLanguage()
if let menu = styleMenu {
menu.style = model.style
menu.returnAppID = model.returnAppID
menu.quickReturnApps = model.quickReturnApps
applyLanguages(to: menu)
}
refreshStrip()
}
private func applyLanguages(to menu: HexStyleMenuView) {
menu.dictationLanguage = model.dictationLanguage
menu.automaticLanguage = model.automaticLanguage
menu.languageChoices = model.showsLanguageSwitch ? model.languageChoices : []
}
private func refreshStrip() {
let status: String? = if !model.hasFullAccess {
"Allow Full Access to dictate with Hex"
@@ -643,7 +697,13 @@ final class FullKeyboardView: UIView, UIInputViewAudioFeedback {
menu.style = model.style
menu.returnAppID = model.returnAppID
menu.quickReturnApps = model.quickReturnApps
applyLanguages(to: menu)
menu.onHoverChanged = { [weak self] in self?.selectionFeedback() }
menu.onLanguage = { [weak self] language in
self?.typingLog?.ui("dictation_language", [TypingLogField("language", .string(language ?? "automatic"))])
self?.model.onLanguageSelected?(language)
self?.dismissStyleMenu()
}
menu.onStyle = { [weak self] style in
self?.typingLog?.ui("style", [
TypingLogField("style", .string(style.rawValue)),
@@ -669,9 +729,10 @@ final class FullKeyboardView: UIView, UIInputViewAudioFeedback {
styleMenu = nil
}
/// Fixtures: open the menu without a long press.
func debugShowStyleMenu() {
/// Fixtures: open the menu without a long press, optionally on another page.
func debugShowStyleMenu(page: HexStyleMenuView.Page = .styles) {
showStyleMenu()
styleMenu?.show(page: page)
}
private func undoDictation() {
+25 -2
View File
@@ -330,14 +330,20 @@ final class PanelKeyButton: UIControl {
/// Holding the Hex key: dictation styles, then "Return to" for the app Hex goes back to.
@MainActor
final class HexStyleMenuView: UIView {
enum Page { case styles, returnApps }
enum Page { case styles, returnApps, languages }
var palette = FullKeyboardPalette(isDark: false) { didSet { rebuild() } }
var style: DictationStyle = .polished { didSet { rebuild() } }
var returnAppID: String? { didSet { rebuild() } }
var quickReturnApps: [HostApp] = HostAppCatalog.quickChips { didSet { if quickReturnApps != oldValue { rebuild() } } }
/// Settings › Dictation language (nil for Automatic), what Automatic resolves to, and the
/// quick choices; with no choices the menu has no Language row.
var dictationLanguage: String? { didSet { if dictationLanguage != oldValue { rebuild() } } }
var automaticLanguage: String? { didSet { if automaticLanguage != oldValue { rebuild() } } }
var languageChoices: [String] = [] { didSet { if languageChoices != oldValue { rebuild() } } }
var onStyle: ((DictationStyle) -> Void)?
var onReturnApp: ((String?) -> Void)?
var onLanguage: ((String?) -> Void)?
var onDismiss: (() -> Void)?
var onHoverChanged: (() -> Void)?
/// The Hex key's keycap, in this view's coordinates; the panel opens above it.
@@ -418,8 +424,14 @@ final class HexStyleMenuView: UIView {
self?.onStyle?(option)
})
}
if !languageChoices.isEmpty {
let current = (dictationLanguage ?? automaticLanguage).map(DictationLanguageSwitch.nativeName(for:))
rows.append(makeRow(title: "Language", detail: current ?? "Automatic", chevron: true, selected: false, showsTopDivider: true) { [weak self] in
self?.show(page: .languages)
})
}
let appName = HostAppCatalog.app(for: returnAppID)?.name ?? "None"
rows.append(makeRow(title: "Return to", detail: appName, chevron: true, selected: false, showsTopDivider: true) { [weak self] in
rows.append(makeRow(title: "Return to", detail: appName, chevron: true, selected: false, showsTopDivider: languageChoices.isEmpty) { [weak self] in
self?.show(page: .returnApps)
})
case .returnApps:
@@ -435,6 +447,17 @@ final class HexStyleMenuView: UIView {
self?.onReturnApp?(app.id)
})
}
case .languages:
let automatic = automaticLanguage.map(DictationLanguageSwitch.nativeName(for:))
rows.append(makeRow(title: "Automatic", detail: automatic, selected: dictationLanguage == nil) { [weak self] in
self?.onLanguage?(nil)
})
let names = DictationLanguageSwitch.nativeNames(for: languageChoices)
for id in languageChoices {
rows.append(makeRow(title: names[id] ?? id, selected: id == dictationLanguage) { [weak self] in
self?.onLanguage?(id)
})
}
}
rows.forEach(panel.addSubview)
setNeedsLayout()
@@ -11,6 +11,8 @@ final class KeyboardKeyPlaneView: UIView {
var returnSymbol: String? = "return" { didSet { if returnSymbol != oldValue { setNeedsDisplay() } } }
var returnIsAction = false { didSet { if returnIsAction != oldValue { setNeedsDisplay() } } }
var returnIsEnabled = true { didSet { if returnIsEnabled != oldValue { setNeedsDisplay() } } }
/// The dictation language's name, drawn faintly on the space bar ("English", "Norsk").
var spaceLabel: String? { didSet { if spaceLabel != oldValue { setNeedsDisplay() } } }
/// Called on every key going down, for clicks and haptics.
var onKeyDown: ((KeyKind) -> Void)?
/// Called when long-press accents open.
@@ -83,6 +85,13 @@ final class KeyboardKeyPlaneView: UIView {
case .space:
let fill = pressed.contains(.space) ? palette.keyPressedFill : palette.keyFill
context.fillKeycap(key.frame, fill: fill, radius: radius)
guard !hideGlyphs, let spaceLabel else { continue }
drawText(
spaceLabel,
in: key.frame,
font: .systemFont(ofSize: isLandscape ? 13 : 14.5, weight: .regular),
color: palette.textMuted
)
case .returnKey:
let isPressed = pressed.contains(.returnKey) && returnIsEnabled
let activeAction = returnIsAction && returnIsEnabled && !hideGlyphs
@@ -0,0 +1,134 @@
import Foundation
/// The full keyboard's letters. Each language is its letter rows, its long-press accents,
/// and the spelling dictionary its autocorrect uses, matching Apple's keyboard for it;
/// another language is a new case with those filled in.
nonisolated enum KeyboardLanguage: String, CaseIterable, Sendable {
case english = "en"
case norwegian = "nb"
case spanish = "es"
case german = "de"
/// Its own name, as Apple's keyboard list shows it.
var displayName: String {
switch self {
case .english: "English"
case .norwegian: "Norsk"
case .spanish: "Español"
case .german: "Deutsch"
}
}
/// The three letter rows, lowercase. Rows 1 and 2 may hold 11 keys (Apple's Nordic and
/// German layouts); row 3 sits between shift and delete.
var letterRows: [[String]] {
let rows = switch self {
case .english: ["qwertyuiop", "asdfghjkl", "zxcvbnm"]
case .norwegian: ["qwertyuiopå", "asdfghjkløæ", "zxcvbnm"]
case .spanish: ["qwertyuiop", "asdfghjklñ", "zxcvbnm"]
case .german: ["qwertzuiopü", "asdfghjklöä", "yxcvbnm"]
}
return rows.map { $0.map(String.init) }
}
/// Letters held for their accents, base letter first, where this language's keyboard
/// differs from `AccentCatalog`'s shared table: letters with their own key drop out of
/// the vowels' lists.
var accentOverrides: [String: [String]] {
switch self {
case .english:
[:]
case .norwegian:
// Æ and Ø offer the Swedish and German Ä and Ö.
[
"a": ["a", "à", "á", "â", "ä", "ã", "ā"],
"o": ["o", "ô", "ö", "ò", "ó", "œ", "ō", "õ"],
"æ": ["æ", "ä"],
"ø": ["ø", "ö"],
]
case .spanish:
// Spanish accents first; ª and º with A and O.
[
"a": ["a", "á", "à", "ä", "â", "ã", "å", "ą", "æ", "ā", "ª"],
"e": ["e", "é", "è", "ë", "ê", "ę", "ė", "ē"],
"i": ["i", "í", "ï", "ì", "î", "į", "ī"],
"o": ["o", "ó", "ò", "ö", "ô", "õ", "ø", "œ", "ō", "º"],
"u": ["u", "ú", "ü", "ù", "û", "ū"],
"n": ["n", "ñ", "ń"],
]
case .german:
// ß stays on a held S (the shared table).
[
"a": ["a", "à", "á", "â", "æ", "ã", "å", "ā"],
"o": ["o", "ô", "ò", "ó", "œ", "ø", "ō", "õ"],
"u": ["u", "û", "ù", "ú", "ū"],
]
}
}
/// `UITextChecker` language codes to try, best first.
var spellCheckCandidates: [String] {
switch self {
case .english: ["en_US", "en"]
case .norwegian: ["nb", "nb_NO", "no", "no_NO"]
case .spanish: ["es", "es_ES", "es_MX"]
case .german: ["de", "de_DE"]
}
}
/// The first candidate the iPhone has a dictionary for, or nil: then autocorrect and
/// suggestions stay off rather than "correcting" this language into another.
func spellCheckLanguage(available: [String]) -> String? {
spellCheckCandidates.first { available.contains($0) }
}
/// Whether the built-in English contractions and typo list apply.
var usesEnglishFixes: Bool { self == .english }
/// Each letter's (row, column) for autocorrect's neighbouring-key cost, with rows 2
/// and 3 shifted as the keys are.
var keyCoordinates: [Character: (Double, Double)] {
var map: [Character: (Double, Double)] = [:]
let offsets = [0.0, 0.5, 1.5]
for (row, letters) in letterRows.enumerated() {
for (column, letter) in letters.enumerated() {
guard let character = letter.first else { continue }
map[character] = (Double(row), Double(column) + offsets[row])
}
}
return map
}
/// The layout for a language identifier ("nb-NO", "no", "nn_NO", "es-MX"), if any.
static func matching(_ identifier: String) -> KeyboardLanguage? {
let code = identifier.lowercased()
.split(whereSeparator: { $0 == "-" || $0 == "_" })
.first.map(String.init) ?? ""
switch code {
case "nb", "no", "nn": return .norwegian
case "en": return .english
case "es": return .spanish
case "de": return .german
default: return nil
}
}
/// Automatic follows the dictation language: its layout, or English for a language
/// without one (Hebrew, French). Before any dictation language is known, the first
/// iPhone keyboard, then preferred language, that has a layout.
static func automatic(
dictationLanguage: String?,
inputModes: [String],
preferredLanguages: [String]
) -> KeyboardLanguage {
if let dictationLanguage {
return matching(dictationLanguage) ?? .english
}
for identifier in inputModes + preferredLanguages {
if let language = matching(identifier) {
return language
}
}
return .english
}
}
+41 -16
View File
@@ -151,6 +151,7 @@ nonisolated struct KeyboardLayout: Equatable, Sendable {
let field: KeyboardFieldKind
let showsGlobe: Bool
let horizontalInset: CGFloat
let language: KeyboardLanguage
let metrics: KeyboardMetrics
let keys: [KeyboardKey]
@@ -160,7 +161,8 @@ nonisolated struct KeyboardLayout: Equatable, Sendable {
layer: KeyboardLayer = .letters,
field: KeyboardFieldKind = .text,
showsGlobe: Bool = false,
horizontalInset: CGFloat = 0
horizontalInset: CGFloat = 0,
language: KeyboardLanguage = .english
) {
self.width = width
self.orientation = orientation
@@ -168,6 +170,7 @@ nonisolated struct KeyboardLayout: Equatable, Sendable {
self.field = field
self.showsGlobe = showsGlobe
self.horizontalInset = max(0, horizontalInset)
self.language = language
let innerWidth = width - 2 * self.horizontalInset
let metrics = KeyboardMetrics.for(orientation, innerWidth: innerWidth)
self.metrics = metrics
@@ -177,7 +180,8 @@ nonisolated struct KeyboardLayout: Equatable, Sendable {
layer: layer,
field: field,
showsGlobe: showsGlobe,
inset: self.horizontalInset
inset: self.horizontalInset,
language: language
)
}
@@ -201,10 +205,22 @@ nonisolated struct KeyboardLayout: Equatable, Sendable {
CGRect(x: 0, y: metrics.stripArea, width: width, height: metrics.keysArea)
}
/// A letter key's width on a 10-key row. Shift, delete and the bottom row are sized
/// from it in every language, as Apple's are.
var letterWidth: CGFloat {
Self.letterWidth(innerWidth: width - 2 * horizontalInset, metrics: metrics)
}
/// The width of this page's character keys: `letterWidth`, or narrower where a row
/// holds 11 keys (Norwegian letters).
var characterWidth: CGFloat {
Self.letterWidth(
innerWidth: width - 2 * horizontalInset,
metrics: metrics,
columns: Self.columns(Self.characterRows(for: layer, language: language))
)
}
var showsBalloons: Bool { orientation == .portrait }
func key(_ kind: KeyKind) -> KeyboardKey? {
@@ -236,15 +252,20 @@ nonisolated struct KeyboardLayout: Equatable, Sendable {
// MARK: - Building
static func letterWidth(innerWidth: CGFloat, metrics: KeyboardMetrics) -> CGFloat {
(innerWidth - 2 * metrics.side - 9 * metrics.gap) / 10
static func letterWidth(innerWidth: CGFloat, metrics: KeyboardMetrics, columns: Int = 10) -> CGFloat {
(innerWidth - 2 * metrics.side - CGFloat(columns - 1) * metrics.gap) / CGFloat(columns)
}
static func characterRows(for layer: KeyboardLayer) -> [[String]] {
/// Keys across the widest of rows 1 and 2, never fewer than 10.
static func columns(_ rows: [[String]]) -> Int {
max(10, rows.prefix(2).map(\.count).max() ?? 10)
}
static func characterRows(for layer: KeyboardLayer, language: KeyboardLanguage = .english) -> [[String]] {
let punctuation = [".", ",", "?", "!", "'"]
switch layer {
case .letters:
return ["qwertyuiop", "asdfghjkl", "zxcvbnm"].map { $0.map(String.init) }
return language.letterRows
case .numbers:
return [
"1234567890".map(String.init),
@@ -266,12 +287,16 @@ nonisolated struct KeyboardLayout: Equatable, Sendable {
layer: KeyboardLayer,
field: KeyboardFieldKind,
showsGlobe: Bool,
inset: CGFloat
inset: CGFloat,
language: KeyboardLanguage
) -> [KeyboardKey] {
let innerWidth = width - 2 * inset
let lw = letterWidth(innerWidth: innerWidth, metrics: m)
let modW = lw * modifierWidthRatio
let rows = characterRows(for: layer)
let rows = characterRows(for: layer, language: language)
// Modifiers and the bottom row keep the 10-key sizes; an 11-key layout narrows only
// its character keys, and row 3's letters sit centred between shift and delete.
let referenceW = letterWidth(innerWidth: innerWidth, metrics: m)
let lw = letterWidth(innerWidth: innerWidth, metrics: m, columns: columns(rows))
let modW = referenceW * modifierWidthRatio
let left = inset + m.side
let right = width - inset - m.side
@@ -327,8 +352,8 @@ nonisolated struct KeyboardLayout: Equatable, Sendable {
// Row 4: 123/ABC, globe?, space, field extras (@ . / . /), Hex, return.
let isCompactBottomRow = showsGlobe || field != .text
let switchW = isCompactBottomRow ? modW : lw * bottomSwitchWidthRatio
let returnW = isCompactBottomRow ? (lw * compactReturnWidthRatio) : (lw * returnWidthRatio + m.gap)
let switchW = isCompactBottomRow ? modW : referenceW * bottomSwitchWidthRatio
let returnW = isCompactBottomRow ? (referenceW * compactReturnWidthRatio) : (referenceW * returnWidthRatio + m.gap)
var leading: [(KeyKind, CGFloat, String)] = [
layer == .letters
? (.layerSwitch(.numbers), switchW, "123")
@@ -346,11 +371,11 @@ nonisolated struct KeyboardLayout: Equatable, Sendable {
case .text:
break
case .url:
trailing.append((.character("."), lw, "."))
trailing.append((.character("/"), lw, "/"))
trailing.append((.character("."), referenceW, "."))
trailing.append((.character("/"), referenceW, "/"))
case .email:
trailing.append((.character("@"), lw, "@"))
trailing.append((.character("."), lw, "."))
trailing.append((.character("@"), referenceW, "@"))
trailing.append((.character("."), referenceW, "."))
}
if placement == .rightOfSpace {
trailing.append((.hex, modW, "hex"))
+89
View File
@@ -29,6 +29,7 @@ final class KeyboardViewModel: ObservableObject {
case none
case style
case returnApp
case language
}
enum StagePhase: Equatable {
@@ -59,6 +60,11 @@ final class KeyboardViewModel: ObservableObject {
@Published var returnAppID: String?
@Published var quickReturnApps: [HostApp] = HostAppCatalog.quickChips
@Published var activeTray: PickerTray = .none
/// Settings › Dictation language (nil for Automatic), what Automatic resolves to, and the
/// languages the keyboard's quick switch offers. Empty choices hide the switch.
@Published var dictationLanguage: String?
@Published var automaticLanguage: String?
@Published var languageChoices: [String] = []
// Non-published live telemetry read directly by `DotScopeHostView` / button tap handlers
// so 25–30 Hz polling never invalidates SwiftUI views mid-spring.
@@ -74,6 +80,8 @@ final class KeyboardViewModel: ObservableObject {
var onCancelTap: (() -> Void)?
var onStyleSelected: ((DictationStyle) -> Void)?
var onReturnAppSelected: ((String?) -> Void)?
/// A language picked in the keyboard's switch; nil is Automatic.
var onLanguageSelected: ((String?) -> Void)?
var onSpace: (() -> Void)?
var onDelete: (() -> Void)?
var onReturn: (() -> Void)?
@@ -84,6 +92,20 @@ final class KeyboardViewModel: ObservableObject {
HostAppCatalog.app(for: returnAppID)
}
/// The language dictation uses now, when known.
var effectiveLanguage: String? {
dictationLanguage ?? automaticLanguage
}
/// Its own short name ("Norsk", "English"), for the switch and the space bar.
var languageLabel: String? {
effectiveLanguage.map(DictationLanguageSwitch.nativeName(for:))
}
var showsLanguageSwitch: Bool {
hasFullAccess && !languageChoices.isEmpty
}
var dotScopePhase: DotScopePhase {
switch stagePhase {
case .idle:
@@ -593,6 +615,9 @@ struct KeyboardSurfaceView: View {
returnAppTrayBar(palette: p)
.hexTrayReveal(model.activeTray == .returnApp, reduceMotion: reduceMotion, offsetY: 3.5)
languageTrayBar(palette: p)
.hexTrayReveal(model.activeTray == .language, reduceMotion: reduceMotion, offsetY: 3.5)
}
.frame(height: 26)
}
@@ -621,6 +646,10 @@ struct KeyboardSurfaceView: View {
Spacer(minLength: 4)
if model.showsLanguageSwitch {
languageTrigger(palette: p)
}
styleTrigger(palette: p)
.hexTrayReveal(model.hasFullAccess, reduceMotion: reduceMotion, offsetY: 0)
}
@@ -761,6 +790,66 @@ struct KeyboardSurfaceView: View {
.accessibilityLabel("Style: \(model.style.label)")
}
private func languageTrigger(palette p: KeyboardChromePalette) -> some View {
let label = model.languageLabel ?? "Automatic"
return Button {
model.setActiveTray(.language, reduceMotion: reduceMotion)
} label: {
HStack(spacing: 4) {
Text(label)
.font(.system(size: 13, weight: .medium))
.lineLimit(1)
Image(systemName: "chevron.down")
.font(.system(size: 10, weight: .medium))
.foregroundStyle(p.textMuted)
}
.foregroundStyle(p.textMuted)
.padding(.horizontal, 8)
.frame(height: 26)
.contentShape(Capsule())
}
.buttonStyle(TopBarTriggerButtonStyle(palette: p))
.accessibilityLabel("Dictation language: \(label)")
}
private func languageTrayBar(palette p: KeyboardChromePalette) -> some View {
let names = DictationLanguageSwitch.nativeNames(for: model.languageChoices)
return HStack(spacing: 4) {
ScrollView(.horizontal, showsIndicators: false) {
HStack(spacing: 4) {
languageChip(id: nil, label: "Auto", palette: p)
ForEach(model.languageChoices, id: \.self) { id in
languageChip(id: id, label: names[id] ?? id, palette: p)
}
}
.padding(.leading, 2)
.padding(.trailing, 10)
}
.hexScrollEdgeEffectHidden()
.hexHorizontalEdgeFade(leading: 4, trailing: 14)
closeTrayButton(palette: p)
}
.frame(height: 26)
}
private func languageChip(id: String?, label: String, palette p: KeyboardChromePalette) -> some View {
let active = model.dictationLanguage == id
return Button {
model.onLanguageSelected?(id)
model.setActiveTray(.none, reduceMotion: reduceMotion)
} label: {
Text(label)
.font(.system(size: 13, weight: .medium))
.foregroundStyle(active ? p.textBase : p.textMuted)
.padding(.horizontal, 10)
.frame(height: 26)
.background(active ? p.chipActiveFill : p.chipIdleFill, in: Capsule())
}
.buttonStyle(.plain)
.accessibilityLabel(id == nil ? "Automatic language" : label)
}
private func returnChip(id: String?, label: String, palette p: KeyboardChromePalette) -> some View {
let active = model.returnAppID == id
return Button {
+24 -6
View File
@@ -32,20 +32,22 @@ nonisolated enum DictationStyle: String, Codable, CaseIterable, Sendable {
}
nonisolated enum TranscriptFormatter {
static func format(_ raw: String, style: DictationStyle) -> String {
/// `language` (BCP 47) picks the fillers a style trims; nil is English.
static func format(_ raw: String, style: DictationStyle, language: String? = nil) -> String {
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return "" }
let fillers = fillerPattern(for: language)
switch style {
case .verbatim:
return collapseHorizontalSpaces(in: trimmed)
case .polished:
let cleaned = stripStandaloneFillers(in: trimmed)
let cleaned = stripStandaloneFillers(in: trimmed, fillers: fillers)
return polishSentences(cleaned)
case .casual:
let cleaned = stripStandaloneFillers(in: trimmed)
let cleaned = stripStandaloneFillers(in: trimmed, fillers: fillers)
var lowered = collapseHorizontalSpaces(in: cleaned).lowercased()
// Keep standalone "i" and "i'm"/"i'll"/"i've"/"i'd" capitalized naturally, or pure casual lowercase?
// Pure texting style: lowercase start and drop single-sentence trailing period, keep ? and !.
@@ -56,7 +58,7 @@ nonisolated enum TranscriptFormatter {
return lowered.trimmingCharacters(in: .whitespacesAndNewlines)
case .notes:
let cleaned = stripStandaloneFillers(in: trimmed)
let cleaned = stripStandaloneFillers(in: trimmed, fillers: fillers)
return formatNotes(cleaned)
}
}
@@ -102,8 +104,24 @@ nonisolated enum TranscriptFormatter {
return ".,!?;:)]}”".contains(character)
}
private static func stripStandaloneFillers(in text: String) -> String {
let pattern = #"(?i)(?:^|(?<=[\s,]))(?:um+|uh+|er+|ah+)(?=(?:[\s,.!?]|$)),?\s*"#
/// The hesitation sounds a style trims, per language. English "er" is a word in
/// Norwegian, Danish, Swedish ("er" = is) and German, and "um" is one in Portuguese, so
/// each language trims only its own; a language without a list trims nothing.
static func fillerPattern(for language: String?) -> String? {
let code = language.flatMap { Locale(identifier: $0).language.languageCode?.identifier } ?? "en"
switch code {
case "en":
return "um+|uh+|er+|ah+"
case "nb", "nn", "no", "da", "sv":
return "e+h+m*|ø+h+m*|ö+h+m*|hm+"
default:
return nil
}
}
private static func stripStandaloneFillers(in text: String, fillers: String?) -> String {
guard let fillers else { return collapseHorizontalSpaces(in: text) }
let pattern = #"(?i)(?:^|(?<=[\s,]))(?:"# + fillers + #")(?=(?:[\s,.!?]|$)),?\s*"#
guard let regex = try? NSRegularExpression(pattern: pattern) else { return text }
let range = NSRange(text.startIndex..<text.endIndex, in: text)
let stripped = regex.stringByReplacingMatches(in: text, range: range, withTemplate: " ")
@@ -215,6 +215,7 @@ final class TypingLogger {
TypingLogField("horizontalInset", .pt(layout.horizontalInset)),
TypingLogField("globe", .bool(layout.showsGlobe)),
TypingLogField("hexPlacement", .string(HexKeyPlacement.current.rawValue)),
TypingLogField("language", .string(layout.language.rawValue)),
]
if current.shiftHeld {
fields.insert(TypingLogField("shiftHeld", .bool(true)), at: 4)
+80 -5
View File
@@ -3,9 +3,10 @@
**Status:** Internal TestFlight app (`HEXMobile`), custom keyboard extension
(`HEXKeyboard`), and Live Activity + Control Center widget extension
(`HEXWidgets`). The containing app records a generation-scoped temporary WAV,
transcribes on-device with Apple Speech (`SpeechTranscriber` on iOS 26+) or
FluidAudio Parakeet V2 (iOS 18+), and removes each capture's file after
inference, cancellation, or failure. During a warm session the keyboard
transcribes on-device with Apple Speech on iOS 26+ (`SpeechTranscriber`, or
`DictationTranscriber` for the languages it lacks, such as Norwegian, Danish,
Swedish and Hebrew) or FluidAudio Parakeet V2 (iOS 18+, English), and removes
each capture's file after inference, cancellation, or failure. During a warm session the keyboard
extension sends commands and inserts completed text over the shared App Group
(`group.anomaly.hex.mobile`). Otherwise it dictates by itself (in-keyboard
dictation, iOS 26+): iOS refuses to record from a keyboard extension
@@ -183,6 +184,60 @@ it on-device with Apple Speech, downloading the speech assets on first use
line and nothing else. Events are buffered on the main thread and encoded and
appended on a utility queue every 2 s. Nothing is sent. The schema is in
[`docs/typing-logs.md`](docs/typing-logs.md).
- **Dictation language (Settings › Dictation language, iOS 26+):** Automatic by
default: the first of the iPhone's preferred languages (then its region's
locale, then US English) that Apple Speech transcribes, so a Norwegian iPhone
dictates Norwegian with no setup; Nynorsk tries Bokmål, the only Norwegian
Apple has. The menu also lists every locale `SpeechTranscriber` or
`DictationTranscriber` supports, by name. `SpeechTranscriber` is used wherever
it has the language (as before); `DictationTranscriber` (preset
`longDictation`) covers the rest (`SpeechLanguagePicker`,
`AppleSpeechLanguages`, `AppleSpeechModule`). The setting lives in the App Group
(`settings.dictationLanguage`, nil for Automatic) so in-keyboard dictation uses
the same rule and the same downloaded assets; both record the language they
resolved (`settings.dictationLanguage.resolved`). Picking a language prepares
(and if needed downloads) its model at once, or after the dictation in flight.
An iPhone without `SpeechTranscriber` keeps English on Parakeet, as before.
The setup card's Speech model row names the language ("Norwegian Bokmål ·
Apple"). Steps that only know English follow the language: Clean up what I
say (an English tagger) skips other languages; Clean up with Apple
Intelligence runs only when the on-device model supports the language; the
styles trim English fillers (um, uh, er, ah) only in English, Nordic ones (eh,
ehm, øh, hm) in Norwegian, Danish and Swedish, and none elsewhere (Norwegian
"er" is "is", Portuguese "um" is "a"); translation's source is the dictation
language rather than a guess. Custom vocabulary works in any language.
- **Switching the dictation language in the keyboard (iOS 26+, Full Access):**
the dictation bar shows the language beside the style ("Norsk ⌄") and opens a
tray of chips; the full keyboard's Hex key menu has a Language row with its
own page, and the space bar names the language faintly ("English", "Norsk").
Both offer Automatic plus up to five languages: the one set, the ones picked
lately (`settings.dictationLanguage.recent`), then the iPhone's preferred
languages, each matched to one Apple Speech has (`DictationLanguageSwitch`;
the keyboard lists Apple's languages once per process and caches them in
`settings.dictationLanguage.supported`). Names are each language's own
("Español", "Deutsch", "עברית"), with a region code only where two share a
language. A pick writes the same App Group setting as Hex's menu, so
in-keyboard dictation uses it at once; a warm Hex session follows it within
100 ms and prepares the language between dictations, and a handoff or the
Action Button picks it up before recording.
- **Keyboard layouts (Experiments › Full keyboard › Keyboard layout):** English,
Norsk (`å` ending row 1, `ø æ` row 2), Español (`ñ` ending row 2) and
Deutsch (QWERTZ with `ü`, `ö ä`), as Apple's iOS keyboards lay them out
(`KeyboardLanguage`: letter rows, long-press accents, spelling dictionary).
Automatic follows the dictation language (Hebrew and other languages without a
layout get English); before any dictation language is known, the first iPhone
keyboard, then preferred language, that has a layout. Rows of 11 keys narrow
only the letters: shift, delete and the bottom row keep their English sizes,
and row 3's letters sit centred between shift and delete. Long-press accents
drop letters that have their own key (Norwegian A has no å or æ; Æ and Ø offer
Ä and Ö), put the language's own first (Spanish á, é, í, ó, ú, ñ, ª, º), and
German keeps ß on a held S. Autocorrect and suggestions use the layout's
`UITextChecker` dictionary (`nb`, `es`, `de`); where the iPhone has none
they stay off except your own text replacements, and the English contractions
and typo list (`i` → `I`) apply only to English. Neighbouring-key costs follow
the layout (on QWERTZ, Z sits beside T). Auto-capitalization is unchanged.
The 123 and #+= pages are English's in every language, and the accent lists
come from Apple's layouts as remembered, not checked against a device.
- **Resilient capture & expiry contract (`DictationController`):**
- Monotonic `jobGeneration`, retained `transcriptionTask`, and unique
per-capture WAV filenames ensure an expiring warm session never drops an
@@ -240,8 +295,28 @@ from `ci_scripts/ci_post_clone.sh`. A new model gets a new release tag and hash.
that condition through `ActiveAudioCapture`.
- **Apple Speech hint:** Custom vocabulary's contextual strings are best effort
and unverified on a physical iPhone. Apple documents contextual strings only
for `DictationTranscriber`, Hex sets them on `SpeechTranscriber`, and
Parakeet's batch path gets no hint.
for `DictationTranscriber`; Hex sets them on both modules, and Parakeet's
batch path gets no hint.
- **Dictation languages:** `SpeechLanguagePickerTests` cover the picking rule
and `DictationControllerTests` a language change during a dictation. The
simulator can't transcribe: `SpeechTranscriber.isAvailable` is false there
and `DictationTranscriber` fails with "Model not available", so
`AppleSpeechClipTests` (skipped unless `HEX_SPEECH_CLIPS` names a folder of
`<language>.wav` clips) reports that. The same code (`AppleSpeechTranscriber`,
`AppleSpeechModule`, the picker) built for macOS 26 transcribed `say` clips
word for word: German and Spanish through `SpeechTranscriber`, Norwegian
(`Nora`) and Hebrew (`Carmit`) through `DictationTranscriber`, which
downloaded each model in 11–16 s. `DictationTranscriber` returned Norwegian
and Hebrew without punctuation under every preset (the short-form presets
returned nothing), so Polished only capitalizes and ends the sentence.
Unverified on a physical iPhone.
- **Keyboard layouts:** `KeyboardLayoutTests` check every language's rows,
widths, hit frames (tiling the key area with no gaps or overlaps), balloons
and accent menus at 320–440 pt and in landscape; `AutocorrectEngineTests` and
`FullKeyboardEngineTests` check typing, dictionaries and English-only fixes.
Whether the simulator or an iPhone has a `UITextChecker` dictionary for
`nb` decides Norwegian autocorrect; how good those corrections are is
unmeasured.
- **Learn how I type:** Dictation edit inference (span location, word diff,
outcomes, the learning threshold, and applying fixes) is covered by unit
tests only. How much `documentContextBeforeInput` and
+8 -4
View File
@@ -81,7 +81,8 @@ The keyboard appeared (`trigger`: `appear`), the field changed while it showed
four preferred), `keyboardLanguage` (the keyboard's `primaryLanguage`),
`inputMode` (the field's input mode language), `fullAccess`.
- `settings`: `autocorrect` (Hex's), `key_popups`, `haptics`, `learnHowIType`, `cleanupTagger`,
`hexKeyPlacement`, `style` (dictation style).
`hexKeyPlacement`, `dictationLanguage` (BCP 47, or `automatic`), `keyboardLayout`
(`en`/`nb`/`es`/`de`, empty for the dictation bar), `style` (dictation style).
A `context` line follows at once with the text already in the field.
@@ -111,13 +112,15 @@ Written when the key layout or shift changes, and when a session starts.
(view size or safe area only), `layer_key` (123/#+=/ABC pressed),
`layer_auto` (space or return went back to letters), `layer_slide` (hold 123,
slide, let go), `shift_key`, `typed` (a one-shot shift turned off), `auto`
(auto-capitalization), `cancel`.
(auto-capitalization), `cancel`, `language` (the keyboard layout changed).
- `layer` (`letters`/`numbers`/`symbols`), `shift` (`off`/`on`/`locked`),
`shiftHeld` (only when true), `field` (`text`/`url`/`email`), `orientation`,
`width`, `height` (the keys' layout), `view` `[w, h]`, `safeArea`,
`horizontalInset` (landscape), `globe`, `hexPlacement`.
`horizontalInset` (landscape), `globe`, `hexPlacement`, `language`
(`en`/`nb`/`es`/`de`).
- The first time a hash appears in a day's file the line also has:
- `letterWidth`, `metrics` (`side`, `gap`, `rowGap`, `keyHeight`,
- `letterWidth` (a 10-key row's; 11-key Norwegian and German letters are
narrower, see `keys`), `metrics` (`side`, `gap`, `rowGap`, `keyHeight`,
`stripHeight`, `keysBottom`, `cornerRadius`),
- `strip`: the suggestion strip's rect, and `slots`: its three suggestion
slots (left, centre, right),
@@ -207,6 +210,7 @@ Taps that aren't text: `action`:
- `trackpad_start` (`id`), `trackpad_end` (`id`, `offset`: total characters);
- `hex_key` (`dictation`: `warm`, `keyboard`, `open_hex`), `hex_menu`, `style`
(`style`, `from`: `hex_menu` or `recording_panel`), `return_app` (`app`),
`dictation_language` (`language`: BCP 47 or `automatic`, from the Hex menu),
`dictation_stop`, `dictation_cancel`;
- `next_keyboard` (strip chip), `globe`, `undo_dictation` (`applied: false` when
the text had changed), `touches_cancelled` (`count`).