mirror of
https://github.com/anomalyco/hex.git
synced 2026-11-22 16:38:03 +08:00
TestFlight (build 32, iOS 27.2.1): onboarding's practice field said the microphone wasn't allowed after it had been, until Skip for now. Cause (reproduced on an iOS 27.2 simulator): a keyboard process keeps the microphone permission it read when it started. AVAudioApplication, AVAudioSession and AVCaptureDevice all still read "undetermined" in it after Hex is allowed the mic, so a Hex keyboard opened before the grant (tried in another app first, then onboarding) hits KeyboardLocalDictation's guard and shows "Allow the microphone in Hex first" on the Try the keyboard page. The app and the try page had the right state; only the keyboard's own reading was stale. What changed - KeyboardBridge.appMicrophoneAllowed (App Group key app.microphoneAllowed): the app's own reading, written by KeyboardSetupMonitor.refresh (every second on onboarding and home, on becoming active, and after the mic prompt), only when it changes. - KeyboardLocalDictation.start accepts the mic when its own reading or the app's says granted; WebKit's capture still checks, and a NotAllowedError still shows the same message. The probe log notes app=granted|no. - Nothing removed. Onboarding is unchanged: #2058's no Allow button before the prompt, and Continue after a decline, stay as they are. #2058 made this easier to hit, not the cause: build 32 needed the mic for Continue, so it took a keyboard opened earlier; now declining leads to the try page with the keyboard open, and allowing it later in Settings left that keyboard stale too. Verified (anomastudio, iPhone 18 Pro Max simulator, iOS 27.2, fresh install, mic reset, Full Access via TCC): Hex keyboard tried in Spotlight first, then onboarding with the mic allowed at the system prompt. Before: same keyboard pid, mic=undetermined, "Allow the microphone in Hex first" on the try page. After: same pid, mic=undetermined app=granted, web capture started in the practice field. xcodebuild test: 267 tests, 0 failures, 1 skipped. Not verified: a physical iPhone (the simulator's WebKit captures without a TCC check and has no speech assets), or a keyboard that started with the mic denied and was allowed in Settings.
394 lines
12 KiB
Swift
394 lines
12 KiB
Swift
import Foundation
|
|
|
|
nonisolated enum KeyboardDictationState: String, Codable, Sendable {
|
|
case offline
|
|
case ready
|
|
case recording
|
|
case transcribing
|
|
case failed
|
|
}
|
|
|
|
nonisolated enum WarmSessionDuration: String, Codable, CaseIterable, Sendable {
|
|
case off
|
|
case fiveMinutes
|
|
case fifteenMinutes
|
|
case oneHour
|
|
|
|
var seconds: TimeInterval? {
|
|
switch self {
|
|
case .off: nil
|
|
case .fiveMinutes: 5 * 60
|
|
case .fifteenMinutes: 15 * 60
|
|
case .oneHour: 60 * 60
|
|
}
|
|
}
|
|
|
|
var shortLabel: String {
|
|
switch self {
|
|
case .off: "Off"
|
|
case .fiveMinutes: "5m"
|
|
case .fifteenMinutes: "15m"
|
|
case .oneHour: "1h"
|
|
}
|
|
}
|
|
|
|
var title: String {
|
|
switch self {
|
|
case .off: "One shot"
|
|
case .fiveMinutes: "5 minutes"
|
|
case .fifteenMinutes: "15 minutes"
|
|
case .oneHour: "1 hour"
|
|
}
|
|
}
|
|
}
|
|
|
|
nonisolated enum KeyboardCommandKind: String, Codable, Sendable {
|
|
case start
|
|
case stop
|
|
case cancel
|
|
}
|
|
|
|
nonisolated struct KeyboardCommand: Codable, Equatable, Sendable {
|
|
let id: String
|
|
let kind: KeyboardCommandKind
|
|
let jobID: String
|
|
let style: DictationStyle
|
|
let sentAt: TimeInterval
|
|
|
|
init(
|
|
id: String = UUID().uuidString,
|
|
kind: KeyboardCommandKind,
|
|
jobID: String,
|
|
style: DictationStyle = .polished,
|
|
sentAt: TimeInterval = Date().timeIntervalSince1970
|
|
) {
|
|
self.id = id
|
|
self.kind = kind
|
|
self.jobID = jobID
|
|
self.style = style
|
|
self.sentAt = sentAt
|
|
}
|
|
}
|
|
|
|
nonisolated enum KeyboardLaunchRequest: Equatable, Sendable {
|
|
/// Any cold-start recording handoff older than 10 seconds warms the session instead of surprise-recording.
|
|
static let handoffWindow: TimeInterval = 10.0
|
|
|
|
/// Opens Hex's keyboard setup screen without starting a session.
|
|
case setup
|
|
case startSession(requestedAt: TimeInterval?)
|
|
case startRecording(
|
|
jobID: String,
|
|
requestedAt: TimeInterval?,
|
|
style: DictationStyle?
|
|
)
|
|
|
|
init?(url: URL) {
|
|
guard url.scheme == "hex-dictation", url.host == "keyboard" else { return nil }
|
|
let components = URLComponents(url: url, resolvingAgainstBaseURL: false)
|
|
let query = components?.queryItems ?? []
|
|
|
|
func value(_ name: String) -> String? {
|
|
guard let v = query.first(where: { $0.name == name })?.value, !v.isEmpty else {
|
|
return nil
|
|
}
|
|
return v
|
|
}
|
|
|
|
let requestedAt = value("ts").flatMap(TimeInterval.init)
|
|
let style = value("style").flatMap(DictationStyle.init(rawValue:))
|
|
|
|
switch url.path {
|
|
case "/setup":
|
|
self = .setup
|
|
case "/start":
|
|
self = .startSession(requestedAt: requestedAt)
|
|
case "/record":
|
|
guard let jobID = value("job") else { return nil }
|
|
self = .startRecording(
|
|
jobID: jobID,
|
|
requestedAt: requestedAt,
|
|
style: style
|
|
)
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
var requestedAt: TimeInterval? {
|
|
switch self {
|
|
case .setup:
|
|
nil
|
|
case .startSession(let ts), .startRecording(_, let ts, _):
|
|
ts
|
|
}
|
|
}
|
|
|
|
func isHandoffExpired(at now: TimeInterval = Date().timeIntervalSince1970) -> Bool {
|
|
guard let requestedAt else { return false }
|
|
return now - requestedAt > Self.handoffWindow
|
|
}
|
|
|
|
var url: URL {
|
|
var components = URLComponents()
|
|
components.scheme = "hex-dictation"
|
|
components.host = "keyboard"
|
|
|
|
var items: [URLQueryItem] = []
|
|
switch self {
|
|
case .setup:
|
|
components.path = "/setup"
|
|
|
|
case .startSession(let requestedAt):
|
|
components.path = "/start"
|
|
if let requestedAt {
|
|
items.append(URLQueryItem(name: "ts", value: String(format: "%.3f", requestedAt)))
|
|
}
|
|
|
|
case .startRecording(let jobID, let requestedAt, let style):
|
|
components.path = "/record"
|
|
items.append(URLQueryItem(name: "job", value: jobID))
|
|
if let requestedAt {
|
|
items.append(URLQueryItem(name: "ts", value: String(format: "%.3f", requestedAt)))
|
|
}
|
|
if let style {
|
|
items.append(URLQueryItem(name: "style", value: style.rawValue))
|
|
}
|
|
}
|
|
|
|
if !items.isEmpty {
|
|
components.queryItems = items
|
|
}
|
|
return components.url!
|
|
}
|
|
}
|
|
|
|
nonisolated struct KeyboardSnapshot: Codable, Equatable, Sendable {
|
|
let state: KeyboardDictationState
|
|
let heartbeat: TimeInterval
|
|
let expiresAt: TimeInterval
|
|
let recordingStartedAt: TimeInterval?
|
|
let audioLevel: Float
|
|
let loudnessDB: Float?
|
|
let jobID: String?
|
|
let resultID: String?
|
|
let resultExpiresAt: TimeInterval?
|
|
let transcript: String?
|
|
let message: String?
|
|
/// The raw transcript behind `transcript`, before cleanup and formatting, sent only while
|
|
/// Learn how I type is on so the keyboard can tell misheard words from formatting.
|
|
/// Nil when it can't be learned from (translated) or the experiment is off.
|
|
let heard: String?
|
|
/// Record typing logs: what each step of the dictation made of it, sent only while
|
|
/// that experiment is on.
|
|
let stages: DictationStages?
|
|
|
|
init(
|
|
state: KeyboardDictationState,
|
|
heartbeat: TimeInterval,
|
|
expiresAt: TimeInterval,
|
|
recordingStartedAt: TimeInterval? = nil,
|
|
audioLevel: Float = 0,
|
|
loudnessDB: Float? = nil,
|
|
jobID: String? = nil,
|
|
resultID: String? = nil,
|
|
resultExpiresAt: TimeInterval? = nil,
|
|
transcript: String? = nil,
|
|
message: String? = nil,
|
|
heard: String? = nil,
|
|
stages: DictationStages? = nil
|
|
) {
|
|
self.state = state
|
|
self.heartbeat = heartbeat
|
|
self.expiresAt = expiresAt
|
|
self.recordingStartedAt = recordingStartedAt
|
|
self.audioLevel = audioLevel
|
|
self.loudnessDB = loudnessDB
|
|
self.jobID = jobID
|
|
self.resultID = resultID
|
|
self.resultExpiresAt = resultExpiresAt
|
|
self.transcript = transcript
|
|
self.message = message
|
|
self.heard = heard
|
|
self.stages = stages
|
|
}
|
|
|
|
var effectiveLoudnessDB: Float {
|
|
loudnessDB ?? DotScopeSimulation.loudness(fromNormalized: audioLevel)
|
|
}
|
|
|
|
static let offline = KeyboardSnapshot(
|
|
state: .offline,
|
|
heartbeat: 0,
|
|
expiresAt: 0
|
|
)
|
|
|
|
var isAvailable: Bool {
|
|
isAvailable(at: Date().timeIntervalSince1970)
|
|
}
|
|
|
|
/// True while the background warm session or an active capture/transcription heartbeat is live.
|
|
func isAvailable(at now: TimeInterval) -> Bool {
|
|
let heartbeatFresh = now - heartbeat < 3.0 && heartbeat > 0
|
|
guard heartbeatFresh else { return false }
|
|
if state == .recording || state == .transcribing {
|
|
return true
|
|
}
|
|
return expiresAt > now
|
|
}
|
|
|
|
/// Accepted jobs keep their completed transcript deliverable even after a one-shot or expiring session closes.
|
|
func hasDeliverableResult(at now: TimeInterval = Date().timeIntervalSince1970) -> Bool {
|
|
guard let resultID, !resultID.isEmpty,
|
|
let transcript, !transcript.isEmpty else {
|
|
return false
|
|
}
|
|
if let resultExpiresAt {
|
|
return resultExpiresAt > now
|
|
}
|
|
return true
|
|
}
|
|
|
|
func recordingElapsed(at now: TimeInterval = Date().timeIntervalSince1970) -> TimeInterval {
|
|
guard state == .recording, let recordingStartedAt else { return 0 }
|
|
return max(0, now - recordingStartedAt)
|
|
}
|
|
}
|
|
|
|
@MainActor
|
|
struct KeyboardBridge {
|
|
nonisolated static let appGroup = "group.anomaly.hex.mobile"
|
|
|
|
private enum Key {
|
|
static let command = "keyboard.command"
|
|
static let modelInstalled = "model.installed"
|
|
static let snapshot = "keyboard.snapshot"
|
|
static let warmDuration = "settings.warmDuration"
|
|
static let style = "settings.dictationStyle"
|
|
static let allowBluetoothMic = "settings.allowBluetoothMic"
|
|
static let lastInsertedResultID = "keyboard.lastInsertedResultID"
|
|
static let fullAccessSeenAt = "keyboard.fullAccessSeenAt"
|
|
static let appMicrophoneAllowed = "app.microphoneAllowed"
|
|
}
|
|
|
|
private let defaults: UserDefaults?
|
|
private let encoder = JSONEncoder()
|
|
private let decoder = JSONDecoder()
|
|
|
|
init(defaults: UserDefaults? = UserDefaults(suiteName: Self.appGroup)) {
|
|
self.defaults = defaults
|
|
}
|
|
|
|
func send(
|
|
_ kind: KeyboardCommandKind,
|
|
jobID: String,
|
|
style: DictationStyle? = nil,
|
|
sentAt: TimeInterval = Date().timeIntervalSince1970
|
|
) {
|
|
let command = KeyboardCommand(
|
|
kind: kind,
|
|
jobID: jobID,
|
|
style: style ?? self.style,
|
|
sentAt: sentAt
|
|
)
|
|
guard let data = try? encoder.encode(command) else { return }
|
|
defaults?.set(data, forKey: Key.command)
|
|
}
|
|
|
|
func latestCommand() -> KeyboardCommand? {
|
|
guard let data = defaults?.data(forKey: Key.command) else { return nil }
|
|
return try? decoder.decode(KeyboardCommand.self, from: data)
|
|
}
|
|
|
|
var isModelInstalled: Bool {
|
|
defaults?.bool(forKey: Key.modelInstalled) ?? false
|
|
}
|
|
|
|
func setModelInstalled(_ installed: Bool) {
|
|
defaults?.set(installed, forKey: Key.modelInstalled)
|
|
}
|
|
|
|
var warmDuration: WarmSessionDuration {
|
|
get {
|
|
guard let raw = defaults?.string(forKey: Key.warmDuration),
|
|
let parsed = WarmSessionDuration(rawValue: raw) else {
|
|
return .fifteenMinutes
|
|
}
|
|
return parsed
|
|
}
|
|
nonmutating set {
|
|
defaults?.set(newValue.rawValue, forKey: Key.warmDuration)
|
|
}
|
|
}
|
|
|
|
var style: DictationStyle {
|
|
get {
|
|
guard let raw = defaults?.string(forKey: Key.style),
|
|
let parsed = DictationStyle(rawValue: raw) else {
|
|
return .polished
|
|
}
|
|
return parsed
|
|
}
|
|
nonmutating set {
|
|
defaults?.set(newValue.rawValue, forKey: Key.style)
|
|
}
|
|
}
|
|
|
|
var allowBluetoothMic: Bool {
|
|
get {
|
|
defaults?.bool(forKey: Key.allowBluetoothMic) ?? false
|
|
}
|
|
nonmutating set {
|
|
defaults?.set(newValue, forKey: Key.allowBluetoothMic)
|
|
}
|
|
}
|
|
|
|
var lastInsertedResultID: String? {
|
|
get {
|
|
defaults?.string(forKey: Key.lastInsertedResultID)
|
|
}
|
|
nonmutating set {
|
|
defaults?.set(newValue, forKey: Key.lastInsertedResultID)
|
|
}
|
|
}
|
|
|
|
/// When the keyboard last ran with Full Access. Without Full Access the keyboard
|
|
/// cannot write to the App Group at all, so only the keyboard sets this.
|
|
var fullAccessSeenAt: TimeInterval? {
|
|
get {
|
|
let value = defaults?.double(forKey: Key.fullAccessSeenAt) ?? 0
|
|
return value > 0 ? value : nil
|
|
}
|
|
nonmutating set {
|
|
if let newValue {
|
|
defaults?.set(newValue, forKey: Key.fullAccessSeenAt)
|
|
} else {
|
|
defaults?.removeObject(forKey: Key.fullAccessSeenAt)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Whether Hex has the microphone, as the app last read it. Only the app sets this. A
|
|
/// keyboard process keeps the permission it read when it started (iOS doesn't update it
|
|
/// there), so a keyboard opened before Hex was allowed the microphone needs the app's word.
|
|
var appMicrophoneAllowed: Bool {
|
|
get {
|
|
defaults?.bool(forKey: Key.appMicrophoneAllowed) ?? false
|
|
}
|
|
nonmutating set {
|
|
guard newValue != appMicrophoneAllowed else { return }
|
|
defaults?.set(newValue, forKey: Key.appMicrophoneAllowed)
|
|
}
|
|
}
|
|
|
|
func publish(_ snapshot: KeyboardSnapshot) {
|
|
guard let data = try? encoder.encode(snapshot) else { return }
|
|
defaults?.set(data, forKey: Key.snapshot)
|
|
}
|
|
|
|
func snapshot() -> KeyboardSnapshot {
|
|
guard let data = defaults?.data(forKey: Key.snapshot) else { return .offline }
|
|
return (try? decoder.decode(KeyboardSnapshot.self, from: data)) ?? .offline
|
|
}
|
|
}
|