Files
jakeaturner a915cdcfc9 feat(AI): add opt-in relevance check for off-topic retrieval
The post-rerank relevance floor rejects noise but passes coherent
off-topic questions, and it gets worse as the library grows. Adding
Gutenberg prose, the Python tutorial and MDN pages to the eval corpus
took the off-topic leak rate from 22% to 67% at the 0.62 floor.

Score cutoff does not fix it. The weakest correct top hit scores 0.666
while "write me a haiku about autumn leaves" pulls Walden at 0.705.
A cosine floor and a top-vs-median contrast test both cost recall
faster than they cut leaks.

- Add rag.relevanceCheck (default off). After the floor, the tasks
  model, or the chat model when none is set, gives one grammar-constrained
  on_topic verdict for the whole retrieved set. Off topic means no
  context and no citations. Any failure keeps the chunks unjudged.
- Add a Settings > Models toggle recommending an 8B+ tasks model.
  On llama3:8b the check cuts off-topic leaks from 67% to 6% for about
  5 points of core recall@5 and ~0.6s per turn. On 3B models it rejects
  most valid questions, which is why it is opt-in.
- Add 6 distractor documents and an off_topic suite (18 unanswerable
  questions, 7 distractor controls) to the eval corpus.
- Report per-tag leak rates and leaked cases with top scores in
  eval:retrieval. Add --judge-model to measure the check and --dump to
  export pre-floor candidate pools for offline gate simulation.
- Pin the check off in eval:generation so a local setting cannot move
  scores.
- Record the distractor sweep on RAG_MIN_FINAL_SCORE. The 0.62 default
  stays: 0.65 halves leaks at no measured recall cost but leaves only
  0.016 of margin.

The corpus fingerprint changes to b4ebe5dce8699f5d. Retrieval reports
from before this commit are not comparable, and a new baseline is
committed.

Refs #1341
2026-09-29 15:54:41 -07:00

189 lines
7.1 KiB
TypeScript

import KVStore from '#models/kv_store'
import { BenchmarkService } from '#services/benchmark_service'
import { ContextWindowService } from '#services/context_window_service'
import { MapService } from '#services/map_service'
import { OllamaService } from '#services/ollama_service'
import { SystemService } from '#services/system_service'
import { getSettingSchema, updateSettingSchema, validateSettingValue } from '#validators/settings'
import { inject } from '@adonisjs/core'
import type { HttpContext } from '@adonisjs/core/http'
import env from '#start/env'
import { parseMinRelevance } from '../utils/misc.js'
import { isRelevanceCheckEnabled } from '../utils/relevance_judge.js'
import { parseResponseStyle } from '../utils/sampler.js'
import { RAG_MIN_FINAL_SCORE } from '../../constants/ollama.js'
@inject()
export default class SettingsController {
constructor(
private systemService: SystemService,
private mapService: MapService,
private benchmarkService: BenchmarkService,
private ollamaService: OllamaService,
private contextWindowService: ContextWindowService
) {}
async system({ inertia }: HttpContext) {
const systemInfo = await this.systemService.getSystemInfo()
return inertia.render('settings/system', {
system: {
info: systemInfo,
},
})
}
async apps({ inertia }: HttpContext) {
const services = await this.systemService.getServices({ installedOnly: false })
return inertia.render('settings/apps', {
system: {
services,
},
})
}
async legal({ inertia }: HttpContext) {
return inertia.render('settings/legal')
}
async support({ inertia }: HttpContext) {
return inertia.render('settings/support')
}
async maps({ inertia }: HttpContext) {
const baseAssetsCheck = await this.mapService.ensureBaseAssets()
const [regionFiles, worldBasemapExists] = await Promise.all([
this.mapService.listRegions(),
this.mapService.checkWorldBasemapExists(),
])
return inertia.render('settings/maps', {
maps: {
baseAssetsExist: baseAssetsCheck,
worldBasemapExists,
regionFiles: regionFiles.files,
},
})
}
async models({ inertia }: HttpContext) {
const availableModels = await this.ollamaService.getAvailableModels({
sort: 'pulls',
recommendedOnly: false,
query: null,
limit: 15,
})
const installedModels = await this.ollamaService.getModels().catch(() => [])
const chatSuggestionsEnabled = await KVStore.getValue('chat.suggestionsEnabled')
const aiAssistantCustomName = await KVStore.getValue('ai.assistantCustomName')
const remoteOllamaUrl = await KVStore.getValue('ai.remoteOllamaUrl')
const ollamaFlashAttention = await KVStore.getValue('ai.ollamaFlashAttention')
const autoThinking = await KVStore.getValue('ai.autoThinking')
const tasksModel = await KVStore.getValue('ai.tasksModel')
const ragEnabled = await KVStore.getValue('rag.enabled')
const contextWindow = await KVStore.getValue('ai.contextWindow')
const minRelevance = await KVStore.getValue('rag.minRelevance')
const relevanceCheck = await KVStore.getValue('rag.relevanceCheck')
const responseStyle = await KVStore.getValue('ai.responseStyle')
// Resolved window per installed model, so the setting shows what "Auto"
// actually produced rather than leaving the user to guess. Best-effort:
// a model whose metadata can't be read simply doesn't get a badge.
const resolvedContextWindows: Record<string, number> = {}
await Promise.all(
(installedModels || []).map(async (model) => {
try {
resolvedContextWindows[model.name] = await this.contextWindowService.windowFor(model.name)
} catch {
/* leave unset */
}
})
)
return inertia.render('settings/models', {
models: {
availableModels: availableModels?.models || [],
installedModels: installedModels || [],
settings: {
chatSuggestionsEnabled: chatSuggestionsEnabled ?? false,
aiAssistantCustomName: aiAssistantCustomName ?? '',
remoteOllamaUrl: remoteOllamaUrl ?? '',
ollamaFlashAttention: ollamaFlashAttention ?? true,
autoThinking: autoThinking ?? false,
tasksModel: tasksModel ?? '',
ragEnabled: ragEnabled ?? true,
contextWindow: contextWindow ?? 'auto',
// Sent as the resolved number so the select can match an option
// without duplicating the "unset means the default" rule in the UI.
minRelevance: parseMinRelevance(minRelevance, RAG_MIN_FINAL_SCORE),
relevanceCheck: isRelevanceCheckEnabled(relevanceCheck),
// Resolved rather than raw, for the same reason: unset means 'auto',
// and the select shouldn't have to know that.
responseStyle: parseResponseStyle(responseStyle),
},
resolvedContextWindows,
},
})
}
async update({ inertia }: HttpContext) {
const updateInfo = await this.systemService.checkLatestVersion()
return inertia.render('settings/update', {
system: {
updateAvailable: updateInfo.updateAvailable,
latestVersion: updateInfo.latestVersion,
currentVersion: updateInfo.currentVersion,
},
})
}
async zim({ inertia }: HttpContext) {
return inertia.render('settings/zim/index')
}
async zimRemote({ inertia }: HttpContext) {
return inertia.render('settings/zim/remote-explorer')
}
async creatorPacks({ inertia }: HttpContext) {
return inertia.render('settings/creator-packs')
}
async benchmark({ inertia }: HttpContext) {
const latestResult = await this.benchmarkService.getLatestResult()
const status = this.benchmarkService.getStatus()
return inertia.render('settings/benchmark', {
benchmark: {
latestResult,
status: status.status,
currentBenchmarkId: status.benchmarkId,
},
})
}
async advanced({ inertia }: HttpContext) {
// When the env var is set it always takes precedence over the stored value,
// so surface that to the UI to disable the field and explain the override.
const envOverride = Boolean(env.get('INTERNET_STATUS_TEST_URL')?.trim())
const internetStatusTestUrl = await KVStore.getValue('system.internetStatusTestUrl')
return inertia.render('settings/advanced', {
advanced: {
internetStatusTestUrl: internetStatusTestUrl ?? '',
internetStatusTestUrlEnvOverride: envOverride,
},
})
}
async getSetting({ request, response }: HttpContext) {
const { key } = await getSettingSchema.validate({ key: request.qs().key });
const value = await KVStore.getValue(key);
return response.status(200).send({ key, value });
}
async updateSetting({ request, response }: HttpContext) {
const reqData = await request.validateUsing(updateSettingSchema)
const valueError = validateSettingValue(reqData.key, reqData.value)
if (valueError) {
return response.status(422).send({ success: false, message: valueError })
}
await this.systemService.updateSetting(reqData.key, reqData.value)
return response.status(200).send({ success: true, message: 'Setting updated successfully' })
}
}