diff --git a/admin/app/controllers/zim_controller.ts b/admin/app/controllers/zim_controller.ts index aed0136b..3c0fb9f4 100644 --- a/admin/app/controllers/zim_controller.ts +++ b/admin/app/controllers/zim_controller.ts @@ -25,8 +25,12 @@ export default class ZimController { async listRemote({ request }: HttpContext) { const payload = await request.validateUsing(listRemoteZimValidator) - const { start = 0, count = 12, query } = payload - return await this.zimService.listRemote({ start, count, query }) + const { start = 0, count = 12, query, language } = payload + return await this.zimService.listRemote({ start, count, query, language }) + } + + async listCatalogLanguages({}: HttpContext) { + return { languages: await this.zimService.listCatalogLanguages() } } async downloadRemote({ request }: HttpContext) { diff --git a/admin/app/services/zim_service.ts b/admin/app/services/zim_service.ts index 75e761f2..e8809892 100644 --- a/admin/app/services/zim_service.ts +++ b/admin/app/services/zim_service.ts @@ -1,4 +1,5 @@ import { + CatalogLanguage, ListRemoteZimFilesResponse, RawRemoteZimFileEntry, RemoteZimFileEntry, @@ -77,10 +78,18 @@ export class ZimService { start, count, query, + language = 'eng', }: { start: number count: number query?: string + /** + * ISO-639-3 code to filter the catalog by, or `all` for no filter. Defaults to + * English, which is what this browser has always shown -- but it is now a default + * the user can change, not a hardcode. English is ~1,300 of the catalog's ~10,900 + * books, so the filter was hiding the large majority of the library. + */ + language?: string }): Promise { // Kiwix returns pages of content unaware of what the user has installed locally. When // the installed set is large, a single 12-item Kiwix page can come back with everything @@ -111,7 +120,8 @@ export class ZimService { params: { start: currentStart, count: KIWIX_PAGE_SIZE, - lang: 'eng', + // `all` means "no filter", which the catalog expresses by omitting `lang`. + ...(language && language !== 'all' ? { lang: language } : {}), ...(query ? { q: query } : {}), }, responseType: 'text', @@ -191,6 +201,62 @@ export class ZimService { } } + /** + * The languages the Kiwix catalog actually holds books in, newest count first. + * + * Sourced live from the catalog rather than a bundled ISO list, so the options are + * always exactly what can be browsed, and each carries a real book count. Titles are + * the catalog's own endonyms ("français", "中文"), which is what a reader scanning for + * their own language recognises. + * + * Requires internet, like the rest of this browser. Failures return an empty list + * rather than throwing: a missing language filter should degrade to "English only", + * never take down the page that lists the books. + */ + async listCatalogLanguages(): Promise { + const LANGUAGES_URL = 'https://opds.library.kiwix.org/catalog/v2/languages' + try { + const res = await axios.get(LANGUAGES_URL, { responseType: 'text', timeout: 15000 }) + const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '', + textNodeName: '#text', + }) + const parsed = parser.parse(res.data) + const rawEntries = parsed?.feed?.entry + ? Array.isArray(parsed.feed.entry) + ? parsed.feed.entry + : [parsed.feed.entry] + : [] + + const languages: CatalogLanguage[] = [] + for (const raw of rawEntries) { + if (!raw || typeof raw !== 'object') continue + // `dc:language` is the ISO-639-3 code; `thr:count` is how many books carry it. + const code = raw['dc:language'] + const label = raw.title + const bookCount = Number(raw['thr:count']) + if (typeof code !== 'string' || !code.trim()) continue + if (!Number.isFinite(bookCount) || bookCount <= 0) continue + languages.push({ + code: code.trim(), + label: typeof label === 'string' && label.trim() ? label.trim() : code.trim(), + book_count: bookCount, + }) + } + + languages.sort((a, b) => b.book_count - a.book_count) + return languages + } catch (error) { + logger.warn( + `[ZimService] Catalog language list unavailable: ${ + error instanceof Error ? error.message : error + }` + ) + return [] + } + } + async downloadRemote(url: string, metadata?: { title?: string; summary?: string; author?: string; size_bytes?: number }): Promise<{ filename: string; jobId?: string }> { const parsed = new URL(url) if (!parsed.pathname.endsWith('.zim')) { diff --git a/admin/app/validators/zim.ts b/admin/app/validators/zim.ts index 5463e527..6cc1efbd 100644 --- a/admin/app/validators/zim.ts +++ b/admin/app/validators/zim.ts @@ -5,6 +5,14 @@ export const listRemoteZimValidator = vine.compile( start: vine.number().min(0).optional(), count: vine.number().min(1).max(100).optional(), query: vine.string().optional(), + // An ISO-639-3 code (`eng`, `fra`, `zho`) or the literal `all`. Constrained to a + // short alphabetic token so this can never smuggle anything else into the upstream + // catalog query string. + language: vine + .string() + .trim() + .regex(/^(all|[a-z]{2,8})$/) + .optional(), }) ) diff --git a/admin/inertia/lib/api.ts b/admin/inertia/lib/api.ts index 788580d2..4c617cea 100644 --- a/admin/inertia/lib/api.ts +++ b/admin/inertia/lib/api.ts @@ -1,5 +1,9 @@ import axios, { AxiosError, AxiosInstance } from 'axios' -import { ListRemoteZimFilesResponse, ListZimFilesResponse } from '../../types/zim' +import { + ListCatalogLanguagesResponse, + ListRemoteZimFilesResponse, + ListZimFilesResponse, +} from '../../types/zim' import { ServiceSlim } from '../../types/services' import { FileEntry } from '../../types/files' import { AppAutoUpdateStatus, AutoUpdateStatus, CheckLatestVersionResult, ContentAutoUpdateStatus, SystemInformationResponse, SystemUpdateStatus } from '../../types/system' @@ -832,10 +836,12 @@ class API { start = 0, count = 12, query, + language, }: { start?: number count?: number query?: string + language?: string }) { return catchInternal(async () => { return await this.client.get('/zim/list-remote', { @@ -843,11 +849,20 @@ class API { start, count, query, + language, }, }) })() } + async listCatalogLanguages() { + return catchInternal(async () => { + const response = + await this.client.get('/zim/catalog-languages') + return response.data.languages + })() + } + async listCustomLibraries() { return catchInternal(async () => { const response = await this.client.get<{ id: number; name: string; base_url: string; is_default: boolean }[]>( diff --git a/admin/inertia/pages/settings/zim/remote-explorer.tsx b/admin/inertia/pages/settings/zim/remote-explorer.tsx index addd5646..05f3c7a1 100644 --- a/admin/inertia/pages/settings/zim/remote-explorer.tsx +++ b/admin/inertia/pages/settings/zim/remote-explorer.tsx @@ -45,6 +45,7 @@ import { ZimFileWithMetadata } from '../../../../types/zim' const CURATED_CATEGORIES_KEY = 'curated-categories' const WIKIPEDIA_STATE_KEY = 'wikipedia-state' const CUSTOM_LIBRARIES_KEY = 'custom-libraries' +const CATALOG_LANGUAGES_KEY = 'catalog-languages' const ZIM_FILES_KEY = 'zim-files' type CustomLibrary = { id: number; name: string; base_url: string; is_default: boolean } @@ -82,6 +83,15 @@ export default function ZimRemoteExplorer() { } catch {} return 'default' }) + // Catalog language filter - also persisted, since someone browsing in their own + // language wants that on every visit, not once per session. + const [language, setLanguage] = useState(() => { + try { + return localStorage.getItem('nomad:zim-library-language') || 'eng' + } catch { + return 'eng' + } + }) const [browseUrl, setBrowseUrl] = useState(null) const [breadcrumbs, setBreadcrumbs] = useState<{ name: string; url: string }[]>([]) const [manageModalOpen, setManageModalOpen] = useState(false) @@ -140,15 +150,31 @@ export default function ZimRemoteExplorer() { retry: false, }) + // The catalog's own language list, so the options are exactly what can be browsed and + // each carries a real book count. Returns [] when offline; the selector hides itself. + const { data: catalogLanguages } = useQuery({ + queryKey: [CATALOG_LANGUAGES_KEY], + queryFn: () => api.listCatalogLanguages(), + refetchOnWindowFocus: false, + staleTime: 1000 * 60 * 60, + }) + const { data, fetchNextPage, isFetching, isLoading } = useInfiniteQuery({ - queryKey: ['remote-zim-files', query], + // `language` is part of the key, so changing it resets pagination rather than + // appending a second language's pages onto the first one's. + queryKey: ['remote-zim-files', query, language], queryFn: async ({ pageParam = 0 }) => { // pageParam is an opaque Kiwix offset returned by the backend as `next_start`. // The backend accumulates across multiple upstream pages when needed (#731), so the // frontend can't derive the next offset from a 12-item page assumption. const start = typeof pageParam === 'number' ? pageParam : 0 - const res = await api.listRemoteZimFiles({ start, count: 12, query: query || undefined }) + const res = await api.listRemoteZimFiles({ + start, + count: 12, + query: query || undefined, + language, + }) if (!res) { throw new Error('Failed to fetch remote ZIM files.') } @@ -232,6 +258,15 @@ export default function ZimRemoteExplorer() { }, [customLibraries, selectedSource]) // When selecting a custom library, navigate to its root + const handleLanguageChange = (value: string) => { + // localStorage can throw (private mode, blocked site data) -- the filter itself must + // still work, it just won't be remembered next visit. + try { + localStorage.setItem('nomad:zim-library-language', value) + } catch {} + setLanguage(value) + } + const handleSourceChange = (value: string) => { localStorage.setItem('nomad:zim-library-source', value) if (value === 'default') { @@ -592,7 +627,7 @@ export default function ZimRemoteExplorer() { {/* Default Kiwix library browser */} {selectedSource === 'default' && ( <> -
+
} /> + {/* Hidden entirely when the catalog list is unavailable (offline), rather + than shown as an empty dropdown the user cannot act on. */} + {catalogLanguages && catalogLanguages.length > 0 && ( +
+ + +
+ )}
data={flatData} diff --git a/admin/start/routes.ts b/admin/start/routes.ts index 1c84cefe..f1f4b21f 100644 --- a/admin/start/routes.ts +++ b/admin/start/routes.ts @@ -701,6 +701,10 @@ router tags: ['zim'], query: listRemoteZimValidator, }) + documented(router.get('/catalog-languages', [ZimController, 'listCatalogLanguages']), { + summary: 'List the languages the Kiwix catalog holds books in', + tags: ['zim'], + }) documented(router.get('/curated-categories', [ZimController, 'listCuratedCategories']), { summary: 'List curated ZIM categories', tags: ['zim'], diff --git a/admin/types/zim.ts b/admin/types/zim.ts index eb6e2b41..adf816db 100644 --- a/admin/types/zim.ts +++ b/admin/types/zim.ts @@ -19,6 +19,19 @@ export type ListRemoteZimFilesResponse = { next_start: number } +/** One selectable language in the Kiwix catalog, with how many books it holds. */ +export type CatalogLanguage = { + /** ISO-639-3 code as the catalog uses it, e.g. `eng`, `fra`, `zho`. */ + code: string + /** The catalog's own label, which is the endonym: "français", "中文". */ + label: string + book_count: number +} + +export type ListCatalogLanguagesResponse = { + languages: CatalogLanguage[] +} + export type RawRemoteZimFileEntry = { 'id': string 'title': string