mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-17 05:31:16 +08:00
373 lines
12 KiB
Python
373 lines
12 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Text content extraction functions."""
|
|
|
|
import re
|
|
from typing import Any
|
|
|
|
from docutils import nodes
|
|
from sphinx.environment import BuildEnvironment
|
|
from sphinx.util import logging
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Constants
|
|
MIN_SUBSTANTIAL_CONTENT_LENGTH = 50
|
|
MAX_SUMMARY_LENGTH = 300
|
|
MIN_KEYWORD_LENGTH = 3
|
|
MAX_KEYWORDS_RETURNED = 50
|
|
|
|
|
|
def extract_raw_markdown(env: BuildEnvironment, docname: str) -> str | None:
|
|
"""Extract raw markdown from source file."""
|
|
try:
|
|
source_path = env.doc2path(docname)
|
|
if not source_path or not source_path.exists():
|
|
return None
|
|
|
|
with open(source_path, encoding="utf-8") as f:
|
|
content = f.read()
|
|
|
|
# Remove frontmatter if present
|
|
if content.startswith("---"):
|
|
end_marker = content.find("\n---\n", 3)
|
|
if end_marker != -1:
|
|
content = content[end_marker + 5 :] # Skip the second ---\n
|
|
|
|
return content.strip()
|
|
|
|
except Exception as e: # noqa: BLE001
|
|
logger.debug(f"Could not extract raw markdown from {docname}: {e}")
|
|
return None
|
|
|
|
|
|
def extract_text_content(doctree: nodes.document) -> str:
|
|
"""Extract plain text content from document tree."""
|
|
text_parts = []
|
|
|
|
for node in doctree.traverse(nodes.Text):
|
|
text_parts.append(node.astext())
|
|
|
|
return " ".join(text_parts).strip()
|
|
|
|
|
|
def extract_clean_text_content(doctree: nodes.document, env: BuildEnvironment | None = None) -> str:
|
|
"""Extract clean text content, filtering out navigation elements.
|
|
|
|
Args:
|
|
doctree: The document tree to extract text from
|
|
env: Optional Sphinx environment for resolving link titles
|
|
|
|
Returns:
|
|
Cleaned text content suitable for search/LLM consumption
|
|
"""
|
|
text_parts = []
|
|
# Track nodes we've already processed (to avoid duplicate text from references)
|
|
processed_refs = set()
|
|
|
|
for node in doctree.traverse():
|
|
# Skip certain node types that aren't content
|
|
if isinstance(node, (nodes.target, nodes.substitution_definition)):
|
|
continue
|
|
|
|
# Skip toctree and other directive content
|
|
if hasattr(node, "tagname") and node.tagname in ["toctree", "index", "meta"]:
|
|
continue
|
|
|
|
# Handle reference nodes specially - extract and potentially improve link text
|
|
if isinstance(node, nodes.reference):
|
|
ref_id = id(node)
|
|
if ref_id not in processed_refs:
|
|
processed_refs.add(ref_id)
|
|
link_text = _get_improved_link_text(node, env)
|
|
if link_text:
|
|
text_parts.append(link_text)
|
|
continue
|
|
|
|
# Extract text from text nodes (but skip if inside a reference we already processed)
|
|
if isinstance(node, nodes.Text):
|
|
# Check if this text node is inside a reference
|
|
parent = node.parent
|
|
if isinstance(parent, nodes.reference) and id(parent) in processed_refs:
|
|
continue # Already handled by reference processing
|
|
|
|
text = node.astext().strip()
|
|
if text and not text.startswith("¶"): # Skip permalink symbols
|
|
text_parts.append(text)
|
|
|
|
# Join and clean up the text
|
|
full_text = " ".join(text_parts)
|
|
|
|
# Clean up whitespace
|
|
full_text = re.sub(r"\s+", " ", full_text)
|
|
|
|
return full_text.strip()
|
|
|
|
|
|
def _get_improved_link_text(node: nodes.reference, env: BuildEnvironment | None) -> str:
|
|
"""Get improved link text, resolving filenames to titles where possible."""
|
|
text = node.astext().strip()
|
|
if not text:
|
|
return ""
|
|
|
|
# If text doesn't look like a filename, use it as-is
|
|
if not _text_looks_like_filename(text):
|
|
return text
|
|
|
|
# Try to resolve to a better title
|
|
attrs = getattr(node, "attributes", {})
|
|
|
|
# Try refdoc first (target document for cross-references)
|
|
target_doc = attrs.get("refdoc", "")
|
|
|
|
# Try reftarget as fallback
|
|
if not target_doc:
|
|
target_doc = attrs.get("reftarget", "")
|
|
# Clean up the target
|
|
target_doc = target_doc.replace(".html", "").replace(".md", "").replace(".rst", "")
|
|
|
|
# Look up title in env.titles
|
|
if target_doc and env and hasattr(env, "titles") and target_doc in env.titles:
|
|
title_node = env.titles[target_doc]
|
|
if title_node:
|
|
resolved_title = title_node.astext().strip()
|
|
if resolved_title:
|
|
return resolved_title
|
|
|
|
# Fallback: humanize the filename
|
|
return _humanize_link_text(text)
|
|
|
|
|
|
def _text_looks_like_filename(text: str) -> bool:
|
|
"""Check if text looks like a filename rather than readable text."""
|
|
if not text:
|
|
return False
|
|
|
|
# Contains path separators
|
|
if "/" in text or "\\" in text:
|
|
return True
|
|
|
|
# Ends with file extension
|
|
if re.search(r"\.(md|rst|html|txt)$", text, re.IGNORECASE):
|
|
return True
|
|
|
|
# Single lowercase word (like "index", "readme", "configuration")
|
|
if " " not in text and text == text.lower() and len(text) > 2:
|
|
# But allow proper nouns that happen to be lowercase in context
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
def _humanize_link_text(text: str) -> str:
|
|
"""Convert filename-like text to human-readable form."""
|
|
# Get just the filename part
|
|
if "/" in text:
|
|
text = text.rsplit("/", 1)[-1]
|
|
|
|
# Remove extension
|
|
for ext in (".md", ".rst", ".html", ".txt"):
|
|
if text.endswith(ext):
|
|
text = text[: -len(ext)]
|
|
break
|
|
|
|
# Replace separators with spaces
|
|
text = text.replace("-", " ").replace("_", " ")
|
|
|
|
# Title case
|
|
return text.title()
|
|
|
|
|
|
def clean_text_for_llm(text: str) -> str:
|
|
"""Clean text content to make it more suitable for LLM processing and search indexing."""
|
|
if not text:
|
|
return ""
|
|
|
|
# Remove SVG content (common in documentation)
|
|
text = re.sub(r"<svg[^>]*>.*?</svg>", "", text, flags=re.DOTALL | re.IGNORECASE)
|
|
|
|
# Remove HTML comments
|
|
text = re.sub(r"<!--.*?-->", "", text, flags=re.DOTALL)
|
|
|
|
# Remove empty directive blocks (common MyST artifacts)
|
|
text = re.sub(r"^\s*```\{[^}]+\}\s*```\s*$", "", text, flags=re.MULTILINE)
|
|
|
|
# Remove toctree artifacts
|
|
text = re.sub(r"^\s*:caption:.*$", "", text, flags=re.MULTILINE)
|
|
text = re.sub(r"^\s*:hidden:\s*$", "", text, flags=re.MULTILINE)
|
|
text = re.sub(r"^\s*:glob:\s*$", "", text, flags=re.MULTILINE)
|
|
text = re.sub(r"^\s*:maxdepth:\s*\d+\s*$", "", text, flags=re.MULTILINE)
|
|
|
|
# Remove common MyST directive markers that aren't useful for search
|
|
text = re.sub(r"^\s*:::\{[^}]+\}\s*$", "", text, flags=re.MULTILINE)
|
|
text = re.sub(r"^\s*:::\s*$", "", text, flags=re.MULTILINE)
|
|
|
|
# Clean up code block language indicators
|
|
text = re.sub(r"```(\w+)\s*\n", "```\n", text)
|
|
|
|
# Remove excessive whitespace but preserve paragraph breaks
|
|
text = re.sub(r"\n\s*\n\s*\n+", "\n\n", text) # Multiple line breaks -> double
|
|
text = re.sub(r"[ \t]+", " ", text) # Multiple spaces/tabs -> single space
|
|
|
|
# Remove lines that are just punctuation or symbols
|
|
lines = text.split("\n")
|
|
cleaned_lines = []
|
|
for line in lines:
|
|
stripped = line.strip()
|
|
# Keep line if it has actual words (not just punctuation/symbols)
|
|
if stripped and re.search(r"[a-zA-Z0-9]", stripped):
|
|
# Remove standalone punctuation at start/end
|
|
stripped = re.sub(r"^[^\w\s]+\s*", "", stripped)
|
|
stripped = re.sub(r"\s*[^\w\s]+$", "", stripped)
|
|
if stripped:
|
|
cleaned_lines.append(stripped)
|
|
|
|
text = "\n".join(cleaned_lines)
|
|
|
|
# Final cleanup
|
|
return text.strip()
|
|
|
|
|
|
def extract_directive_content(directive_block: str) -> str:
|
|
"""Extract meaningful content from MyST directive blocks."""
|
|
if not directive_block:
|
|
return ""
|
|
|
|
# Remove the directive syntax but keep the content
|
|
lines = directive_block.split("\n")
|
|
content_lines = []
|
|
in_content = False
|
|
|
|
for line in lines:
|
|
# Skip directive header lines
|
|
if line.strip().startswith(":::") or line.strip().startswith("```{"):
|
|
in_content = True
|
|
continue
|
|
elif line.strip() == ":::" or line.strip() == "```":
|
|
continue
|
|
elif line.strip().startswith(":") and not in_content:
|
|
# Skip directive options
|
|
continue
|
|
|
|
# Include content lines
|
|
if in_content or not line.strip().startswith(":"):
|
|
content_lines.append(line)
|
|
|
|
return "\n".join(content_lines).strip()
|
|
|
|
|
|
def extract_summary(doctree: nodes.document) -> str:
|
|
"""Extract a summary from the document (first paragraph or section)."""
|
|
# Try to find the first substantial paragraph
|
|
for node in doctree.traverse(nodes.paragraph):
|
|
text = node.astext().strip()
|
|
if text and len(text) > MIN_SUBSTANTIAL_CONTENT_LENGTH: # Substantial content
|
|
# Clean and truncate
|
|
text = re.sub(r"\s+", " ", text)
|
|
if len(text) > MAX_SUMMARY_LENGTH:
|
|
text = text[:297] + "..."
|
|
return text
|
|
|
|
# Fallback: use first MAX_SUMMARY_LENGTH characters of any text
|
|
text = extract_text_content(doctree)
|
|
if text:
|
|
text = re.sub(r"\s+", " ", text)
|
|
if len(text) > MAX_SUMMARY_LENGTH:
|
|
text = text[:297] + "..."
|
|
return text
|
|
|
|
return ""
|
|
|
|
|
|
def extract_keywords(content: str, headings: list[dict[str, Any]]) -> list[str]:
|
|
"""Extract relevant keywords from content for search optimization."""
|
|
if not content:
|
|
return []
|
|
|
|
keywords = set()
|
|
|
|
# Add heading text as keywords
|
|
for heading in headings:
|
|
if "text" in heading:
|
|
# Split heading into words and add significant ones
|
|
words = re.findall(r"\b[a-zA-Z]{3,}\b", heading["text"].lower())
|
|
keywords.update(words)
|
|
|
|
# Extract technical terms (often capitalized or have specific patterns)
|
|
# API names, class names, function names, etc.
|
|
tech_terms = re.findall(r"\b[A-Z][a-zA-Z0-9_]*[a-z][a-zA-Z0-9_]*\b", content)
|
|
keywords.update(term.lower() for term in tech_terms)
|
|
|
|
# Extract quoted terms (often important concepts)
|
|
quoted_terms = re.findall(r'["`]([^"`]{3,20})["`]', content)
|
|
for term in quoted_terms:
|
|
if re.match(r"^[a-zA-Z][a-zA-Z0-9_\-\s]*$", term):
|
|
keywords.add(term.lower().strip())
|
|
|
|
# Extract common patterns for documentation keywords
|
|
# Configuration keys, file extensions, command names
|
|
config_keys = re.findall(r"\b[a-z_]+[a-z0-9_]*\s*[:=]", content)
|
|
keywords.update(key.rstrip(":=").strip() for key in config_keys)
|
|
|
|
# File extensions
|
|
extensions = re.findall(r"\.[a-z]{2,4}\b", content.lower())
|
|
keywords.update(ext.lstrip(".") for ext in extensions)
|
|
|
|
# Remove common stop words and very short terms
|
|
stop_words = {
|
|
"the",
|
|
"and",
|
|
"for",
|
|
"are",
|
|
"but",
|
|
"not",
|
|
"you",
|
|
"all",
|
|
"can",
|
|
"had",
|
|
"her",
|
|
"was",
|
|
"one",
|
|
"our",
|
|
"out",
|
|
"day",
|
|
"get",
|
|
"has",
|
|
"him",
|
|
"his",
|
|
"how",
|
|
"its",
|
|
"may",
|
|
"new",
|
|
"now",
|
|
"old",
|
|
"see",
|
|
"two",
|
|
"who",
|
|
"boy",
|
|
"did",
|
|
"she",
|
|
"use",
|
|
"way",
|
|
"what",
|
|
"when",
|
|
"will",
|
|
}
|
|
keywords = {kw for kw in keywords if len(kw) >= MIN_KEYWORD_LENGTH and kw not in stop_words}
|
|
|
|
# Return sorted list, limited to reasonable number
|
|
return sorted(keywords)[:MAX_KEYWORDS_RETURNED]
|