fix(importer): parse OMC frontmatter with CRLF line endings or a BOM

parse_markdown found the frontmatter fence only as "---\n" at byte zero
and "\n---\n" to close it. A wiki checked out on Windows with
core.autocrlf=true has CRLF throughout, and one saved as "UTF-8 with
BOM" opens with U+FEFF, so either missed the fence: kind, tier, tags and
pinned were dropped, the title came from the body's H1 instead of the
frontmatter, and the whole YAML block was imported as the top of the
page body.

The fence is now found the way ai_memory_wiki::markdown::parse finds it
since #663 and #908: past a leading BOM, with LF or CRLF fence lines,
and the YAML's trailing CR trimmed. The body keeps its own line endings.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
kevin9327
2026-09-29 07:35:19 +09:00
co-authored by Claude Opus 5.5
parent b46b1d18d7
commit e24d76786d
+40 -4
View File
@@ -1089,11 +1089,23 @@ struct ParsedMarkdown {
pinned: bool,
}
/// Split an OMC page into frontmatter fields and body. The fence is found
/// the way `ai_memory_wiki::markdown::parse` finds it: past a leading UTF-8
/// BOM, and with either LF or CRLF fence lines, since a wiki checked out on
/// Windows with `core.autocrlf=true` has CRLF throughout. Missing it dropped
/// kind, tier, tags and pin, and imported the YAML block as body text.
fn parse_markdown(input: &str) -> Result<ParsedMarkdown> {
let mut out = ParsedMarkdown::default();
let body = if let Some(rest) = input.strip_prefix("---\n") {
if let Some(end) = rest.find("\n---\n") {
let yaml = &rest[..end];
let input = input.strip_prefix('\u{FEFF}').unwrap_or(input);
let (rest, newline) = if let Some(rest) = input.strip_prefix("---\r\n") {
(Some(rest), "\r\n")
} else {
(input.strip_prefix("---\n"), "\n")
};
let close = format!("\n---{newline}");
let body = if let Some(rest) = rest {
if let Some(end) = rest.find(&close) {
let yaml = rest[..end].trim_end_matches('\r');
let value: serde_yaml::Value =
serde_yaml::from_str(yaml).context("parse YAML frontmatter")?;
if let Some(map) = value.as_mapping() {
@@ -1103,7 +1115,7 @@ fn parse_markdown(input: &str) -> Result<ParsedMarkdown> {
out.pinned = yaml_bool(map, "pinned").unwrap_or(false);
out.tags = yaml_tags(map);
}
rest[end + "\n---\n".len()..].to_owned()
rest[end + close.len()..].to_owned()
} else {
input.to_owned()
}
@@ -1454,6 +1466,30 @@ mod tests {
assert_eq!(parsed.body, "# Body\ntext");
}
/// A wiki checked out on Windows with `core.autocrlf=true`, or saved
/// there by an editor, has CRLF line endings; one saved as "UTF-8 with
/// BOM" opens with U+FEFF. Either used to miss the fence, so kind, tier,
/// tags and pin were dropped and the YAML block was imported as body.
#[test]
fn parses_omc_frontmatter_with_crlf_or_a_bom() {
let crlf = "---\r\ntitle: T\r\nkind: rule\r\ntier: procedural\r\ntags: [a, b]\r\npinned: true\r\n---\r\n# Body\r\ntext";
let parsed = parse_markdown(crlf).unwrap();
assert_eq!(parsed.title.as_deref(), Some("T"));
assert_eq!(parsed.kind.as_deref(), Some("rule"));
assert_eq!(parsed.tier.as_deref(), Some("procedural"));
assert_eq!(parsed.tags, vec!["a", "b"]);
assert!(parsed.pinned);
assert_eq!(
parsed.body, "# Body\r\ntext",
"the body keeps its line endings"
);
let bom = parse_markdown("\u{feff}---\ntier: procedural\n---\n# Body\ntext").unwrap();
assert_eq!(bom.tier.as_deref(), Some("procedural"));
assert_eq!(bom.title.as_deref(), Some("Body"));
assert_eq!(bom.body, "# Body\ntext");
}
#[test]
fn planning_rejects_unknown_tier_before_live_write() {
let td = tempdir().unwrap();