mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-02 07:34:45 +08:00
feat(ocsf): configurable schema version for SIEM backward compatibility (#2717)
* feat(ocsf): configurable schema version for SIEM backward compatibility Add a gateway-configurable OCSF schema version target that downgrades JSONL output for SIEMs that only support older schema versions. AWS Security Lake requires v1.1.0, Splunk CIM Add-On targets v1.1-v1.3. The downgrade filter strips profile-gated fields (ai_model, container, observation_point_id), removes unknown profiles from metadata.profiles, rewrites metadata.version, and adds an unmapped.downgraded_from breadcrumb so auditors can distinguish "no model involved" from "model attribution stripped." Supported target versions (1.1, 1.3) are enforced by an allow-list in the settings registry. Invalid values are rejected with a clear error. The setting flows to sandboxes via the settings bundle and takes effect on the next poll cycle. The shorthand log output is unaffected. Closes #2662 Signed-off-by: Adel Zaalouk <zanetworker@gmail.com> Signed-off-by: Adel Zaalouk <azaalouk@redhat.com> * fix(ocsf): align downgrade with schema version Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --------- Signed-off-by: Adel Zaalouk <zanetworker@gmail.com> Signed-off-by: Adel Zaalouk <azaalouk@redhat.com> Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> Co-authored-by: John Myers <9696606+johntmyers@users.noreply.github.com>
This commit is contained in:
co-authored by
John Myers
parent
86a107fbf3
commit
48a8a4bf09
@@ -106,6 +106,14 @@ pub const PROPOSAL_APPROVAL_MODE_KEY: &str = "proposal_approval_mode";
|
||||
/// fail-closes on unknown persisted values for defense in depth.
|
||||
pub const PROPOSAL_APPROVAL_MODE_VALUES: &[&str] = &["manual", "auto"];
|
||||
|
||||
/// Allowed values for `ocsf_schema_version`.
|
||||
///
|
||||
/// Only versions with actual downgrade transforms in
|
||||
/// `openshell_ocsf::format::downgrade` are accepted. Empty string disables
|
||||
/// downgrade (equivalent to unsetting the key). Malformed or unsupported
|
||||
/// versions (e.g. `"banana"`, `"1.6"`) are rejected at configure time.
|
||||
pub const OCSF_SCHEMA_VERSION_VALUES: &[&str] = &["", "1.1", "1.3"];
|
||||
|
||||
pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[
|
||||
// When true the sandbox writes OCSF v1.8.0 JSONL records to
|
||||
// `/var/log/openshell-ocsf*.log` (daily rotation, 3 files) in addition
|
||||
@@ -115,6 +123,14 @@ pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[
|
||||
kind: SettingValueKind::Bool,
|
||||
allowed_string_values: None,
|
||||
},
|
||||
// Target OCSF schema version for JSONL downgrade. When set (e.g. "1.1"
|
||||
// or "1.3"), the JSONL layer strips fields and profiles that don't exist
|
||||
// in the target version. Empty or unset means no downgrade.
|
||||
RegisteredSetting {
|
||||
key: "ocsf_schema_version",
|
||||
kind: SettingValueKind::String,
|
||||
allowed_string_values: Some(OCSF_SCHEMA_VERSION_VALUES),
|
||||
},
|
||||
// Sandbox-level opt-in for the agent-driven policy proposal surface.
|
||||
// See AGENT_POLICY_PROPOSALS_ENABLED_KEY for details. Defaults to false.
|
||||
RegisteredSetting {
|
||||
@@ -160,8 +176,9 @@ pub fn parse_bool_like(raw: &str) -> Option<bool> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
PROPOSAL_APPROVAL_MODE_KEY, PROPOSAL_APPROVAL_MODE_VALUES, REGISTERED_SETTINGS,
|
||||
RegisteredSetting, SettingValueKind, parse_bool_like, registered_keys_csv, setting_for_key,
|
||||
OCSF_SCHEMA_VERSION_VALUES, PROPOSAL_APPROVAL_MODE_KEY, PROPOSAL_APPROVAL_MODE_VALUES,
|
||||
REGISTERED_SETTINGS, RegisteredSetting, SettingValueKind, parse_bool_like,
|
||||
registered_keys_csv, setting_for_key,
|
||||
};
|
||||
|
||||
#[test]
|
||||
@@ -226,6 +243,36 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// ---- ocsf_schema_version validation ----
|
||||
|
||||
#[test]
|
||||
fn ocsf_schema_version_accepts_supported_versions() {
|
||||
let setting = setting_for_key("ocsf_schema_version")
|
||||
.expect("ocsf_schema_version should be registered");
|
||||
assert_eq!(setting.kind, SettingValueKind::String);
|
||||
assert_eq!(
|
||||
setting.allowed_string_values,
|
||||
Some(OCSF_SCHEMA_VERSION_VALUES)
|
||||
);
|
||||
assert!(setting.validate_string_value("").is_ok());
|
||||
assert!(setting.validate_string_value("1.1").is_ok());
|
||||
assert!(setting.validate_string_value("1.3").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ocsf_schema_version_rejects_malformed_and_unsupported() {
|
||||
let setting = setting_for_key("ocsf_schema_version")
|
||||
.expect("ocsf_schema_version should be registered");
|
||||
for bad in [
|
||||
"banana", "1.6", "1.5", "1.7", "1.7.0", "1.1.0", "2.0", " 1.1", "1.1 ", "v1.1",
|
||||
] {
|
||||
let err = setting
|
||||
.validate_string_value(bad)
|
||||
.expect_err(&format!("expected '{bad}' to be rejected"));
|
||||
assert_eq!(err, OCSF_SCHEMA_VERSION_VALUES);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- parse_bool_like ----
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! OCSF schema version downgrade filter.
|
||||
//!
|
||||
//! Transforms serialized OCSF JSON events to conform to older schema versions
|
||||
//! by stripping fields and profiles that don't exist in the target version.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
/// Fields to strip when downgrading to v1.3.0 or earlier.
|
||||
const STRIP_FOR_V1_3: &[&str] = &["ai_model", "container", "observation_point_id"];
|
||||
|
||||
/// Profile names to remove from `metadata.profiles` when downgrading to v1.3.0 or earlier.
|
||||
const STRIP_PROFILES_V1_3: &[&str] = &["ai_operation", "container"];
|
||||
|
||||
/// Downgrade a serialized OCSF event to the target schema version.
|
||||
///
|
||||
/// Modifies the JSON in place: strips fields that don't exist in the target
|
||||
/// version, removes unknown profile names from `metadata.profiles`, and
|
||||
/// rewrites `metadata.version` to match.
|
||||
///
|
||||
/// Returns `true` if the event was modified, `false` if no changes were needed
|
||||
/// (target is current version or newer).
|
||||
pub fn downgrade_event(event: &mut Value, target_version: &str) -> bool {
|
||||
let target = parse_version(target_version);
|
||||
let v1_3 = (1, 3, 0);
|
||||
|
||||
if target >= parse_version(crate::OCSF_VERSION) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(obj) = event.as_object_mut() else {
|
||||
return false;
|
||||
};
|
||||
|
||||
let mut modified = false;
|
||||
|
||||
if target <= v1_3 {
|
||||
for field in STRIP_FOR_V1_3 {
|
||||
if obj.remove(*field).is_some() {
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(profiles) = obj
|
||||
.get_mut("metadata")
|
||||
.and_then(Value::as_object_mut)
|
||||
.and_then(|m| m.get_mut("profiles"))
|
||||
.and_then(Value::as_array_mut)
|
||||
{
|
||||
let before = profiles.len();
|
||||
profiles.retain(|p| !p.as_str().is_some_and(|s| STRIP_PROFILES_V1_3.contains(&s)));
|
||||
if profiles.len() != before {
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(metadata) = obj.get_mut("metadata").and_then(Value::as_object_mut) {
|
||||
let original_version = metadata
|
||||
.get("version")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or(crate::OCSF_VERSION)
|
||||
.to_string();
|
||||
metadata.insert(
|
||||
"version".to_string(),
|
||||
Value::String(target_version.to_string()),
|
||||
);
|
||||
modified = true;
|
||||
|
||||
let unmapped = obj
|
||||
.entry("unmapped")
|
||||
.or_insert_with(|| Value::Object(serde_json::Map::new()));
|
||||
if let Some(u) = unmapped.as_object_mut() {
|
||||
u.insert(
|
||||
"downgraded_from".to_string(),
|
||||
Value::String(original_version),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
modified
|
||||
}
|
||||
|
||||
fn parse_version(v: &str) -> (u32, u32, u32) {
|
||||
let parts: Vec<u32> = v.split('.').filter_map(|s| s.parse().ok()).collect();
|
||||
(
|
||||
parts.first().copied().unwrap_or(0),
|
||||
parts.get(1).copied().unwrap_or(0),
|
||||
parts.get(2).copied().unwrap_or(0),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn test_event() -> Value {
|
||||
serde_json::json!({
|
||||
"class_uid": 4002,
|
||||
"class_name": "HTTP Activity",
|
||||
"time": 1_234_567_890,
|
||||
"severity_id": 1,
|
||||
"metadata": {
|
||||
"version": crate::OCSF_VERSION,
|
||||
"profiles": ["security_control", "network_proxy", "container", "host"]
|
||||
},
|
||||
"device": {"hostname": "sandbox-1"},
|
||||
"container": {"name": "test-sandbox"},
|
||||
"observation_point_id": 2,
|
||||
"unmapped": {"key": "value"}
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_downgrade_to_v1_3_strips_fields() {
|
||||
let mut event = test_event();
|
||||
let modified = downgrade_event(&mut event, "1.3.0");
|
||||
|
||||
assert!(modified);
|
||||
assert!(event.get("container").is_none());
|
||||
assert!(event.get("observation_point_id").is_none());
|
||||
assert!(event.get("device").is_some());
|
||||
assert!(event.get("unmapped").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_downgrade_to_v1_1_strips_fields() {
|
||||
let mut event = test_event();
|
||||
let modified = downgrade_event(&mut event, "1.1.0");
|
||||
|
||||
assert!(modified);
|
||||
assert!(event.get("container").is_none());
|
||||
assert!(event.get("observation_point_id").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_downgrade_strips_profiles() {
|
||||
let mut event = test_event();
|
||||
downgrade_event(&mut event, "1.3.0");
|
||||
|
||||
let profiles = event["metadata"]["profiles"].as_array().unwrap();
|
||||
assert!(!profiles.iter().any(|p| p == "container"));
|
||||
assert!(profiles.iter().any(|p| p == "security_control"));
|
||||
assert!(profiles.iter().any(|p| p == "host"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_downgrade_rewrites_version() {
|
||||
let mut event = test_event();
|
||||
downgrade_event(&mut event, "1.1.0");
|
||||
|
||||
assert_eq!(event["metadata"]["version"], "1.1.0");
|
||||
assert_eq!(event["unmapped"]["downgraded_from"], crate::OCSF_VERSION);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_downgrade_for_current_version() {
|
||||
let mut event = test_event();
|
||||
let modified = downgrade_event(&mut event, crate::OCSF_VERSION);
|
||||
|
||||
assert!(!modified);
|
||||
assert_eq!(event["metadata"]["version"], crate::OCSF_VERSION);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_downgrade_for_newer_version() {
|
||||
let mut event = test_event();
|
||||
let modified = downgrade_event(&mut event, "1.9.0");
|
||||
|
||||
assert!(!modified);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_downgrade_strips_ai_model_when_present() {
|
||||
let mut event = serde_json::json!({
|
||||
"class_uid": 6003,
|
||||
"metadata": {
|
||||
"version": "1.8.0",
|
||||
"profiles": ["container", "host", "ai_operation"]
|
||||
},
|
||||
"ai_model": {"name": "claude-3-haiku", "ai_provider": "anthropic"},
|
||||
"unmapped": {"latency_ms": 701}
|
||||
});
|
||||
let modified = downgrade_event(&mut event, "1.3.0");
|
||||
|
||||
assert!(modified);
|
||||
assert!(event.get("ai_model").is_none());
|
||||
assert!(
|
||||
!event["metadata"]["profiles"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|p| p == "ai_operation")
|
||||
);
|
||||
assert_eq!(event["metadata"]["version"], "1.3.0");
|
||||
assert_eq!(event["unmapped"]["downgraded_from"], "1.8.0");
|
||||
assert_eq!(event["unmapped"]["latency_ms"], 701);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_downgrade_creates_unmapped_when_absent() {
|
||||
let mut event = serde_json::json!({
|
||||
"class_uid": 4001,
|
||||
"metadata": {
|
||||
"version": crate::OCSF_VERSION,
|
||||
"profiles": ["container"]
|
||||
},
|
||||
"container": {"name": "sandbox-1"}
|
||||
});
|
||||
let modified = downgrade_event(&mut event, "1.1.0");
|
||||
|
||||
assert!(modified);
|
||||
assert_eq!(event["unmapped"]["downgraded_from"], crate::OCSF_VERSION);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_downgrade_rewrites_version_without_strippable_fields() {
|
||||
let mut event = serde_json::json!({
|
||||
"class_uid": 0,
|
||||
"metadata": {
|
||||
"version": crate::OCSF_VERSION,
|
||||
"profiles": ["host"]
|
||||
}
|
||||
});
|
||||
|
||||
let modified = downgrade_event(&mut event, "1.3.0");
|
||||
|
||||
assert!(modified);
|
||||
assert_eq!(event["metadata"]["version"], "1.3.0");
|
||||
assert_eq!(event["unmapped"]["downgraded_from"], crate::OCSF_VERSION);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_downgrade_omits_breadcrumb() {
|
||||
let mut event = test_event();
|
||||
downgrade_event(&mut event, crate::OCSF_VERSION);
|
||||
|
||||
assert!(
|
||||
event
|
||||
.get("unmapped")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|u| u.get("downgraded_from"))
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -3,5 +3,6 @@
|
||||
|
||||
//! OCSF event formatters: shorthand (human-readable) and JSONL.
|
||||
|
||||
pub mod downgrade;
|
||||
pub mod jsonl;
|
||||
pub mod shorthand;
|
||||
|
||||
@@ -12,6 +12,7 @@ use tracing::Subscriber;
|
||||
use tracing_subscriber::Layer;
|
||||
use tracing_subscriber::layer::Context;
|
||||
|
||||
use crate::format::downgrade::downgrade_event;
|
||||
use crate::tracing_layers::event_bridge::{OCSF_TARGET, clone_current_event};
|
||||
|
||||
/// A tracing `Layer` that intercepts OCSF events and writes JSONL output.
|
||||
@@ -23,9 +24,15 @@ use crate::tracing_layers::event_bridge::{OCSF_TARGET, clone_current_event};
|
||||
/// `false`, the layer short-circuits without writing. This allows the sandbox
|
||||
/// to hot-toggle OCSF JSONL output at runtime via the `ocsf_json_enabled`
|
||||
/// setting without rebuilding the subscriber.
|
||||
///
|
||||
/// An optional target schema version can be set via
|
||||
/// [`with_target_version`](Self::with_target_version). When set, events are
|
||||
/// downgraded to the target version before writing (stripping fields and
|
||||
/// profiles that don't exist in older schema versions).
|
||||
pub struct OcsfJsonlLayer<W: Write + Send + 'static> {
|
||||
writer: Mutex<W>,
|
||||
enabled: Option<Arc<AtomicBool>>,
|
||||
target_version: Option<Arc<Mutex<String>>>,
|
||||
}
|
||||
|
||||
impl<W: Write + Send + 'static> OcsfJsonlLayer<W> {
|
||||
@@ -35,6 +42,7 @@ impl<W: Write + Send + 'static> OcsfJsonlLayer<W> {
|
||||
Self {
|
||||
writer: Mutex::new(writer),
|
||||
enabled: None,
|
||||
target_version: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,6 +55,16 @@ impl<W: Write + Send + 'static> OcsfJsonlLayer<W> {
|
||||
self.enabled = Some(flag);
|
||||
self
|
||||
}
|
||||
|
||||
/// Attach a shared target schema version for downgrade filtering.
|
||||
///
|
||||
/// When set, events are downgraded to the target version before writing.
|
||||
/// The version can be changed at runtime via the shared mutex.
|
||||
#[must_use]
|
||||
pub fn with_target_version(mut self, version: Arc<Mutex<String>>) -> Self {
|
||||
self.target_version = Some(version);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, W> Layer<S> for OcsfJsonlLayer<W>
|
||||
@@ -67,9 +85,29 @@ where
|
||||
}
|
||||
|
||||
if let Some(ocsf_event) = clone_current_event()
|
||||
&& let Ok(line) = ocsf_event.to_json_line()
|
||||
&& let Ok(mut w) = self.writer.lock()
|
||||
{
|
||||
let line = if let Some(ref target) = self.target_version
|
||||
&& let Ok(version) = target.lock()
|
||||
&& !version.is_empty()
|
||||
{
|
||||
let Ok(mut json) = serde_json::to_value(&ocsf_event) else {
|
||||
return;
|
||||
};
|
||||
downgrade_event(&mut json, &version);
|
||||
match serde_json::to_string(&json) {
|
||||
Ok(mut s) => {
|
||||
s.push('\n');
|
||||
s
|
||||
}
|
||||
Err(_) => return,
|
||||
}
|
||||
} else {
|
||||
match ocsf_event.to_json_line() {
|
||||
Ok(l) => l,
|
||||
Err(_) => return,
|
||||
}
|
||||
};
|
||||
let _ = w.write_all(line.as_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,6 +124,7 @@ pub async fn run_sandbox(
|
||||
_health_port: u16,
|
||||
inference_routes: Option<String>,
|
||||
ocsf_enabled: Arc<AtomicBool>,
|
||||
ocsf_schema_version: Arc<std::sync::Mutex<String>>,
|
||||
network_enabled: bool,
|
||||
process_enabled: bool,
|
||||
upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs,
|
||||
@@ -744,6 +745,7 @@ pub async fn run_sandbox(
|
||||
let poll_endpoint = endpoint.to_string();
|
||||
let poll_engine = engine.clone();
|
||||
let poll_ocsf_enabled = ocsf_enabled.clone();
|
||||
let poll_ocsf_schema_version = ocsf_schema_version.clone();
|
||||
let poll_pid = entrypoint_pid.clone();
|
||||
let poll_provider_credentials = provider_credentials.clone();
|
||||
let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone());
|
||||
@@ -759,6 +761,7 @@ pub async fn run_sandbox(
|
||||
entrypoint_pid: poll_pid,
|
||||
interval_secs: poll_interval_secs,
|
||||
ocsf_enabled: poll_ocsf_enabled,
|
||||
ocsf_schema_version: poll_ocsf_schema_version,
|
||||
provider_credentials: poll_provider_credentials,
|
||||
policy_local_ctx: poll_policy_local,
|
||||
agent_proposals: agent_proposals.clone(),
|
||||
@@ -3339,6 +3342,7 @@ struct PolicyPollLoopContext {
|
||||
entrypoint_pid: Arc<AtomicU32>,
|
||||
interval_secs: u64,
|
||||
ocsf_enabled: Arc<AtomicBool>,
|
||||
ocsf_schema_version: Arc<std::sync::Mutex<String>>,
|
||||
provider_credentials: ProviderCredentialState,
|
||||
policy_local_ctx: Option<Arc<openshell_supervisor_network::policy_local::PolicyLocalContext>>,
|
||||
agent_proposals: AgentProposals,
|
||||
@@ -3781,6 +3785,7 @@ async fn run_policy_poll_loop_with_client<C: PolicyGatewayClient>(
|
||||
match initial_poll_disposition(&ctx.loaded_policy_origin, &result) {
|
||||
InitialPollDisposition::Acknowledge(candidate) => {
|
||||
apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings);
|
||||
apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings);
|
||||
apply_agent_proposals_enabled(
|
||||
&ctx.agent_proposals,
|
||||
agent_proposals_enabled_from_settings(&result.settings),
|
||||
@@ -3808,6 +3813,7 @@ async fn run_policy_poll_loop_with_client<C: PolicyGatewayClient>(
|
||||
InitialPollDisposition::Reconcile => pending_result = Some(result),
|
||||
InitialPollDisposition::TrackOnly => {
|
||||
apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings);
|
||||
apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings);
|
||||
apply_agent_proposals_enabled(
|
||||
&ctx.agent_proposals,
|
||||
agent_proposals_enabled_from_settings(&result.settings),
|
||||
@@ -4296,6 +4302,7 @@ async fn run_policy_poll_loop_with_client<C: PolicyGatewayClient>(
|
||||
|
||||
// Apply OCSF JSON toggle from the `ocsf_json_enabled` setting.
|
||||
apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings);
|
||||
apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings);
|
||||
|
||||
// Apply the agent-proposals feature toggle. On a false→true transition
|
||||
// we lazily install the skill so a sandbox that started with the flag
|
||||
@@ -4350,6 +4357,37 @@ fn extract_bool_setting(
|
||||
})
|
||||
}
|
||||
|
||||
fn apply_ocsf_schema_version_setting(
|
||||
version: &std::sync::Mutex<String>,
|
||||
settings: &std::collections::HashMap<String, openshell_core::proto::EffectiveSetting>,
|
||||
) {
|
||||
let new_version = extract_string_setting(settings, "ocsf_schema_version").unwrap_or_default();
|
||||
if let Ok(mut current) = version.lock()
|
||||
&& *current != new_version
|
||||
{
|
||||
info!(
|
||||
ocsf_schema_version = %new_version,
|
||||
"OCSF schema version target changed"
|
||||
);
|
||||
*current = new_version;
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_string_setting(
|
||||
settings: &std::collections::HashMap<String, openshell_core::proto::EffectiveSetting>,
|
||||
key: &str,
|
||||
) -> Option<String> {
|
||||
use openshell_core::proto::setting_value;
|
||||
settings
|
||||
.get(key)
|
||||
.and_then(|es| es.value.as_ref())
|
||||
.and_then(|sv| sv.value.as_ref())
|
||||
.and_then(|v| match v {
|
||||
setting_value::Value::StringValue(s) => Some(s.clone()),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
fn agent_proposals_enabled_from_settings(
|
||||
settings: &std::collections::HashMap<String, openshell_core::proto::EffectiveSetting>,
|
||||
) -> bool {
|
||||
@@ -5045,6 +5083,7 @@ network_policies:
|
||||
entrypoint_pid: Arc::new(AtomicU32::new(0)),
|
||||
interval_secs: 0,
|
||||
ocsf_enabled: Arc::new(AtomicBool::new(false)),
|
||||
ocsf_schema_version: Arc::new(std::sync::Mutex::new(String::new())),
|
||||
provider_credentials: ProviderCredentialState::from_child_env_snapshot(
|
||||
0,
|
||||
std::collections::HashMap::new(),
|
||||
|
||||
@@ -602,6 +602,7 @@ fn main() -> Result<()> {
|
||||
// `ocsf_json_enabled` setting changes. The JSONL layer checks it
|
||||
// on each event and short-circuits when false.
|
||||
let ocsf_enabled = Arc::new(AtomicBool::new(false));
|
||||
let ocsf_schema_version = Arc::new(std::sync::Mutex::new(String::new()));
|
||||
|
||||
// Keep guards alive for the entire process. When a guard is dropped the
|
||||
// non-blocking writer flushes remaining logs.
|
||||
@@ -620,7 +621,9 @@ fn main() -> Result<()> {
|
||||
.ok()
|
||||
.map(|roller| {
|
||||
let (writer, guard) = tracing_appender::non_blocking(roller);
|
||||
let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone());
|
||||
let layer = OcsfJsonlLayer::new(writer)
|
||||
.with_enabled_flag(ocsf_enabled.clone())
|
||||
.with_target_version(ocsf_schema_version.clone());
|
||||
(layer, guard)
|
||||
});
|
||||
let (jsonl_layer, jsonl_guard) = match jsonl_logging {
|
||||
@@ -716,6 +719,7 @@ fn main() -> Result<()> {
|
||||
args.health_port,
|
||||
args.inference_routes,
|
||||
ocsf_enabled,
|
||||
ocsf_schema_version,
|
||||
args.mode.network,
|
||||
args.mode.process,
|
||||
upstream_proxy_args,
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 168 KiB |
@@ -3,7 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
title: "OCSF JSON Export"
|
||||
sidebar-title: "OCSF JSON Export"
|
||||
description: "How to enable full OCSF JSON logging for SIEM integration, compliance, and structured analysis."
|
||||
description: "How to enable full OCSF JSON logging for SIEM integration, compliance, and structured analysis. Includes schema version downgrade for AWS Security Lake, Splunk, and CrowdStrike compatibility."
|
||||
keywords: "Generative AI, Cybersecurity, OCSF, JSON, SIEM, Compliance, Observability"
|
||||
---
|
||||
|
||||
@@ -168,14 +168,66 @@ The shorthand log renders this as:
|
||||
OCSF API:INFERENCE [INFO] Success claude-haiku-4-5-20251001 via https://api.anthropic.com/v1 701ms [POST /v1/messages]
|
||||
```
|
||||
|
||||
## SIEM Schema Version Compatibility
|
||||
|
||||
OpenShell emits OCSF v1.8.0 events internally, but many SIEMs only support older schema versions. The `ocsf_schema_version` setting tells the JSONL layer to downgrade events before writing, stripping fields and profiles that don't exist in the target version.
|
||||
|
||||
Set the target version globally:
|
||||
|
||||
```shell
|
||||
openshell settings set --global --key ocsf_schema_version --value "1.1"
|
||||
```
|
||||
|
||||
Or per-sandbox:
|
||||
|
||||
```shell
|
||||
openshell settings set my-sandbox --key ocsf_schema_version --value "1.3"
|
||||
```
|
||||
|
||||
The setting takes effect on the next poll cycle, by default every 10 seconds. No sandbox restart is required.
|
||||
|
||||
Supported target versions are `1.1` and `1.3`. When set, the JSONL layer applies the following transformations:
|
||||
|
||||
- Strips fields added after the target version: `ai_model`, `container`, `observation_point_id`
|
||||
- Removes unknown profile names from `metadata.profiles`: `ai_operation`, `container`
|
||||
- Rewrites `metadata.version` to match the target version
|
||||
|
||||
The shorthand log output is unaffected. The internal event model stays at v1.8.0; only the serialized JSONL is transformed.
|
||||
|
||||
When unset or empty, no downgrade is applied and events are written at the current schema version.
|
||||
|
||||
| SIEM | Required OCSF Version | `ocsf_schema_version` Value |
|
||||
|---|---|---|
|
||||
| AWS Security Lake | v1.1.0 | `1.1` |
|
||||
| Splunk CIM Add-On | v1.1-v1.3 | `1.1` or `1.3` |
|
||||
| CrowdStrike FDR | v1.5.0 | Not yet supported |
|
||||
| Datadog Cloud SIEM | v1.5.0 (selectable) | Not yet supported |
|
||||
|
||||
<Warning>
|
||||
Downgrading is lossy. The `ai_model` field and `ai_operation` profile are stripped for v1.1 and v1.3 targets, so AI model attribution is not present in downgraded events. If your audit workflow requires knowing which model handled a request, use the full schema version (leave `ocsf_schema_version` unset). OpenShell supports OCSF v1.8.0 natively; upgrade your SIEM's OCSF schema if you need AI attribution alongside backward-compatible ingestion.
|
||||
</Warning>
|
||||
|
||||
Downgraded events include a `downgraded_from` marker in the `unmapped` object so auditors can distinguish "no model was involved" from "model attribution was stripped":
|
||||
|
||||
```json
|
||||
{
|
||||
"unmapped": {
|
||||
"downgraded_from": "1.8.0"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The core OCSF event structure (class UIDs, activity IDs, network and HTTP fields) is identical across v1.1 through v1.8. The stripped fields are all profile-gated additions.
|
||||
|
||||
## Integration with External Tools
|
||||
|
||||
The JSONL file can be shipped to any tool that accepts OCSF-formatted data:
|
||||
|
||||
| Tool | Integration Path |
|
||||
|---|---|
|
||||
| Splunk | Use the [Splunk OCSF Add-on](https://splunkbase.splunk.com/app/6943) to ingest OCSF JSONL files. |
|
||||
| Amazon Security Lake | OCSF is the native schema for Security Lake. |
|
||||
| Splunk | Use the [Splunk OCSF Add-on](https://splunkbase.splunk.com/app/6943) to ingest OCSF JSONL files. Set `ocsf_schema_version` to `1.3` for CIM Add-On compatibility. |
|
||||
| Amazon Security Lake | OCSF is the native schema for Security Lake. Set `ocsf_schema_version` to `1.1` for v1.1.0 compatibility. |
|
||||
| CrowdStrike FDR | Ship JSONL files to Falcon Data Replicator. v1.5 downgrade target is not yet supported. |
|
||||
| Elastic | Use Filebeat to ship JSONL files with the OCSF field mappings. |
|
||||
| Custom pipelines | Parse the JSONL file with `jq`, Python, or any JSON-capable tool. |
|
||||
|
||||
@@ -188,7 +240,7 @@ cat /var/log/openshell-ocsf.2026-04-01.log | \
|
||||
|
||||
## Relationship to Shorthand Logs
|
||||
|
||||
The shorthand format in `openshell.YYYY-MM-DD.log` and the JSON format in `openshell-ocsf.YYYY-MM-DD.log` are derived from the same OCSF events. The shorthand is a human-readable projection; the JSON is the complete record. Both are generated at the same time from the same event data.
|
||||
The shorthand format in `openshell.YYYY-MM-DD.log` and the JSON format in `openshell-ocsf.YYYY-MM-DD.log` are derived from the same OCSF events. The shorthand is a human-readable projection; the JSON is the full structured record when no schema downgrade is configured. When `ocsf_schema_version` is set, the JSON export is a lossy projection of the internal event model. Both formats are generated at the same time from the same event data.
|
||||
|
||||
The shorthand log is always active. The JSON export is opt-in through `ocsf_json_enabled`.
|
||||
|
||||
|
||||
@@ -770,6 +770,9 @@ openshell settings delete work-session --key ocsf_json_enabled
|
||||
|
||||
openshell settings get --global --json
|
||||
openshell settings set --global --key ocsf_json_enabled --value true
|
||||
|
||||
# OCSF schema version downgrade for SIEM compatibility (allowed: "1.1", "1.3")
|
||||
openshell settings set --global --key ocsf_schema_version --value "1.1"
|
||||
```
|
||||
|
||||
Global mutations prompt for confirmation. Use `--yes` only in reviewed automation.
|
||||
|
||||
Reference in New Issue
Block a user