diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 00648f8b8..f106a1e7e 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -38,9 +38,6 @@ pub const OTLP_OCSF_ENDPOINT: &str = "OPENSHELL_OTLP_OCSF_ENDPOINT"; /// keeps the relay inactive and leaves the workload environment unchanged. pub const OTLP_AGENT_ENDPOINT: &str = "OPENSHELL_OTLP_AGENT_ENDPOINT"; -/// OCSF schema version that supervisors downgrade exported events to. -pub const OTLP_OCSF_SCHEMA_VERSION: &str = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION"; - /// Name of the gateway that launched the sandbox, reported as /// `openshell.gateway.name` on supervisor telemetry. pub const GATEWAY_NAME: &str = "OPENSHELL_GATEWAY_NAME"; @@ -49,19 +46,12 @@ pub const GATEWAY_NAME: &str = "OPENSHELL_GATEWAY_NAME"; #[derive(Clone, Debug, Eq, PartialEq)] pub struct OcsfOtlpExport { pub endpoint: String, - pub schema_version: Option, } impl OcsfOtlpExport { /// The supervisor environment variables for this export. pub fn environment(&self) -> Vec<(&'static str, String)> { - std::iter::once((OTLP_OCSF_ENDPOINT, self.endpoint.clone())) - .chain( - self.schema_version - .clone() - .map(|version| (OTLP_OCSF_SCHEMA_VERSION, version)), - ) - .collect() + vec![(OTLP_OCSF_ENDPOINT, self.endpoint.clone())] } } @@ -392,23 +382,14 @@ mod tests { use super::*; #[test] - fn ocsf_export_environment_carries_the_schema_version_only_when_set() { - let mut export = OcsfOtlpExport { + fn ocsf_export_environment_carries_the_endpoint() { + let export = OcsfOtlpExport { endpoint: "http://siem:4317".into(), - schema_version: None, }; assert_eq!( export.environment(), [(OTLP_OCSF_ENDPOINT, "http://siem:4317".to_string())] ); - export.schema_version = Some("1.3".into()); - assert_eq!( - export.environment(), - [ - (OTLP_OCSF_ENDPOINT, "http://siem:4317".to_string()), - (OTLP_OCSF_SCHEMA_VERSION, "1.3".to_string()), - ] - ); } #[test] diff --git a/crates/openshell-core/src/settings.rs b/crates/openshell-core/src/settings.rs index c3db1f731..f819d8ab3 100644 --- a/crates/openshell-core/src/settings.rs +++ b/crates/openshell-core/src/settings.rs @@ -106,14 +106,6 @@ pub const PROPOSAL_APPROVAL_MODE_KEY: &str = "proposal_approval_mode"; /// fail-closes on unknown persisted values for defense in depth. pub const PROPOSAL_APPROVAL_MODE_VALUES: &[&str] = &["manual", "auto"]; -/// Allowed values for `ocsf_schema_version`. -/// -/// Only versions with actual downgrade transforms in -/// `openshell_ocsf::format::downgrade` are accepted. Empty string disables -/// downgrade (equivalent to unsetting the key). Malformed or unsupported -/// versions (e.g. `"banana"`, `"1.6"`) are rejected at configure time. -pub const OCSF_SCHEMA_VERSION_VALUES: &[&str] = &["", "1.1", "1.3"]; - pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[ // When true the supervisor emits OCSF-JSON records to stderr and writes // JSONL to `/var/log/openshell-ocsf*.jsonl` when file logging is available. @@ -123,13 +115,12 @@ pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[ kind: SettingValueKind::Bool, allowed_string_values: None, }, - // Target OCSF schema version for JSONL downgrade. When set (e.g. "1.1" - // or "1.3"), the JSONL layer strips fields and profiles that don't exist - // in the target version. Empty or unset means no downgrade. + // Deprecated and ignored: OpenShell emits OCSF 1.8.0 only. Still + // registered so existing values load; the supervisor reports a set value. RegisteredSetting { key: "ocsf_schema_version", kind: SettingValueKind::String, - allowed_string_values: Some(OCSF_SCHEMA_VERSION_VALUES), + allowed_string_values: None, }, // Sandbox-level opt-in for the agent-driven policy proposal surface. // See AGENT_POLICY_PROPOSALS_ENABLED_KEY for details. Defaults to false. @@ -176,9 +167,8 @@ pub fn parse_bool_like(raw: &str) -> Option { #[cfg(test)] mod tests { use super::{ - OCSF_SCHEMA_VERSION_VALUES, PROPOSAL_APPROVAL_MODE_KEY, PROPOSAL_APPROVAL_MODE_VALUES, - REGISTERED_SETTINGS, RegisteredSetting, SettingValueKind, parse_bool_like, - registered_keys_csv, setting_for_key, + PROPOSAL_APPROVAL_MODE_KEY, PROPOSAL_APPROVAL_MODE_VALUES, REGISTERED_SETTINGS, + RegisteredSetting, SettingValueKind, parse_bool_like, registered_keys_csv, setting_for_key, }; #[test] @@ -243,33 +233,15 @@ mod tests { } } - // ---- ocsf_schema_version validation ---- + // ---- ocsf_schema_version (deprecated) ---- #[test] - fn ocsf_schema_version_accepts_supported_versions() { + fn deprecated_ocsf_schema_version_accepts_any_value() { let setting = setting_for_key("ocsf_schema_version") - .expect("ocsf_schema_version should be registered"); + .expect("ocsf_schema_version should stay registered while deprecated"); assert_eq!(setting.kind, SettingValueKind::String); - assert_eq!( - setting.allowed_string_values, - Some(OCSF_SCHEMA_VERSION_VALUES) - ); - assert!(setting.validate_string_value("").is_ok()); - assert!(setting.validate_string_value("1.1").is_ok()); - assert!(setting.validate_string_value("1.3").is_ok()); - } - - #[test] - fn ocsf_schema_version_rejects_malformed_and_unsupported() { - let setting = setting_for_key("ocsf_schema_version") - .expect("ocsf_schema_version should be registered"); - for bad in [ - "banana", "1.6", "1.5", "1.7", "1.7.0", "1.1.0", "2.0", " 1.1", "1.1 ", "v1.1", - ] { - let err = setting - .validate_string_value(bad) - .expect_err(&format!("expected '{bad}' to be rejected")); - assert_eq!(err, OCSF_SCHEMA_VERSION_VALUES); + for value in ["", "1.3", "anything"] { + assert!(setting.validate_string_value(value).is_ok(), "{value}"); } } diff --git a/crates/openshell-driver-docker/src/main.rs b/crates/openshell-driver-docker/src/main.rs index dc63c1e0f..da45b1284 100644 --- a/crates/openshell-driver-docker/src/main.rs +++ b/crates/openshell-driver-docker/src/main.rs @@ -51,12 +51,8 @@ struct Args { #[arg(long, env = "OPENSHELL_OTLP_OCSF_ENDPOINT")] otlp_ocsf_endpoint: Option, - /// OCSF schema version that supervisors downgrade exported events to. - #[arg( - long, - env = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION", - requires = "otlp_ocsf_endpoint" - )] + /// Deprecated and ignored: `OpenShell` emits OCSF 1.8.0 only. + #[arg(long, env = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION", hide = true)] otlp_ocsf_schema_version: Option, /// OTLP/gRPC collector endpoint that supervisors relay workload traces @@ -80,6 +76,11 @@ async fn main() -> Result<()> { log_level: &args.log_level, }, ); + if args.otlp_ocsf_schema_version.is_some() { + tracing::warn!( + "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION is deprecated and ignored; OpenShell emits OCSF 1.8.0" + ); + } let config_source = std::fs::read_to_string(&args.config).into_diagnostic()?; let mut docker_config: DockerComputeConfig = @@ -97,12 +98,10 @@ async fn main() -> Result<()> { docker_config.supervisor_otlp_endpoint = args.otlp_endpoint.clone(); docker_config.supervisor_otlp_agent_endpoint = args.otlp_agent_endpoint.clone(); docker_config.supervisor_gateway_name = args.gateway_name.clone(); - docker_config.supervisor_ocsf_export = args.otlp_ocsf_endpoint.clone().map(|endpoint| { - openshell_core::sandbox_env::OcsfOtlpExport { - endpoint, - schema_version: args.otlp_ocsf_schema_version.clone(), - } - }); + docker_config.supervisor_ocsf_export = args + .otlp_ocsf_endpoint + .clone() + .map(|endpoint| openshell_core::sandbox_env::OcsfOtlpExport { endpoint }); let driver = DockerComputeDriver::new(args.gateway_bind, &args.log_level, &docker_config, None) .await .into_diagnostic()?; diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 2eab3e5ee..7e99f37e2 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -1517,8 +1517,7 @@ fn docker_child_environment_strips_supervisor_control_keys() { #[test] fn supervisor_otlp_environment_carries_only_configured_signals() { use openshell_core::sandbox_env::{ - GATEWAY_NAME, OTLP_AGENT_ENDPOINT, OTLP_ENDPOINT, OTLP_OCSF_ENDPOINT, - OTLP_OCSF_SCHEMA_VERSION, OcsfOtlpExport, + GATEWAY_NAME, OTLP_AGENT_ENDPOINT, OTLP_ENDPOINT, OTLP_OCSF_ENDPOINT, OcsfOtlpExport, }; let mut config = runtime_config(); @@ -1533,14 +1532,10 @@ fn supervisor_otlp_environment_carries_only_configured_signals() { config.supervisor_otlp_endpoint = None; config.supervisor_ocsf_export = Some(OcsfOtlpExport { endpoint: "http://siem:4317".to_string(), - schema_version: Some("1.3".to_string()), }); assert_eq!( supervisor_otlp_environment(&config), - [ - format!("{OTLP_OCSF_ENDPOINT}=http://siem:4317"), - format!("{OTLP_OCSF_SCHEMA_VERSION}=1.3"), - ] + [format!("{OTLP_OCSF_ENDPOINT}=http://siem:4317")] ); config.supervisor_ocsf_export = None; diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 65be627dc..e5b12dd7e 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -8978,9 +8978,7 @@ mod tests { #[tokio::test] async fn supervisor_otlp_environment_carries_only_configured_signals() { - use openshell_core::sandbox_env::{ - OTLP_ENDPOINT, OTLP_OCSF_ENDPOINT, OTLP_OCSF_SCHEMA_VERSION, OcsfOtlpExport, - }; + use openshell_core::sandbox_env::{OTLP_ENDPOINT, OTLP_OCSF_ENDPOINT, OcsfOtlpExport}; let names = |config: KubernetesComputeConfig| { KubernetesComputeDriver::new_for_test(config) @@ -9002,16 +9000,12 @@ mod tests { let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig { supervisor_ocsf_export: Some(OcsfOtlpExport { endpoint: "http://siem:4317".into(), - schema_version: Some("1.1".into()), }), ..Default::default() }); assert_eq!( driver.supervisor_otlp_environment(), - [ - (OTLP_OCSF_ENDPOINT, "http://siem:4317".to_string()), - (OTLP_OCSF_SCHEMA_VERSION, "1.1".to_string()), - ] + [(OTLP_OCSF_ENDPOINT, "http://siem:4317".to_string())] ); let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig { diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 7803d1561..7326d904a 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -49,12 +49,8 @@ struct Args { #[arg(long, env = "OPENSHELL_OTLP_OCSF_ENDPOINT")] otlp_ocsf_endpoint: Option, - /// OCSF schema version that supervisors downgrade exported events to. - #[arg( - long, - env = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION", - requires = "otlp_ocsf_endpoint" - )] + /// Deprecated and ignored: `OpenShell` emits OCSF 1.8.0 only. + #[arg(long, env = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION", hide = true)] otlp_ocsf_schema_version: Option, /// OTLP/gRPC collector endpoint that supervisors relay workload traces @@ -248,6 +244,11 @@ async fn main() -> Result<()> { log_level: &args.log_level, }, ); + if args.otlp_ocsf_schema_version.is_some() { + tracing::warn!( + "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION is deprecated and ignored; OpenShell emits OCSF 1.8.0" + ); + } let managed_ssh_gateway_pod_selector = args .managed_ssh_gateway_pod_selector @@ -305,12 +306,10 @@ async fn main() -> Result<()> { supervisor_otlp_endpoint: args.otlp_endpoint.clone(), supervisor_otlp_agent_endpoint: args.otlp_agent_endpoint.clone(), supervisor_gateway_name: args.gateway_name.clone(), - supervisor_ocsf_export: args.otlp_ocsf_endpoint.clone().map(|endpoint| { - openshell_core::sandbox_env::OcsfOtlpExport { - endpoint, - schema_version: args.otlp_ocsf_schema_version.clone(), - } - }), + supervisor_ocsf_export: args + .otlp_ocsf_endpoint + .clone() + .map(|endpoint| openshell_core::sandbox_env::OcsfOtlpExport { endpoint }), ssh_socket_path: args.sandbox_ssh_socket_path, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 7964485d6..a63c58d5f 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -2928,7 +2928,6 @@ mod tests { let config = PodmanComputeConfig { supervisor_ocsf_export: Some(openshell_core::sandbox_env::OcsfOtlpExport { endpoint: "http://siem:4317".to_string(), - schema_version: Some("1.1".to_string()), }), ..test_config() }; @@ -2937,10 +2936,6 @@ mod tests { spec["env"][openshell_core::sandbox_env::OTLP_OCSF_ENDPOINT], "http://siem:4317" ); - assert_eq!( - spec["env"][openshell_core::sandbox_env::OTLP_OCSF_SCHEMA_VERSION], - "1.1" - ); assert!( spec["env"] .get(openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT) diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index fb0ff3181..00eb79c0d 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -109,7 +109,6 @@ fn select_supervisor_ocsf_export( config.supervisor_ocsf_export.clone().map(|export| { openshell_core::sandbox_env::OcsfOtlpExport { endpoint: supervisor_collector_endpoint(&export.endpoint, environment), - ..export } }) } @@ -3575,7 +3574,6 @@ mod tests { fn supervisor_ocsf_endpoint_uses_host_alias_for_loopback_on_podman_machine() { let export = |endpoint: &str| openshell_core::sandbox_env::OcsfOtlpExport { endpoint: endpoint.to_string(), - schema_version: Some("1.3".to_string()), }; let config = PodmanComputeConfig { supervisor_ocsf_export: Some(export("http://127.0.0.1:4317")), diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 8d22241a7..2d510e465 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -46,12 +46,8 @@ struct Args { #[arg(long, env = "OPENSHELL_OTLP_OCSF_ENDPOINT")] otlp_ocsf_endpoint: Option, - /// OCSF schema version that supervisors downgrade exported events to. - #[arg( - long, - env = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION", - requires = "otlp_ocsf_endpoint" - )] + /// Deprecated and ignored: `OpenShell` emits OCSF 1.8.0 only. + #[arg(long, env = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION", hide = true)] otlp_ocsf_schema_version: Option, /// OTLP/gRPC collector endpoint that supervisors relay workload traces @@ -231,6 +227,11 @@ async fn main() -> Result<()> { log_level: &args.log_level, }, ); + if args.otlp_ocsf_schema_version.is_some() { + tracing::warn!( + "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION is deprecated and ignored; OpenShell emits OCSF 1.8.0" + ); + } // External drivers have no gateway-local staged artifact. Do not accept // a path or silently emulate destination-CA delivery outside the gateway. @@ -245,12 +246,10 @@ async fn main() -> Result<()> { supervisor_otlp_endpoint: args.otlp_endpoint.clone(), supervisor_otlp_agent_endpoint: args.otlp_agent_endpoint.clone(), supervisor_gateway_name: args.gateway_name.clone(), - supervisor_ocsf_export: args.otlp_ocsf_endpoint.clone().map(|endpoint| { - openshell_core::sandbox_env::OcsfOtlpExport { - endpoint, - schema_version: args.otlp_ocsf_schema_version.clone(), - } - }), + supervisor_ocsf_export: args + .otlp_ocsf_endpoint + .clone() + .map(|endpoint| openshell_core::sandbox_env::OcsfOtlpExport { endpoint }), gateway_port: args.gateway_port, host_gateway_ip: args .host_gateway_ip diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index cba0caf12..0ab474377 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -104,12 +104,8 @@ struct Args { #[arg(long, env = "OPENSHELL_OTLP_OCSF_ENDPOINT")] otlp_ocsf_endpoint: Option, - /// OCSF schema version that supervisors downgrade exported events to. - #[arg( - long, - env = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION", - requires = "otlp_ocsf_endpoint" - )] + /// Deprecated and ignored: `OpenShell` emits OCSF 1.8.0 only. + #[arg(long, env = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION", hide = true)] otlp_ocsf_schema_version: Option, /// OTLP/gRPC collector endpoint that supervisors relay workload traces @@ -318,6 +314,11 @@ async fn main() -> Result<()> { log_level: &args.log_level, }, ); + if args.otlp_ocsf_schema_version.is_some() { + tracing::warn!( + "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION is deprecated and ignored; OpenShell emits OCSF 1.8.0" + ); + } let listen_mode = compute_driver_listen_mode(&args).map_err(|err| miette::miette!("{err}"))?; @@ -342,12 +343,10 @@ async fn main() -> Result<()> { supervisor_otlp_endpoint: args.otlp_endpoint.clone(), supervisor_otlp_agent_endpoint: args.otlp_agent_endpoint.clone(), supervisor_gateway_name: args.gateway_name.clone(), - supervisor_ocsf_export: args.otlp_ocsf_endpoint.clone().map(|endpoint| { - openshell_core::sandbox_env::OcsfOtlpExport { - endpoint, - schema_version: args.otlp_ocsf_schema_version.clone(), - } - }), + supervisor_ocsf_export: args + .otlp_ocsf_endpoint + .clone() + .map(|endpoint| openshell_core::sandbox_env::OcsfOtlpExport { endpoint }), state_dir: args.state_dir.clone(), launcher_bin: None, default_image: args.default_image.clone(), diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index a6a50774b..5dd475a7e 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -58,9 +58,6 @@ fn supervisor_ocsf_export( let ocsf = otlp?.ocsf.as_ref()?; Some(openshell_core::sandbox_env::OcsfOtlpExport { endpoint: ocsf.endpoint.clone(), - schema_version: ocsf - .schema_version - .map(|version| version.as_str().to_string()), }) } diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index 2cf677d1d..2f9ea5cc8 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -696,11 +696,6 @@ fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gat } if let Some(ocsf) = ocsf { command.arg("--otlp-ocsf-endpoint").arg(&ocsf.endpoint); - if let Some(version) = ocsf.schema_version { - command - .arg("--otlp-ocsf-schema-version") - .arg(version.as_str()); - } } let agent_endpoint = otlp_config.and_then(OtlpConfig::agent_endpoint); if let Some(endpoint) = agent_endpoint { @@ -808,9 +803,7 @@ mod tests { validate_vm_sandbox_identity, }; use openshell_core::{NetworkSupervisorTrustBundle, UpstreamProxyConfig}; - use openshell_server::config_file::{ - OcsfOtlpConfig, OcsfSchemaVersion, OtlpConfig, OtlpTracesConfig, - }; + use openshell_server::config_file::{OcsfOtlpConfig, OtlpConfig, OtlpTracesConfig}; use std::os::unix::fs::PermissionsExt; use std::os::unix::net::UnixListener as StdUnixListener; use std::path::PathBuf; @@ -894,7 +887,7 @@ mod tests { traces: None, ocsf: Some(OcsfOtlpConfig { endpoint: "http://siem.internal:4317".to_string(), - schema_version: Some(OcsfSchemaVersion::V1_3), + schema_version: None, }), agent: None, }), @@ -911,8 +904,6 @@ mod tests { [ "--otlp-ocsf-endpoint", "http://siem.internal:4317", - "--otlp-ocsf-schema-version", - "1.3", "--gateway-name", "production-us-west" ] diff --git a/crates/openshell-ocsf-validate/src/lib.rs b/crates/openshell-ocsf-validate/src/lib.rs index b867ca361..7c6d7bf49 100644 --- a/crates/openshell-ocsf-validate/src/lib.rs +++ b/crates/openshell-ocsf-validate/src/lib.rs @@ -379,9 +379,9 @@ mod tests { #[test] fn rejects_versions_without_vendored_schemas() { let mut event = base_event(); - event["metadata"]["version"] = json!("1.1.0"); + event["metadata"]["version"] = json!("2.0.0"); assert!( - matches!(validate_event(&event), Err(Error::UnsupportedVersion(found)) if found == "1.1.0") + matches!(validate_event(&event), Err(Error::UnsupportedVersion(found)) if found == "2.0.0") ); } diff --git a/crates/openshell-ocsf/src/format/downgrade.rs b/crates/openshell-ocsf/src/format/downgrade.rs deleted file mode 100644 index 06bd4edd2..000000000 --- a/crates/openshell-ocsf/src/format/downgrade.rs +++ /dev/null @@ -1,290 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! OCSF schema version downgrade filter. -//! -//! Transforms serialized OCSF JSON events to conform to older schema versions -//! by stripping fields and profiles that don't exist in the target version. - -use serde::{Deserialize, Serialize}; -use serde_json::Value; - -/// An older OCSF schema version that events can be downgraded to. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum OcsfSchemaVersion { - #[serde(rename = "1.1")] - V1_1, - #[serde(rename = "1.3")] - V1_3, -} - -impl OcsfSchemaVersion { - pub const fn as_str(self) -> &'static str { - match self { - Self::V1_1 => "1.1", - Self::V1_3 => "1.3", - } - } -} - -impl std::str::FromStr for OcsfSchemaVersion { - type Err = String; - - fn from_str(version: &str) -> Result { - match version { - "1.1" => Ok(Self::V1_1), - "1.3" => Ok(Self::V1_3), - other => Err(format!( - "unsupported OCSF schema version {other:?}; expected 1.1 or 1.3" - )), - } - } -} - -/// Fields to strip when downgrading to v1.3.0 or earlier. -const STRIP_FOR_V1_3: &[&str] = &["ai_model", "container", "observation_point_id"]; - -/// Profile names to remove from `metadata.profiles` when downgrading to v1.3.0 or earlier. -const STRIP_PROFILES_V1_3: &[&str] = &["ai_operation", "container"]; - -/// Downgrade a serialized OCSF event to the target schema version. -/// -/// Modifies the JSON in place: strips fields that don't exist in the target -/// version, removes unknown profile names from `metadata.profiles`, and -/// rewrites `metadata.version` to match. -/// -/// Returns `true` if the event was modified, `false` if no changes were needed -/// (target is current version or newer). -pub fn downgrade_event(event: &mut Value, target_version: &str) -> bool { - let target = parse_version(target_version); - let v1_3 = (1, 3, 0); - - if target >= parse_version(crate::OCSF_VERSION) { - return false; - } - - let Some(obj) = event.as_object_mut() else { - return false; - }; - - let mut modified = false; - - if target <= v1_3 { - for field in STRIP_FOR_V1_3 { - if obj.remove(*field).is_some() { - modified = true; - } - } - - if let Some(profiles) = obj - .get_mut("metadata") - .and_then(Value::as_object_mut) - .and_then(|m| m.get_mut("profiles")) - .and_then(Value::as_array_mut) - { - let before = profiles.len(); - profiles.retain(|p| !p.as_str().is_some_and(|s| STRIP_PROFILES_V1_3.contains(&s))); - if profiles.len() != before { - modified = true; - } - } - } - - if let Some(metadata) = obj.get_mut("metadata").and_then(Value::as_object_mut) { - let original_version = metadata - .get("version") - .and_then(Value::as_str) - .unwrap_or(crate::OCSF_VERSION) - .to_string(); - metadata.insert( - "version".to_string(), - Value::String(target_version.to_string()), - ); - modified = true; - - let unmapped = obj - .entry("unmapped") - .or_insert_with(|| Value::Object(serde_json::Map::new())); - if let Some(u) = unmapped.as_object_mut() { - u.insert( - "downgraded_from".to_string(), - Value::String(original_version), - ); - } - } - - modified -} - -fn parse_version(v: &str) -> (u32, u32, u32) { - let parts: Vec = v.split('.').filter_map(|s| s.parse().ok()).collect(); - ( - parts.first().copied().unwrap_or(0), - parts.get(1).copied().unwrap_or(0), - parts.get(2).copied().unwrap_or(0), - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn schema_versions_parse_from_their_names() { - for version in [OcsfSchemaVersion::V1_1, OcsfSchemaVersion::V1_3] { - assert_eq!(version.as_str().parse(), Ok(version)); - } - assert!("1.6".parse::().is_err()); - assert!("".parse::().is_err()); - } - - fn test_event() -> Value { - serde_json::json!({ - "class_uid": 4002, - "class_name": "HTTP Activity", - "time": 1_234_567_890, - "severity_id": 1, - "metadata": { - "version": crate::OCSF_VERSION, - "profiles": ["security_control", "network_proxy", "container", "host"] - }, - "device": {"hostname": "sandbox-1"}, - "container": {"name": "test-sandbox"}, - "observation_point_id": 2, - "unmapped": {"key": "value"} - }) - } - - #[test] - fn test_downgrade_to_v1_3_strips_fields() { - let mut event = test_event(); - let modified = downgrade_event(&mut event, "1.3.0"); - - assert!(modified); - assert!(event.get("container").is_none()); - assert!(event.get("observation_point_id").is_none()); - assert!(event.get("device").is_some()); - assert!(event.get("unmapped").is_some()); - } - - #[test] - fn test_downgrade_to_v1_1_strips_fields() { - let mut event = test_event(); - let modified = downgrade_event(&mut event, "1.1.0"); - - assert!(modified); - assert!(event.get("container").is_none()); - assert!(event.get("observation_point_id").is_none()); - } - - #[test] - fn test_downgrade_strips_profiles() { - let mut event = test_event(); - downgrade_event(&mut event, "1.3.0"); - - let profiles = event["metadata"]["profiles"].as_array().unwrap(); - assert!(!profiles.iter().any(|p| p == "container")); - assert!(profiles.iter().any(|p| p == "security_control")); - assert!(profiles.iter().any(|p| p == "host")); - } - - #[test] - fn test_downgrade_rewrites_version() { - let mut event = test_event(); - downgrade_event(&mut event, "1.1.0"); - - assert_eq!(event["metadata"]["version"], "1.1.0"); - assert_eq!(event["unmapped"]["downgraded_from"], crate::OCSF_VERSION); - } - - #[test] - fn test_no_downgrade_for_current_version() { - let mut event = test_event(); - let modified = downgrade_event(&mut event, crate::OCSF_VERSION); - - assert!(!modified); - assert_eq!(event["metadata"]["version"], crate::OCSF_VERSION); - } - - #[test] - fn test_no_downgrade_for_newer_version() { - let mut event = test_event(); - let modified = downgrade_event(&mut event, "1.9.0"); - - assert!(!modified); - } - - #[test] - fn test_downgrade_strips_ai_model_when_present() { - let mut event = serde_json::json!({ - "class_uid": 6003, - "metadata": { - "version": "1.8.0", - "profiles": ["container", "host", "ai_operation"] - }, - "ai_model": {"name": "claude-3-haiku", "ai_provider": "anthropic"}, - "unmapped": {"latency_ms": 701} - }); - let modified = downgrade_event(&mut event, "1.3.0"); - - assert!(modified); - assert!(event.get("ai_model").is_none()); - assert!( - !event["metadata"]["profiles"] - .as_array() - .unwrap() - .iter() - .any(|p| p == "ai_operation") - ); - assert_eq!(event["metadata"]["version"], "1.3.0"); - assert_eq!(event["unmapped"]["downgraded_from"], "1.8.0"); - assert_eq!(event["unmapped"]["latency_ms"], 701); - } - - #[test] - fn test_downgrade_creates_unmapped_when_absent() { - let mut event = serde_json::json!({ - "class_uid": 4001, - "metadata": { - "version": crate::OCSF_VERSION, - "profiles": ["container"] - }, - "container": {"name": "sandbox-1"} - }); - let modified = downgrade_event(&mut event, "1.1.0"); - - assert!(modified); - assert_eq!(event["unmapped"]["downgraded_from"], crate::OCSF_VERSION); - } - - #[test] - fn test_downgrade_rewrites_version_without_strippable_fields() { - let mut event = serde_json::json!({ - "class_uid": 0, - "metadata": { - "version": crate::OCSF_VERSION, - "profiles": ["host"] - } - }); - - let modified = downgrade_event(&mut event, "1.3.0"); - - assert!(modified); - assert_eq!(event["metadata"]["version"], "1.3.0"); - assert_eq!(event["unmapped"]["downgraded_from"], crate::OCSF_VERSION); - } - - #[test] - fn test_no_downgrade_omits_breadcrumb() { - let mut event = test_event(); - downgrade_event(&mut event, crate::OCSF_VERSION); - - assert!( - event - .get("unmapped") - .and_then(Value::as_object) - .and_then(|u| u.get("downgraded_from")) - .is_none() - ); - } -} diff --git a/crates/openshell-ocsf/src/format/mod.rs b/crates/openshell-ocsf/src/format/mod.rs index 17518a561..084a013d9 100644 --- a/crates/openshell-ocsf/src/format/mod.rs +++ b/crates/openshell-ocsf/src/format/mod.rs @@ -3,6 +3,5 @@ //! OCSF event formatters: shorthand (human-readable) and JSONL. -pub mod downgrade; pub mod jsonl; pub mod shorthand; diff --git a/crates/openshell-ocsf/src/lib.rs b/crates/openshell-ocsf/src/lib.rs index 569e1bd1d..661b4c6f5 100644 --- a/crates/openshell-ocsf/src/lib.rs +++ b/crates/openshell-ocsf/src/lib.rs @@ -60,8 +60,6 @@ pub use objects::{ // --- Sanitizers --- pub use sanitize::sanitize_reason_for_audit; -pub use format::downgrade::OcsfSchemaVersion; - // --- Builders --- pub use builders::{ ApiActivityBuilder, AppLifecycleBuilder, AuthenticationBuilder, BaseEventBuilder, diff --git a/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs b/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs index 40484ce43..245f92e59 100644 --- a/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs +++ b/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs @@ -13,18 +13,15 @@ use tracing::Subscriber; use tracing_subscriber::Layer; use tracing_subscriber::layer::Context; -use crate::format::downgrade::downgrade_event; use crate::tracing_layers::event_bridge::{OCSF_TARGET, clone_current_event}; /// OCSF JSON output settings owned by one hosted sandbox instance. /// -/// Instances apply their own `ocsf_json_enabled` and schema-version settings -/// to these values. Binding them to the instance's runtime threads makes every +/// Instances apply their own `ocsf_json_enabled` setting to this value. Binding them to the instance's runtime threads makes every /// [`OcsfJsonlLayer`] use them for events those threads emit. #[derive(Clone, Debug, Default)] pub struct OcsfJsonSettings { pub enabled: Arc, - pub target_version: Arc>, } std::thread_local! { @@ -47,15 +44,9 @@ pub fn bind_thread_json_settings(settings: Option) { /// `false`, the layer short-circuits without writing. This allows the sandbox /// to hot-toggle OCSF JSONL output at runtime via the `ocsf_json_enabled` /// setting without rebuilding the subscriber. -/// -/// An optional target schema version can be set via -/// [`with_target_version`](Self::with_target_version). When set, events are -/// downgraded to the target version before writing (stripping fields and -/// profiles that don't exist in older schema versions). pub struct OcsfJsonlLayer { writer: Mutex, enabled: Option>, - target_version: Option>>, console_format: bool, } @@ -66,7 +57,6 @@ impl OcsfJsonlLayer { Self { writer: Mutex::new(writer), enabled: None, - target_version: None, console_format: false, } } @@ -81,16 +71,6 @@ impl OcsfJsonlLayer { self } - /// Attach a shared target schema version for downgrade filtering. - /// - /// When set, events are downgraded to the target version before writing. - /// The version can be changed at runtime via the shared mutex. - #[must_use] - pub fn with_target_version(mut self, version: Arc>) -> Self { - self.target_version = Some(version); - self - } - /// Prefix each compact JSON record with a UTC timestamp and `OCSF-JSON`. /// /// Console collectors can match the marker and decode the remainder as @@ -114,10 +94,9 @@ where // A hosted sandbox instance's settings replace the process-wide ones. let instance = THREAD_JSON_SETTINGS.with(|current| current.borrow().clone()); - let (enabled, target_version) = instance.as_ref().map_or( - (self.enabled.as_ref(), self.target_version.as_ref()), - |settings| (Some(&settings.enabled), Some(&settings.target_version)), - ); + let enabled = instance + .as_ref() + .map_or(self.enabled.as_ref(), |settings| Some(&settings.enabled)); // If an enabled flag is set and it reads `false`, skip writing. if let Some(flag) = enabled @@ -129,26 +108,8 @@ where if let Some(ocsf_event) = clone_current_event() && let Ok(mut w) = self.writer.lock() { - let line = if let Some(target) = target_version - && let Ok(version) = target.lock() - && !version.is_empty() - { - let Ok(mut json) = serde_json::to_value(&ocsf_event) else { - return; - }; - downgrade_event(&mut json, &version); - match serde_json::to_string(&json) { - Ok(mut s) => { - s.push('\n'); - s - } - Err(_) => return, - } - } else { - match ocsf_event.to_json_line() { - Ok(l) => l, - Err(_) => return, - } + let Ok(line) = ocsf_event.to_json_line() else { + return; }; let line = if self.console_format { let ts = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ"); diff --git a/crates/openshell-ocsf/tests/console_output.rs b/crates/openshell-ocsf/tests/console_output.rs index 07fa1f31b..38249ff35 100644 --- a/crates/openshell-ocsf/tests/console_output.rs +++ b/crates/openshell-ocsf/tests/console_output.rs @@ -51,15 +51,13 @@ fn context() -> EventContext { } #[test] -fn console_json_toggles_and_downgrades_without_changing_event_identity() { +fn console_json_toggles_without_changing_event_identity() { let records = Records::default(); let enabled = Arc::new(AtomicBool::new(false)); - let version = Arc::new(Mutex::new(String::new())); let subscriber = tracing_subscriber::registry().with( OcsfJsonlLayer::new(records.clone()) .with_console_format() - .with_enabled_flag(enabled.clone()) - .with_target_version(version.clone()), + .with_enabled_flag(enabled.clone()), ); let event = NetworkActivityBuilder::new(&context()) .dst_endpoint(openshell_ocsf::Endpoint::from_domain("example.com", 443)) @@ -71,18 +69,12 @@ fn console_json_toggles_and_downgrades_without_changing_event_identity() { tracing::warn!("diagnostics are not JSON audit events"); enabled.store(true, Ordering::Relaxed); ocsf_emit!(event.clone()); - *version.lock().unwrap() = "1.3".to_string(); - ocsf_emit!(event.clone()); enabled.store(false, Ordering::Relaxed); ocsf_emit!(event); }); let records = records.0.lock().unwrap(); - assert_eq!(records.len(), 2); + assert_eq!(records.len(), 1); assert_eq!(payload(&records[0]), expected); - let downgraded = payload(&records[1]); - assert_eq!(downgraded["metadata"]["version"], "1.3"); - assert_eq!(downgraded["metadata"]["uid"], expected["metadata"]["uid"]); - assert_eq!(downgraded["message"], expected["message"]); } #[test] diff --git a/crates/openshell-otel/src/ocsf.rs b/crates/openshell-otel/src/ocsf.rs index 2dc33cee8..680b4c1e3 100644 --- a/crates/openshell-otel/src/ocsf.rs +++ b/crates/openshell-otel/src/ocsf.rs @@ -7,8 +7,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use std::time::{Duration, SystemTime}; -use openshell_ocsf::format::downgrade::downgrade_event; -use openshell_ocsf::{OCSF_TARGET, OcsfEvent, OcsfSchemaVersion, SeverityId, clone_current_event}; +use openshell_ocsf::{OCSF_TARGET, OcsfEvent, SeverityId, clone_current_event}; use opentelemetry::InstrumentationScope; use opentelemetry::logs::{AnyValue, LogRecord as _, Logger as _, LoggerProvider as _, Severity}; use opentelemetry_sdk::Resource; @@ -45,11 +44,7 @@ pub struct OcsfOtlp { impl OcsfOtlp { /// Start exporting through `exporter`. - pub fn start( - mut exporter: E, - resource: Resource, - schema_version: Option, - ) -> Self + pub fn start(mut exporter: E, resource: Resource) -> Self where E: LogExporter + 'static, { @@ -71,7 +66,6 @@ impl OcsfOtlp { layer: OcsfOtlpLayer { sender, logger, - schema_version, gap, pending_bytes, }, @@ -110,7 +104,6 @@ impl OcsfOtlp { pub struct OcsfOtlpLayer { sender: mpsc::Sender, logger: SdkLogger, - schema_version: Option, gap: Arc, pending_bytes: Arc, } @@ -123,8 +116,7 @@ impl Layer for OcsfOtlpLayer { let Some(event) = clone_current_event() else { return; }; - let Ok(record) = log_record(&self.logger, &event, self.schema_version, SystemTime::now()) - else { + let Ok(record) = log_record(&self.logger, &event, SystemTime::now()) else { self.gap.record_loss("serialization", 1); return; }; @@ -377,14 +369,11 @@ fn logger() -> SdkLogger { /// Map one OCSF event onto an OTLP log record. /// -/// The body is the OCSF JSON record, downgraded when a schema version is -/// configured. Attributes come from the event itself, so identifiers that a -/// downgrade strips, such as the sandbox container, remain available for -/// routing. +/// The body is the OCSF JSON record. Attributes repeat identifiers from the +/// event, such as the sandbox container, for routing. fn log_record( logger: &SdkLogger, event: &OcsfEvent, - schema_version: Option, observed: SystemTime, ) -> Result { let base = event.base(); @@ -410,14 +399,7 @@ fn log_record( record.add_attribute("openshell.sandbox.id", sandbox_id); } - let body = match schema_version { - None => serde_json::to_string(event)?, - Some(schema_version) => { - let mut body = serde_json::to_value(event)?; - downgrade_event(&mut body, schema_version.as_str()); - serde_json::to_string(&body)? - } - }; + let body = serde_json::to_string(event)?; record.set_body(AnyValue::String(body.into())); Ok(record) } @@ -491,29 +473,21 @@ mod tests { .map(|(_, value)| value.clone()) } - fn record(event: &OcsfEvent, schema_version: Option) -> SdkLogRecord { - log_record(&logger(), event, schema_version, SystemTime::UNIX_EPOCH).unwrap() + fn record(event: &OcsfEvent) -> SdkLogRecord { + log_record(&logger(), event, SystemTime::UNIX_EPOCH).unwrap() } #[test] fn body_is_the_native_ocsf_json_record() { let event = event("sandbox-1", SeverityId::Informational); - assert_eq!(body(&record(&event, None)), native(&event)); - } - - #[test] - fn body_follows_the_configured_schema_version() { - let event = event("sandbox-1", SeverityId::Informational); - let body = body(&record(&event, Some(OcsfSchemaVersion::V1_3))); - assert_eq!(body["metadata"]["version"], "1.3"); - assert!(body.get("container").is_none()); + assert_eq!(body(&record(&event)), native(&event)); } #[test] fn timestamps_are_event_time_and_observed_time() { let event = event("sandbox-1", SeverityId::Informational); let observed = SystemTime::UNIX_EPOCH + Duration::from_secs(42); - let record = log_record(&logger(), &event, None, observed).unwrap(); + let record = log_record(&logger(), &event, observed).unwrap(); let time_ms = native(&event)["time"].as_u64().unwrap(); assert_eq!( record.timestamp(), @@ -534,7 +508,7 @@ mod tests { (SeverityId::Fatal, Severity::Fatal), (SeverityId::Other, Severity::Info), ] { - let record = record(&event("", severity), None); + let record = record(&event("", severity)); assert_eq!(record.severity_number(), Some(number), "{severity:?}"); assert_eq!(record.severity_text(), Some(severity.label())); } @@ -544,8 +518,7 @@ mod tests { fn attributes_identify_the_event_and_its_sandbox() { let event = event("sandbox-1", SeverityId::Medium); let native = native(&event); - // The sandbox identity survives a downgrade that strips `container`. - let record = record(&event, Some(OcsfSchemaVersion::V1_1)); + let record = record(&event); let int = |key| match attribute(&record, key) { Some(AnyValue::Int(value)) => value, other => panic!("{key}: {other:?}"), @@ -574,7 +547,7 @@ mod tests { #[test] fn events_without_a_sandbox_carry_no_sandbox_attribute() { - let record = record(&event("", SeverityId::Informational), None); + let record = record(&event("", SeverityId::Informational)); assert!(attribute(&record, "openshell.sandbox.id").is_none()); } @@ -696,7 +669,7 @@ mod tests { #[tokio::test] async fn emitted_ocsf_events_reach_the_exporter_with_its_resource() { let exporter = RecordingExporter::default(); - let otlp = OcsfOtlp::start(exporter.clone(), exporter_resource(), None); + let otlp = OcsfOtlp::start(exporter.clone(), exporter_resource()); let subscriber = tracing_subscriber::registry().with(otlp.layer()); let event = event("sandbox-1", SeverityId::Medium); let expected = native(&event); @@ -716,7 +689,7 @@ mod tests { #[tokio::test] async fn shutdown_exports_every_queued_record_in_bounded_batches() { let exporter = RecordingExporter::default(); - let otlp = OcsfOtlp::start(exporter.clone(), exporter_resource(), None); + let otlp = OcsfOtlp::start(exporter.clone(), exporter_resource()); emit(&otlp, BATCH_SIZE * 2 + 1); otlp.shutdown().await; assert_eq!(exporter.exported().len(), BATCH_SIZE * 2 + 1); @@ -734,7 +707,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn transient_export_failures_are_retried_without_loss() { let exporter = RecordingExporter::failing(MAX_EXPORT_ATTEMPTS - 1); - let otlp = OcsfOtlp::start(exporter.clone(), exporter_resource(), None); + let otlp = OcsfOtlp::start(exporter.clone(), exporter_resource()); emit(&otlp, 3); settle().await; otlp.shutdown().await; @@ -748,7 +721,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn exhausted_retries_drop_the_batch_and_report_the_gap_next() { let exporter = RecordingExporter::failing(MAX_EXPORT_ATTEMPTS); - let otlp = OcsfOtlp::start(exporter.clone(), exporter_resource(), None); + let otlp = OcsfOtlp::start(exporter.clone(), exporter_resource()); emit(&otlp, 3); settle().await; emit(&otlp, 1); @@ -778,7 +751,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn batches_stay_within_the_byte_limit() { let exporter = RecordingExporter::default(); - let otlp = OcsfOtlp::start(exporter.clone(), exporter_resource(), None); + let otlp = OcsfOtlp::start(exporter.clone(), exporter_resource()); emit_sized(&otlp, 5, BATCH_MAX_BYTES * 2 / 5); settle().await; let pending_bytes = Arc::clone(&otlp.layer.pending_bytes); @@ -797,7 +770,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn records_larger_than_a_batch_are_dropped_and_reported() { let exporter = RecordingExporter::default(); - let otlp = OcsfOtlp::start(exporter.clone(), exporter_resource(), None); + let otlp = OcsfOtlp::start(exporter.clone(), exporter_resource()); emit_sized(&otlp, 1, BATCH_MAX_BYTES); emit(&otlp, 1); settle().await; @@ -814,7 +787,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn the_queue_rejects_records_beyond_its_byte_limit() { - let otlp = OcsfOtlp::start(HangingExporter::default(), exporter_resource(), None); + let otlp = OcsfOtlp::start(HangingExporter::default(), exporter_resource()); let events = QUEUE_MAX_BYTES / (BATCH_MAX_BYTES / 2) + 4; emit_sized(&otlp, events, BATCH_MAX_BYTES / 2); @@ -841,7 +814,7 @@ mod tests { async fn shutdown_stops_a_worker_that_outlasts_the_budget() { let exporter = HangingExporter::default(); let alive = Arc::clone(&exporter.alive); - let otlp = OcsfOtlp::start(exporter, exporter_resource(), None); + let otlp = OcsfOtlp::start(exporter, exporter_resource()); emit(&otlp, 1); settle().await; otlp.shutdown().await; diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 9b16b5391..035b3bcda 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -793,6 +793,13 @@ async fn run_from_args( if prepared.legacy_compute_driver_env_seen { warn!("OPENSHELL_DRIVERS is deprecated; migrate to OPENSHELL_COMPUTE_DRIVER"); } + for field in prepared + .config_file + .iter() + .flat_map(ConfigFile::deprecated_ocsf_schema_versions) + { + warn!("{field} is deprecated and ignored; OpenShell emits OCSF 1.8.0"); + } if let Some(bundle) = prepared.network_trust.as_ref() { info!( certificate_count = bundle.certificate_count(), diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index cee598a99..a01b3c119 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -261,14 +261,13 @@ pub enum OcsfLogRotation { Daily, } -pub use openshell_ocsf::OcsfSchemaVersion; - #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(try_from = "RawOcsfLogConfig")] pub struct OcsfLogConfig { pub path: PathBuf, + /// Deprecated and ignored: `OpenShell` emits OCSF 1.8.0 only. #[serde(skip_serializing_if = "Option::is_none")] - pub schema_version: Option, + pub schema_version: Option, pub rotation: OcsfLogRotation, #[serde(skip_serializing_if = "Option::is_none")] pub max_files: Option, @@ -280,7 +279,7 @@ pub struct OcsfLogConfig { #[serde(deny_unknown_fields)] struct RawOcsfLogConfig { path: PathBuf, - schema_version: Option, + schema_version: Option, #[serde(default)] rotation: OcsfLogRotation, max_files: Option, @@ -402,9 +401,9 @@ pub struct OcsfOtlpConfig { /// OTLP/gRPC collector endpoint for OCSF log records. pub endpoint: String, - /// Optional OCSF downgrade target for exported log records. + /// Deprecated and ignored: `OpenShell` emits OCSF 1.8.0 only. #[serde(default, skip_serializing_if = "Option::is_none")] - pub schema_version: Option, + pub schema_version: Option, } /// `[openshell.gateway.otlp.agent]`. @@ -454,7 +453,7 @@ struct RawOcsfOtlp { #[serde(default)] endpoint: Option, #[serde(default)] - schema_version: Option, + schema_version: Option, } impl TryFrom for OtlpConfig { @@ -535,6 +534,28 @@ pub struct SupervisorNetworkFileSection { } impl ConfigFile { + /// Deprecated `schema_version` fields that are set; they are ignored. + pub fn deprecated_ocsf_schema_versions(&self) -> Vec<&'static str> { + let gateway = &self.openshell.gateway; + let mut set = Vec::new(); + if gateway + .ocsf_log + .as_ref() + .is_some_and(|ocsf| ocsf.schema_version.is_some()) + { + set.push("openshell.gateway.ocsf_log.schema_version"); + } + if gateway + .otlp + .as_ref() + .and_then(|otlp| otlp.ocsf.as_ref()) + .is_some_and(|ocsf| ocsf.schema_version.is_some()) + { + set.push("openshell.gateway.otlp.ocsf.schema_version"); + } + set + } + /// Resolve the selected driver's exact backend name without sandbox input. pub(crate) fn isolation_backend( &self, @@ -1389,20 +1410,40 @@ service_name = "openshell-gateway-dev" assert_eq!(config.queue_capacity.get(), 10_000); assert_eq!(config.queue_max_bytes.get(), 16 * 1024 * 1024); assert!(ConfigFile::default().openshell.gateway.ocsf_log.is_none()); + } + #[test] + fn names_the_deprecated_schema_versions_that_are_set() { + assert!( + ConfigFile::default() + .deprecated_ocsf_schema_versions() + .is_empty() + ); + let tmp = write_tmp( + "[openshell.gateway.ocsf_log]\npath = 'events.jsonl'\nschema_version = '1.3'\n\n\ + [openshell.gateway.otlp.ocsf]\nenabled = true\nendpoint = 'http://siem:4317'\nschema_version = '1.3'\n", + ); + assert_eq!( + load(tmp.path()).unwrap().deprecated_ocsf_schema_versions(), + [ + "openshell.gateway.ocsf_log.schema_version", + "openshell.gateway.otlp.ocsf.schema_version" + ] + ); + } + + #[test] + fn ocsf_log_accepts_the_deprecated_schema_version() { let tmp = write_tmp( "[openshell.gateway.ocsf_log]\npath = 'events.jsonl'\nschema_version = '1.3'\n", ); - assert_eq!( - load(tmp.path()) - .unwrap() - .openshell - .gateway - .ocsf_log - .unwrap() - .schema_version, - Some(OcsfSchemaVersion::V1_3) - ); + let config = load(tmp.path()) + .unwrap() + .openshell + .gateway + .ocsf_log + .unwrap(); + assert_eq!(config.schema_version.as_deref(), Some("1.3")); } #[test] @@ -1414,8 +1455,6 @@ service_name = "openshell-gateway-dev" "path = 'log'\nqueue_max_bytes = 0", "path = 'log'\nmax_files = 0", "path = 'log'\nrotation = 'hourly'", - "path = 'log'\nschema_version = ''", - "path = 'log'\nschema_version = '1.8'", "path = 'log'\nkind = 'jsonl'", "path = 'log'\nrotation = 'never'\nmax_files = 7", ] { @@ -1467,18 +1506,29 @@ service_name = "openshell-gateway-dev" let otlp = otlp( "[openshell.gateway.otlp]\nendpoint = 'http://shared:4317'\n\n\ [openshell.gateway.otlp.traces]\nendpoint = 'http://traces:4317'\n\n\ - [openshell.gateway.otlp.ocsf]\nenabled = true\nendpoint = 'http://siem:4317'\nschema_version = '1.1'\n", + [openshell.gateway.otlp.ocsf]\nenabled = true\nendpoint = 'http://siem:4317'\n", ); assert_eq!(otlp.traces_endpoint(), Some("http://traces:4317")); assert_eq!( otlp.ocsf, Some(OcsfOtlpConfig { endpoint: "http://siem:4317".into(), - schema_version: Some(OcsfSchemaVersion::V1_1), + schema_version: None, }) ); } + #[test] + fn otlp_ocsf_accepts_the_deprecated_schema_version() { + let otlp = otlp( + "[openshell.gateway.otlp.ocsf]\nenabled = true\nendpoint = 'http://siem:4317'\nschema_version = '1.3'\n", + ); + assert_eq!( + otlp.ocsf.and_then(|ocsf| ocsf.schema_version).as_deref(), + Some("1.3") + ); + } + #[test] fn otlp_ocsf_exports_without_traces() { let otlp = @@ -1546,7 +1596,7 @@ service_name = "openshell-gateway-dev" fn otlp_config_round_trips() { let tmp = write_tmp( "[openshell.gateway.otlp]\nendpoint = 'http://shared:4317'\nservice_name = 'gw'\n\n\ - [openshell.gateway.otlp.ocsf]\nenabled = true\nschema_version = '1.3'\n\n\ + [openshell.gateway.otlp.ocsf]\nenabled = true\n\n\ [openshell.gateway.otlp.agent]\nenabled = true\nendpoint = 'http://agents:4317'\n", ); let file = load(tmp.path()).unwrap(); @@ -1562,7 +1612,6 @@ service_name = "openshell-gateway-dev" "[openshell.gateway.otlp.traces]\n", "[openshell.gateway.otlp.ocsf]\n", "[openshell.gateway.otlp.ocsf]\nenabled = true\n", - "[openshell.gateway.otlp.ocsf]\nenabled = true\nendpoint = 'http://siem:4317'\nschema_version = '1.8'\n", "[openshell.gateway.otlp.ocsf]\nenabled = 'yes'\nendpoint = 'http://siem:4317'\n", "[openshell.gateway.otlp.metrics]\nendpoint = 'http://metrics:4317'\n", "[openshell.gateway.otlp]\nendpoint = 'http://127.0.0.1:4317'\nocsf_logs = true\n", diff --git a/crates/openshell-server/src/ocsf_log.rs b/crates/openshell-server/src/ocsf_log.rs index 8858c9c58..6ecf9a790 100644 --- a/crates/openshell-server/src/ocsf_log.rs +++ b/crates/openshell-server/src/ocsf_log.rs @@ -9,12 +9,12 @@ use std::sync::{Arc, Condvar, Mutex}; use std::time::{Duration, Instant}; use chrono::{DateTime, NaiveDate, Utc}; -use openshell_ocsf::{OcsfEvent, format::downgrade::downgrade_event}; +use openshell_ocsf::OcsfEvent; use tracing::{Event, Subscriber}; use tracing_subscriber::Layer; use tracing_subscriber::layer::Context; -use crate::config_file::{OcsfLogConfig, OcsfLogRotation, OcsfSchemaVersion}; +use crate::config_file::{OcsfLogConfig, OcsfLogRotation}; const BATCH_SIZE: usize = 100; const FLUSH_INTERVAL: Duration = Duration::from_millis(500); @@ -49,7 +49,7 @@ impl Collector { } pub(crate) fn collect(&self, event: &OcsfEvent) { - match serialize_event(event, self.0.config.schema_version) { + match event.to_json_line().map(String::into_bytes) { Ok(line) => { self.enqueue(line); } @@ -158,20 +158,6 @@ impl Collector { } } -fn serialize_event( - event: &OcsfEvent, - schema_version: Option, -) -> Result, serde_json::Error> { - let Some(schema_version) = schema_version else { - return event.to_json_line().map(String::into_bytes); - }; - let mut event = serde_json::to_value(event)?; - downgrade_event(&mut event, schema_version.as_str()); - let mut line = serde_json::to_vec(&event)?; - line.push(b'\n'); - Ok(line) -} - pub struct OcsfLog { collector: Collector, done: tokio::sync::oneshot::Receiver<()>, @@ -483,11 +469,11 @@ mod tests { } #[tokio::test] - async fn gateway_events_use_the_configured_schema_version() { + async fn deprecated_schema_version_is_ignored() { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("events.jsonl"); let mut settings = config(&path); - settings.schema_version = Some(OcsfSchemaVersion::V1_3); + settings.schema_version = Some("1.3".to_string()); let log = OcsfLog::start(settings).unwrap(); let event = ConfigStateChangeBuilder::new(&crate::gateway_ocsf::context("", "")) .message("Gateway configuration changed") @@ -498,11 +484,7 @@ mod tests { let event: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); - assert_eq!(event["metadata"]["version"], "1.3"); - assert_eq!( - event["unmapped"]["downgraded_from"], - openshell_ocsf::OCSF_VERSION - ); + assert_eq!(event["metadata"]["version"], openshell_ocsf::OCSF_VERSION); } #[tokio::test] diff --git a/crates/openshell-server/src/otel_tracing.rs b/crates/openshell-server/src/otel_tracing.rs index 0d147c0e8..a50ba1e21 100644 --- a/crates/openshell-server/src/otel_tracing.rs +++ b/crates/openshell-server/src/otel_tracing.rs @@ -171,11 +171,7 @@ impl OtlpExport { let target = target(&ocsf.endpoint)?; Ok::<_, SetupError>(Signal { endpoint: target.endpoint().to_string(), - exporter: OcsfOtlp::start( - target.log_exporter()?, - target.resource().clone(), - ocsf.schema_version, - ), + exporter: OcsfOtlp::start(target.log_exporter()?, target.resource().clone()), }) }) .transpose()?; @@ -727,7 +723,6 @@ mod tests { .log_exporter_with_headers(Some("authorization=Bearer%20collector-token")) .unwrap(), target.resource().clone(), - None, ); let event = ConfigStateChangeBuilder::new(&crate::gateway_ocsf::context("sandbox-1", "sandbox")) diff --git a/crates/openshell-supervisor/src/config_runtime.rs b/crates/openshell-supervisor/src/config_runtime.rs index 1a83fd088..8ced34ffe 100644 --- a/crates/openshell-supervisor/src/config_runtime.rs +++ b/crates/openshell-supervisor/src/config_runtime.rs @@ -689,7 +689,7 @@ impl ConfigRuntime { // Apply OCSF JSON toggle from the `ocsf_json_enabled` setting. apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); + report_deprecated_ocsf_schema_version(&ctx.ocsf_schema_version, &result.settings); // Apply the agent-proposals feature toggle. On a false→true transition // we lazily install the skill so a sandbox that started with the flag @@ -1277,7 +1277,10 @@ async fn run_policy_poll_loop_with_client( runtime.applied_revision = Some(sandbox_config_revision(&result)); } apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); + report_deprecated_ocsf_schema_version( + &ctx.ocsf_schema_version, + &result.settings, + ); apply_agent_proposals_enabled( &ctx.agent_proposals, agent_proposals_enabled_from_settings(&result.settings), @@ -1316,7 +1319,10 @@ async fn run_policy_poll_loop_with_client( } (InitialPollDisposition::TrackOnly, _) => { apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); + report_deprecated_ocsf_schema_version( + &ctx.ocsf_schema_version, + &result.settings, + ); apply_agent_proposals_enabled( &ctx.agent_proposals, agent_proposals_enabled_from_settings(&result.settings), @@ -1465,20 +1471,33 @@ fn extract_bool_setting( }) } -fn apply_ocsf_schema_version_setting( - version: &std::sync::Mutex, +/// Report a newly set `ocsf_schema_version`, which is deprecated and ignored. +fn report_deprecated_ocsf_schema_version( + seen: &std::sync::Mutex, settings: &std::collections::HashMap, -) { - let new_version = extract_string_setting(settings, "ocsf_schema_version").unwrap_or_default(); - if let Ok(mut current) = version.lock() - && *current != new_version - { - info!( - ocsf_schema_version = %new_version, - "OCSF schema version target changed" - ); - *current = new_version; +) -> bool { + let value = extract_string_setting(settings, "ocsf_schema_version").unwrap_or_default(); + let Ok(mut seen) = seen.lock() else { + return false; + }; + if *seen == value { + return false; } + seen.clone_from(&value); + if value.is_empty() { + return false; + } + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .state(StateId::Other, "deprecated") + .unmapped("ocsf_schema_version", serde_json::json!(value)) + .message( + "Setting ocsf_schema_version is deprecated and ignored; OpenShell emits OCSF 1.8.0" + ) + .build() + ); + true } /// Extract a string value from an effective setting, if present. @@ -1694,9 +1713,11 @@ mod tests { } #[test] - fn apply_ocsf_schema_version_setting_updates_from_initial_settings_snapshot() { - let version = std::sync::Mutex::new(String::new()); + fn deprecated_ocsf_schema_version_is_reported_once_per_change() { + let seen = std::sync::Mutex::new(String::new()); let mut settings = std::collections::HashMap::new(); + assert!(!report_deprecated_ocsf_schema_version(&seen, &settings)); + settings.insert( "ocsf_schema_version".to_string(), openshell_core::proto::EffectiveSetting { @@ -1708,20 +1729,11 @@ mod tests { scope: openshell_core::proto::SettingScope::Sandbox.into(), }, ); + assert!(report_deprecated_ocsf_schema_version(&seen, &settings)); + assert!(!report_deprecated_ocsf_schema_version(&seen, &settings)); - apply_ocsf_schema_version_setting(&version, &settings); - - assert_eq!(*version.lock().unwrap(), "1.3"); - } - - #[test] - fn apply_ocsf_schema_version_setting_clears_when_setting_is_unset() { - let version = std::sync::Mutex::new("1.1".to_string()); - let settings = std::collections::HashMap::new(); - - apply_ocsf_schema_version_setting(&version, &settings); - - assert_eq!(*version.lock().unwrap(), ""); + settings.clear(); + assert!(!report_deprecated_ocsf_schema_version(&seen, &settings)); } #[test] diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index dddf8990c..8b1035bec 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -1590,7 +1590,7 @@ pub async fn run_sandbox_instance( let poll_gateway = gateway.clone(); let poll_engine = engine.clone(); let poll_ocsf_enabled = ocsf_json.enabled.clone(); - let poll_ocsf_schema_version = ocsf_json.target_version.clone(); + let poll_ocsf_schema_version = Arc::new(std::sync::Mutex::new(String::new())); let poll_pid = entrypoint_pid.clone(); let poll_provider_credentials = provider_credentials.clone(); let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); @@ -4207,6 +4207,7 @@ struct PolicyPollLoopContext { entrypoint_pid: Arc, interval_secs: u64, ocsf_enabled: Arc, + /// Last value of the deprecated `ocsf_schema_version` setting. ocsf_schema_version: Arc>, provider_credentials: ProviderCredentialState, provider_readiness: ProviderReadinessTracker, diff --git a/crates/openshell-supervisor/src/logging.rs b/crates/openshell-supervisor/src/logging.rs index 89b313c3d..d75e6704f 100644 --- a/crates/openshell-supervisor/src/logging.rs +++ b/crates/openshell-supervisor/src/logging.rs @@ -6,7 +6,6 @@ use std::io::Write; use std::path::Path; use std::sync::Arc; -use std::sync::Mutex; use std::sync::atomic::AtomicBool; use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer}; @@ -24,7 +23,6 @@ pub fn layers( directory: &Path, console_filter: EnvFilter, enabled: Arc, - schema_version: Arc>, ) -> (impl Layer, Vec, bool) where S: Subscriber + for<'a> LookupSpan<'a> + 'static, @@ -49,7 +47,6 @@ where let json_file = json_file.map(|writer| { OcsfJsonlLayer::new(writer) .with_enabled_flag(enabled.clone()) - .with_target_version(schema_version.clone()) .with_filter(LevelFilter::INFO) }); // Keep per-destination filters independent: a diagnostic filter must not @@ -61,7 +58,6 @@ where OcsfJsonlLayer::new(console) .with_console_format() .with_enabled_flag(enabled) - .with_target_version(schema_version) .with_filter(LevelFilter::INFO) .boxed(), shorthand_file.boxed(), @@ -92,6 +88,7 @@ fn file_writer( mod tests { use super::*; use openshell_ocsf::{EventContext, NetworkActivityBuilder, ocsf_emit}; + use std::sync::Mutex; use tracing_subscriber::layer::SubscriberExt; #[derive(Clone)] @@ -132,7 +129,6 @@ mod tests { &invalid_directory, EnvFilter::new("off"), Arc::new(AtomicBool::new(true)), - Arc::new(Mutex::new(String::new())), ); assert!(!file_available); let subscriber = tracing_subscriber::registry().with(layers); @@ -162,7 +158,6 @@ mod tests { directory.path(), EnvFilter::new("info"), Arc::new(AtomicBool::new(true)), - Arc::new(Mutex::new(String::new())), ); assert!(file_available); let subscriber = tracing_subscriber::registry().with(layers); @@ -244,7 +239,6 @@ mod tests { &invalid_directory, EnvFilter::new("info"), Arc::new(AtomicBool::new(false)), - Arc::new(Mutex::new(String::new())), ); let dispatch = tracing::Dispatch::new(tracing_subscriber::registry().with(layers)); tracing::dispatcher::with_default(&dispatch, || tracing::info!("occupy writer")); diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index be91a6455..7c35b9682 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -142,10 +142,6 @@ struct Args { #[arg(long, env = openshell_core::sandbox_env::OTLP_OCSF_ENDPOINT)] otlp_ocsf_endpoint: Option, - /// OCSF schema version that exported events are downgraded to. - #[arg(long, env = openshell_core::sandbox_env::OTLP_OCSF_SCHEMA_VERSION)] - otlp_ocsf_schema_version: Option, - /// OTLP/gRPC collector endpoint that workload traces are relayed to. /// Absent keeps the relay inactive. #[arg(long, env = openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT)] @@ -496,7 +492,6 @@ fn main() -> Result<()> { start_ocsf_export( args.otlp_ocsf_endpoint.as_deref().map(export_config), ocsf_headers, - args.otlp_ocsf_schema_version, ) }); let otlp_layer_provider = otlp_provider.clone(); @@ -527,14 +522,12 @@ fn main() -> Result<()> { let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone()); let _log_push_handle = log_push_state.map(|(_, handle)| handle); let ocsf_enabled = Arc::new(AtomicBool::new(false)); - let ocsf_schema_version = Arc::new(std::sync::Mutex::new(String::new())); let (logging_layers, _logging_guards, file_available) = logging::layers( std::io::stderr(), Path::new("/var/log"), console_filter, ocsf_enabled.clone(), - ocsf_schema_version.clone(), ); tracing_subscriber::registry() .with(logging_layers) @@ -642,7 +635,6 @@ fn main() -> Result<()> { gateway_credentials, openshell_ocsf::OcsfJsonSettings { enabled: ocsf_enabled, - target_version: ocsf_schema_version, }, ); let result = Box::pin(openshell_supervisor::run_sandbox_with_backend( @@ -695,7 +687,6 @@ fn main() -> Result<()> { fn start_ocsf_export( config: Option>, headers: Option<&str>, - schema_version: Option, ) -> ( Option, Option, @@ -707,7 +698,6 @@ fn start_ocsf_export( Ok(openshell_otel::OcsfOtlp::start( target.log_exporter_with_headers(headers)?, target.resource().clone(), - schema_version, )) }); match started { @@ -921,27 +911,6 @@ mod tests { assert!(supervisor_resource_attributes(None, None).is_empty()); } - #[test] - fn ocsf_export_takes_a_supported_schema_version() { - let args = Args::try_parse_from([ - "openshell-supervisor", - "--otlp-ocsf-endpoint", - "http://siem:4317", - "--otlp-ocsf-schema-version", - "1.3", - ]) - .expect("parse OCSF export"); - assert_eq!(args.otlp_ocsf_endpoint.as_deref(), Some("http://siem:4317")); - assert_eq!( - args.otlp_ocsf_schema_version, - Some(openshell_ocsf::OcsfSchemaVersion::V1_3) - ); - assert!( - Args::try_parse_from(["openshell-supervisor", "--otlp-ocsf-schema-version", "1.8"]) - .is_err() - ); - } - #[test] fn agent_relay_endpoint_is_separate_from_trace_export() { let args = Args::try_parse_from([ @@ -975,17 +944,16 @@ mod tests { service_version: None, resource_attributes: Vec::new(), }; - let (export, error) = start_ocsf_export(None, None, None); + let (export, error) = start_ocsf_export(None, None); assert!(export.is_none() && error.is_none()); - let (export, error) = start_ocsf_export(Some(config("not a url")), None, None); + let (export, error) = start_ocsf_export(Some(config("not a url")), None); assert!(export.is_none()); assert!(error.is_some()); let (export, error) = start_ocsf_export( Some(config("http://127.0.0.1:4317")), Some("authorization=Bearer%20t"), - None, ); assert!(error.is_none(), "unexpected setup error: {error:?}"); export.expect("OCSF export").shutdown().await; diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index 2f449327e..243f18165 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -94,7 +94,6 @@ from sandbox runtimes)", std::path::Path::new("/var/log"), console_filter, Arc::new(AtomicBool::new(false)), - Arc::new(std::sync::Mutex::new(String::new())), ); tracing_subscriber::registry() .with(logging_layers) diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index ff240bc77..df46e0a4a 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -646,7 +646,7 @@ discovery endpoint or its TLS CA. | server.ocsfLog.queueCapacity | int | `10000` | Maximum records waiting for the file writer. | | server.ocsfLog.queueMaxBytes | int | `16777216` | Maximum encoded bytes waiting for the file writer. | | server.ocsfLog.rotation | string | `"daily"` | Rotate the active file daily in UTC, or never. | -| server.ocsfLog.schemaVersion | string | `""` | Optional OCSF downgrade target. Empty emits native OCSF 1.8.0. Supported values: "1.1", "1.3". | +| server.ocsfLog.schemaVersion | string | `""` | Deprecated and ignored: the gateway emits native OCSF 1.8.0 and logs a warning at startup when this is set. | | server.oidc.adminRole | string | `""` | Role name for admin access. Leave empty (with userRole also empty) for authentication-only mode. Both must be set or both empty. | | server.oidc.audience | string | `"openshell-cli"` | Expected audience claim for the API resource server. This should match the server's --oidc-audience, NOT the CLI client ID. | | server.oidc.caConfigMapName | string | `""` | Name of a ConfigMap containing a CA certificate bundle (key: ca.crt) for verifying the OIDC issuer's TLS certificate. Required when the issuer uses a non-public CA (e.g. OpenShift ingress, private PKI). | diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index cf8bf6120..cc79d51d7 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -183,7 +183,7 @@ Top-level lists represent TOML arrays of tables and preserve their YAML order. * {{- if not (hasKey $ocsfConfig "rotation") -}}{{- $rotation := get $legacyOcsf "rotation" | default "daily" -}}{{- if not (has $rotation (list "daily" "never")) -}}{{- fail "server.ocsfLog.rotation must be daily or never" -}}{{- end -}}{{- $_ := set $ocsfConfig "rotation" $rotation -}}{{- end -}} {{- if not (hasKey $ocsfConfig "queue_capacity") -}}{{- $queueCapacity := 10000 -}}{{- if and (hasKey $legacyOcsf "queueCapacity") (ne (get $legacyOcsf "queueCapacity") nil) -}}{{- $queueCapacity = get $legacyOcsf "queueCapacity" -}}{{- end -}}{{- if lt (int $queueCapacity) 1 -}}{{- fail "server.ocsfLog.queueCapacity and queueMaxBytes must be positive" -}}{{- end -}}{{- $_ := set $ocsfConfig "queue_capacity" (int $queueCapacity) -}}{{- end -}} {{- if not (hasKey $ocsfConfig "queue_max_bytes") -}}{{- $queueMaxBytes := 16777216 -}}{{- if and (hasKey $legacyOcsf "queueMaxBytes") (ne (get $legacyOcsf "queueMaxBytes") nil) -}}{{- $queueMaxBytes = get $legacyOcsf "queueMaxBytes" -}}{{- end -}}{{- if lt (int $queueMaxBytes) 1 -}}{{- fail "server.ocsfLog.queueCapacity and queueMaxBytes must be positive" -}}{{- end -}}{{- $_ := set $ocsfConfig "queue_max_bytes" (int $queueMaxBytes) -}}{{- end -}} -{{- if not (hasKey $ocsfConfig "schema_version") -}}{{- $schemaVersion := get $legacyOcsf "schemaVersion" | default "" -}}{{- if not (has $schemaVersion (list "" "1.1" "1.3")) -}}{{- fail "server.ocsfLog.schemaVersion must be empty, 1.1, or 1.3" -}}{{- end -}}{{- if $schemaVersion -}}{{- $_ := set $ocsfConfig "schema_version" $schemaVersion -}}{{- end -}}{{- end -}} +{{- if not (hasKey $ocsfConfig "schema_version") -}}{{- $schemaVersion := get $legacyOcsf "schemaVersion" | default "" -}}{{- if $schemaVersion -}}{{- $_ := set $ocsfConfig "schema_version" (toString $schemaVersion) -}}{{- end -}}{{- end -}} {{- if and (eq (get $ocsfConfig "rotation") "daily") (not (hasKey $ocsfConfig "max_files")) -}}{{- $maxFiles := 7 -}}{{- if and (hasKey $legacyOcsf "maxFiles") (ne (get $legacyOcsf "maxFiles") nil) -}}{{- $maxFiles = get $legacyOcsf "maxFiles" -}}{{- end -}}{{- if lt (int $maxFiles) 1 -}}{{- fail "server.ocsfLog.maxFiles must be positive when rotation is daily" -}}{{- end -}}{{- $_ := set $ocsfConfig "max_files" (int $maxFiles) -}}{{- end -}} {{- end -}} {{- $_ := set $config "openshell.gateway.ocsf_log" $ocsfConfig -}} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 16639d892..f298fe93b 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -341,11 +341,10 @@ tests: openshell.gateway.otlp.ocsf: enabled: true endpoint: http://siem-collector:4317 - schema_version: "1.3" asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.gateway\.otlp\.ocsf\]\nenabled\s*=\s*true\nendpoint\s*=\s*"http://siem-collector:4317"\nschema_version\s*=\s*"1\.3"' + pattern: '(?ms)\[openshell\.gateway\.otlp\.ocsf\]\nenabled\s*=\s*true\nendpoint\s*=\s*"http://siem-collector:4317"' - matchRegex: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\.otlp\.traces\]\nendpoint\s*=\s*"http://traces:4317"' @@ -440,7 +439,6 @@ tests: set: server.ocsfLog.enabled: true server.ocsfLog.path: /var/openshell/gateway-ocsf.jsonl - server.ocsfLog.schemaVersion: "1.3" server.ocsfLog.rotation: daily server.ocsfLog.maxFiles: 14 server.ocsfLog.queueCapacity: 20000 @@ -448,7 +446,7 @@ tests: asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.gateway\.ocsf_log\].*?max_files\s*=\s*14.*?path\s*=\s*"/var/openshell/gateway-ocsf\.jsonl".*?queue_capacity\s*=\s*20000.*?queue_max_bytes\s*=\s*33554432.*?rotation\s*=\s*"daily".*?schema_version\s*=\s*"1\.3"' + pattern: '(?ms)\[openshell\.gateway\.ocsf_log\].*?max_files\s*=\s*14.*?path\s*=\s*"/var/openshell/gateway-ocsf\.jsonl".*?queue_capacity\s*=\s*20000.*?queue_max_bytes\s*=\s*33554432.*?rotation\s*=\s*"daily"' - it: omits OCSF retention when rotation is disabled template: templates/gateway-config.yaml @@ -513,15 +511,16 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.gateway\.ocsf_log\].*?max_files\s*=\s*7.*?queue_capacity\s*=\s*10000.*?queue_max_bytes\s*=\s*16777216' - - it: rejects an unsupported OCSF schema version - template: templates/statefulset.yaml + - it: passes the deprecated OCSF schema version through for the gateway to report + template: templates/gateway-config.yaml set: server.ocsfLog.enabled: true server.ocsfLog.path: /var/openshell/gateway-ocsf.jsonl - server.ocsfLog.schemaVersion: "1.8" + server.ocsfLog.schemaVersion: "1.3" asserts: - - failedTemplate: - errorMessage: "server.ocsfLog.schemaVersion must be empty, 1.1, or 1.3" + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\.ocsf_log\].*?schema_version\s*=\s*"1\.3"' - it: mounts extra gateway volumes for OCSF output in a Deployment template: templates/deployment.yaml diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 3f6f0f86d..fa1519e02 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -406,8 +406,8 @@ server: # When replicas share the volume, set subPathExpr: $(OPENSHELL_POD_NAME) # on the mount so each replica writes its own file. path: /tmp/gateway-ocsf.jsonl - # -- Optional OCSF downgrade target. Empty emits native OCSF 1.8.0. - # Supported values: "1.1", "1.3". + # -- Deprecated and ignored: the gateway emits native OCSF 1.8.0 and logs + # a warning at startup when this is set. schemaVersion: "" # -- Rotate the active file daily in UTC, or never. rotation: daily diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index ef49f8ec8..715135e63 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -528,7 +528,6 @@ Gateway records use `[openshell.gateway] name` as `device.uid` and `device.name` ```toml [openshell.gateway.ocsf_log] path = "/var/log/openshell/gateway-ocsf.jsonl" -schema_version = "1.3" rotation = "daily" max_files = 7 queue_capacity = 10000 @@ -538,7 +537,7 @@ queue_max_bytes = 16777216 | Field | Default | Behavior | |---|---|---| | `path` | Required | Nonempty file path; relative paths use the gateway's working directory. | -| `schema_version` | Native OCSF 1.8.0 | Optional downgrade target. Accepted values are `1.1` and `1.3`. The configured version applies to every record in the file. | +| `schema_version` | None | Deprecated and ignored; records are native OCSF 1.8.0. The gateway logs a warning at startup when it is set. | | `rotation` | `daily` | Rotate on the first write of a new UTC day. `never` allows unbounded file growth. | | `max_files` | `7` | Positive number of rotated segments to retain, in addition to the active file. Rejects explicit use with `rotation = "never"`. | | `queue_capacity` | `10000` | Positive queued-record limit. Overflow discards incoming records. | @@ -546,7 +545,7 @@ queue_max_bytes = 16777216 Unknown fields are rejected. This is one file destination, not an exporter registry. To send the same events to a collector, use [OCSF log export](#ocsf-log-export). The existing `ocsf_json_enabled` sandbox setting remains independent. -For Helm installations, set `server.ocsfLog.enabled` to render this table and optionally set `server.ocsfLog.schemaVersion`. The default `server.ocsfLog.path`, `/tmp/gateway-ocsf.jsonl`, is writable in the gateway container with either the StatefulSet or Deployment workload but does not persist across pod restarts. To keep records, mount a volume with `server.extraVolumes` and `server.extraVolumeMounts` and set the path on it. When replicas share a volume, add `subPathExpr: $(OPENSHELL_POD_NAME)` to the mount so each replica writes its own file. +For Helm installations, set `server.ocsfLog.enabled` to render this table. The default `server.ocsfLog.path`, `/tmp/gateway-ocsf.jsonl`, is writable in the gateway container with either the StatefulSet or Deployment workload but does not persist across pod restarts. To keep records, mount a volume with `server.extraVolumes` and `server.extraVolumeMounts` and set the path on it. When replicas share a volume, add `subPathExpr: $(OPENSHELL_POD_NAME)` to the mount so each replica writes its own file. Give each gateway replica its own writable file and an external shipper read access to the containing directory. New files use owner-only permissions on Unix; arrange shipper access deliberately. Rotation renames the active file to a date- and UUID-suffixed sibling. The shipper must follow rotated files and checkpoint its progress. Do not use multiple writers or external copy-truncate rotation on this path. @@ -621,23 +620,24 @@ In-process compute drivers read their backend-specific settings from `[openshell ### OCSF Log Export -Set `enabled = true` in `[openshell.gateway.otlp.ocsf]` to export gateway OCSF events as OTLP log records. It defaults to `false`, so trace export alone never sends OCSF events. Records go to the table's `endpoint`, or to `[openshell.gateway.otlp]` `endpoint` when it is omitted, with the same resource attributes as spans, under the `openshell.ocsf` instrumentation scope. Use a separate endpoint to send OCSF events to a collector that feeds a SIEM while spans go elsewhere. Sandbox supervisors receive the same endpoint and schema version and export their own OCSF events there directly, so the endpoint must also be reachable from sandboxes. Supervisor records carry the supervisor's resource attributes, including `openshell.sandbox.id` and `openshell.gateway.name`. Supervisors also send the collector headers that the gateway uses for OCSF export, from `OTEL_EXPORTER_OTLP_LOGS_HEADERS` or else `OTEL_EXPORTER_OTLP_HEADERS`, so a collector that requires authentication accepts their records too. The gateway delivers the headers with each sandbox's launch credentials, never through the sandbox environment. Other `OTEL_*` variables do not reach supervisors. +Set `enabled = true` in `[openshell.gateway.otlp.ocsf]` to export gateway OCSF events as OTLP log records. It defaults to `false`, so trace export alone never sends OCSF events. Records go to the table's `endpoint`, or to `[openshell.gateway.otlp]` `endpoint` when it is omitted, with the same resource attributes as spans, under the `openshell.ocsf` instrumentation scope. Use a separate endpoint to send OCSF events to a collector that feeds a SIEM while spans go elsewhere. Sandbox supervisors receive the same endpoint and export their own OCSF events there directly, so the endpoint must also be reachable from sandboxes. Supervisor records carry the supervisor's resource attributes, including `openshell.sandbox.id` and `openshell.gateway.name`. Supervisors also send the collector headers that the gateway uses for OCSF export, from `OTEL_EXPORTER_OTLP_LOGS_HEADERS` or else `OTEL_EXPORTER_OTLP_HEADERS`, so a collector that requires authentication accepts their records too. The gateway delivers the headers with each sandbox's launch credentials, never through the sandbox environment. Other `OTEL_*` variables do not reach supervisors. ```toml [openshell.gateway.otlp.ocsf] enabled = true -endpoint = "http://siem-collector.security.svc:4317" # optional -schema_version = "1.3" # optional +endpoint = "http://siem-collector.security.svc:4317" # optional ``` Each OCSF event becomes one log record: | Field | Value | |---|---| -| Body | The OCSF event as a JSON string. With `schema_version` set to `1.1` or `1.3`, the record is downgraded as for [OCSF JSONL output](#ocsf-jsonl-output). | +| Body | The OCSF event as a JSON string. | | Timestamp | The OCSF `time`. The observed timestamp is when the gateway received the event. | | Severity | Unknown, Informational, and Other are INFO, Low is WARN, Medium is WARN2, High is ERROR, Critical is ERROR2, and Fatal is FATAL. The severity text is the OCSF severity name. | -| Attributes | `ocsf.class_uid`, `ocsf.activity_id`, `ocsf.severity_id`, `ocsf.metadata.uid`, and, for events associated with a sandbox, `openshell.sandbox.id`. These come from the event before any downgrade, so they remain when a downgrade removes the fields from the body. | +| Attributes | `ocsf.class_uid`, `ocsf.activity_id`, `ocsf.severity_id`, `ocsf.metadata.uid`, and, for events associated with a sandbox, `openshell.sandbox.id`. | + +`schema_version` in this table is deprecated and ignored; the gateway logs a warning at startup when it is set. Log export is independent of `RUST_LOG` and of `[openshell.gateway.ocsf_log]`; you can enable both. The gateway queues up to 10,000 records, and up to 16 MiB of records including the batch being exported, without blocking event producers. It exports a batch as soon as 100 records or 1 MiB are queued, or 500 ms after the first record of a batch arrives. A record larger than 1 MiB, such as an event for a request with a very long path, is dropped rather than exported. @@ -669,7 +669,7 @@ The relayed hop does not yet support TLS or authentication to the collector; an ### Tuning -`[openshell.gateway.otlp]` decides whether and where to export. How the SDK exports is controlled by the standard OpenTelemetry environment variables, which the gateway reads through the SDK rather than mirroring as TOML keys. They apply to the gateway and its in-process compute drivers. The gateway does not pass them to sandbox supervisors, which receive only the endpoints, the OCSF schema version, and the [OCSF collector headers](#ocsf-log-export). +`[openshell.gateway.otlp]` decides whether and where to export. How the SDK exports is controlled by the standard OpenTelemetry environment variables, which the gateway reads through the SDK rather than mirroring as TOML keys. They apply to the gateway and its in-process compute drivers. The gateway does not pass them to sandbox supervisors, which receive only the endpoints and the [OCSF collector headers](#ocsf-log-export). The gateway honors these variables: diff --git a/docs/images/splunk-cim-v1.1-downgrade.png b/docs/images/splunk-cim-v1.1-downgrade.png deleted file mode 100644 index feb8e2168..000000000 Binary files a/docs/images/splunk-cim-v1.1-downgrade.png and /dev/null differ diff --git a/docs/observability/ocsf-json-export.mdx b/docs/observability/ocsf-json-export.mdx index cdc5f9418..f6c5d306c 100644 --- a/docs/observability/ocsf-json-export.mdx +++ b/docs/observability/ocsf-json-export.mdx @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 title: "OCSF JSON Export" sidebar-title: "OCSF JSON Export" -description: "How to enable full OCSF JSON logging for SIEM integration, compliance, and structured analysis. Includes schema version downgrade for AWS Security Lake, Splunk, and CrowdStrike compatibility." +description: "How to enable full OCSF JSON logging for SIEM integration, compliance, and structured analysis." keywords: "Generative AI, Cybersecurity, OCSF, JSON, SIEM, Compliance, Observability" --- @@ -119,7 +119,7 @@ When `ocsf_json_enabled=true`, sandbox supervisors emit full OCSF JSON to stderr 2026-10-07T23:21:19.000Z OCSF-JSON {"class_uid":4001,...} ``` -The example payload is abbreviated. Each actual `OCSF-JSON` line contains one complete compact JSON object. Configure your collector to match the `OCSF-JSON` token after the UTC timestamp and decode everything after the following space as JSON. Select these lines to avoid ingesting the shorthand copy of each event. The JSON preserves the original `metadata.uid` and follows `ocsf_schema_version`; with that setting unset, it uses the full native schema. +The example payload is abbreviated. Each actual `OCSF-JSON` line contains one complete compact JSON object. Configure your collector to match the `OCSF-JSON` token after the UTC timestamp and decode everything after the following space as JSON. Select these lines to avoid ingesting the shorthand copy of each event. The JSON preserves the original `metadata.uid`. Collect stderr through the supervisor's platform log interface: @@ -153,7 +153,7 @@ Some fields depend on the event class: - Process Activity records from the sandbox name the `openshell-sandbox` launcher as `actor.process`. - `connection_info` carries `direction_id` and `direction`. OpenShell reports Unknown (`0`) where it does not track the direction. -Each JSONL file line, or the payload after the stderr marker, is a complete OCSF v1.8.0 JSON object when no downgrade is configured. Records validate against OCSF v1.8.0 except for the top-level `container` object, which OCSF does not define at the event level. Here is an example of a network connection event: +Each JSONL file line, or the payload after the stderr marker, is a complete OCSF v1.8.0 JSON object. Records validate against OCSF v1.8.0 except for the top-level `container` object, which OCSF does not define at the event level. Here is an example of a network connection event: ```json { @@ -310,66 +310,9 @@ failures without an observed network endpoint, use Base Event (`class_uid: 0`). Include these classes when filtering supervisor operational events. Relay-control records that previously appeared as `NET:*` now appear as `EVENT`. -## SIEM Schema Version Compatibility +## Schema Version -OpenShell emits OCSF v1.8.0 events internally, but many SIEMs only support older schema versions. The `ocsf_schema_version` setting tells the JSONL layer to downgrade events before writing, stripping fields and profiles that don't exist in the target version. - -For gateway JSONL, set `schema_version` on the output destination. The target applies uniformly to every record written to that file and requires a gateway restart: - -```toml -[openshell.gateway.ocsf_log] -path = "/var/log/openshell/gateway-ocsf.jsonl" -schema_version = "1.3" -``` - -The sandbox-scoped setting below controls the legacy supervisor-local JSONL file only. - -Set the target version globally: - -```shell -openshell settings set --global --key ocsf_schema_version --value "1.1" -``` - -Or per-sandbox: - -```shell -openshell settings set my-sandbox --key ocsf_schema_version --value "1.3" -``` - -The setting takes effect on the next poll cycle, by default every 10 seconds. No sandbox restart is required. - -Supported target versions are `1.1` and `1.3`. When set, the JSONL layer applies the following transformations: - -- Strips fields added after the target version: `ai_model`, `container`, `observation_point_id` -- Removes unknown profile names from `metadata.profiles`: `ai_operation`, `container` -- Rewrites `metadata.version` to match the target version - -The shorthand log output is unaffected. The internal event model stays at v1.8.0; only the serialized JSONL is transformed. - -When unset or empty, no downgrade is applied and events are written at the current schema version. - -| SIEM | Required OCSF Version | `ocsf_schema_version` Value | -|---|---|---| -| AWS Security Lake | v1.1.0 | `1.1` | -| Splunk CIM Add-On | v1.1-v1.3 | `1.1` or `1.3` | -| CrowdStrike FDR | v1.5.0 | Not yet supported | -| Datadog Cloud SIEM | v1.5.0 (selectable) | Not yet supported | - - -Downgrading is lossy. The `ai_model` field and `ai_operation` profile are stripped for v1.1 and v1.3 targets, so AI model attribution is not present in downgraded events. If your audit workflow requires knowing which model handled a request, use the full schema version (leave `ocsf_schema_version` unset). OpenShell supports OCSF v1.8.0 natively; upgrade your SIEM's OCSF schema if you need AI attribution alongside backward-compatible ingestion. - - -Downgraded events include a `downgraded_from` marker in the `unmapped` object so auditors can distinguish "no model was involved" from "model attribution was stripped": - -```json -{ - "unmapped": { - "downgraded_from": "1.8.0" - } -} -``` - -The core OCSF event structure (class UIDs, activity IDs, network and HTTP fields) is identical across v1.1 through v1.8. The stripped fields are all profile-gated additions. +OpenShell emits OCSF v1.8.0 only. The `ocsf_schema_version` sandbox setting and the gateway `schema_version` fields under `[openshell.gateway.ocsf_log]` and `[openshell.gateway.otlp.ocsf]` are deprecated and ignored; remove them from your configuration. The gateway logs a warning at startup for each field that is set, and a sandbox reports a set `ocsf_schema_version` as a Device Config State Change event. ## Integration with External Tools @@ -377,9 +320,9 @@ The JSONL file can be shipped to any tool that accepts OCSF-formatted data: | Tool | Integration Path | |---|---| -| Splunk | Use the [Splunk OCSF Add-on](https://splunkbase.splunk.com/app/6943) to ingest OCSF JSONL files. Set `ocsf_schema_version` to `1.3` for CIM Add-On compatibility. | -| Amazon Security Lake | OCSF is the native schema for Security Lake. Set `ocsf_schema_version` to `1.1` for v1.1.0 compatibility. | -| CrowdStrike FDR | Ship JSONL files to Falcon Data Replicator. v1.5 downgrade target is not yet supported. | +| Splunk | Use the [Splunk OCSF Add-on](https://splunkbase.splunk.com/app/6943) to ingest OCSF JSONL files. | +| Amazon Security Lake | OCSF is the native schema for Security Lake. | +| CrowdStrike FDR | Ship JSONL files to Falcon Data Replicator. | | Elastic | Use Filebeat to ship JSONL files with the OCSF field mappings. | | Custom pipelines | Parse the JSONL file with `jq`, Python, or any JSON-capable tool. | @@ -393,7 +336,7 @@ jq -c 'select(.action == "Denied")' ./supervisor-logs/openshell-ocsf.*.jsonl Sandbox events identify the sandbox environment with `device.type_id: 99` (Other) and `device.type: "Sandbox"`. The operating system is reported separately in `device.os.name`; the device type does not classify the underlying host or compute backend. -The shorthand format in `openshell.YYYY-MM-DD.log` and the JSON format in `openshell-ocsf.YYYY-MM-DD.jsonl` are derived from the same OCSF events. The shorthand is a human-readable projection; the JSON is the full structured record when no schema downgrade is configured. When `ocsf_schema_version` is set, the JSON export is a lossy projection of the internal event model. Both formats are generated at the same time from the same event data. +The shorthand format in `openshell.YYYY-MM-DD.log` and the JSON format in `openshell-ocsf.YYYY-MM-DD.jsonl` are derived from the same OCSF events. The shorthand is a human-readable projection; the JSON is the full structured record. Both formats are generated at the same time from the same event data. Shorthand stderr output follows the diagnostic log filter; its local file is available when file logging starts successfully. JSON stderr and file export are opt-in through `ocsf_json_enabled`. File export requires the supervisor's JSONL sink to start successfully; stderr export is independent of that sink. diff --git a/docs/observability/overview.mdx b/docs/observability/overview.mdx index 1bef85662..73ae96470 100644 --- a/docs/observability/overview.mdx +++ b/docs/observability/overview.mdx @@ -77,7 +77,7 @@ path = "/var/log/openshell/gateway-ocsf.jsonl" openshell settings set --global --key ocsf_json_enabled --value true ``` -For the record structure, file locations, rotation, and schema version downgrades for older SIEMs, refer to [OCSF JSON Export](/observability/ocsf-json-export). +For the record structure, file locations, and rotation, refer to [OCSF JSON Export](/observability/ocsf-json-export). ### OTLP Log Export diff --git a/e2e/rust/tests/ocsf_jsonl_conformance.rs b/e2e/rust/tests/ocsf_jsonl_conformance.rs index 64b26219a..130d26572 100644 --- a/e2e/rust/tests/ocsf_jsonl_conformance.rs +++ b/e2e/rust/tests/ocsf_jsonl_conformance.rs @@ -229,17 +229,12 @@ async fn supervisor_jsonl_conforms_to_the_native_schema() { ); let mut failures = Vec::new(); for record in &records { - match openshell_ocsf_validate::validate_event(record) { - Ok(()) => {} - Err(openshell_ocsf_validate::Error::UnsupportedVersion(version)) => panic!( - "record labelled {version}; native validation needs ocsf_schema_version unset \ - for this sandbox and the gateway" - ), - Err(reason) => failures.push(format!( + if let Err(reason) = openshell_ocsf_validate::validate_event(record) { + failures.push(format!( "{} \"{}\": {reason}", record["class_name"], record["message"].as_str().unwrap_or_default() - )), + )); } } assert!( diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 29b440756..264d4ca18 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1493,7 +1493,7 @@ credential failures. |---|---|---| | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs | | Collector receives no OCSF log records, or `openshell.ocsf.gap` records | `[openshell.gateway.otlp.ocsf]` missing or without `enabled = true`, an unreachable or non-gRPC endpoint, or export failures and queue pressure | Check `[openshell.gateway.otlp.ocsf]` for `enabled = true` and the endpoint it resolves to, the gateway's `OTLP exporting enabled` startup log for `ocsf_endpoint`, export warnings, and `openshell_ocsf_otlp_*` metrics. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). | -| Gateway OCSF JSONL stops growing, has gaps, or is rejected by a SIEM | File errors, queue pressure, shipper falling behind retention, or a schema mismatch | Inspect gateway warnings and `openshell_ocsf_log_*` metrics; check `[openshell.gateway.ocsf_log]`, `schema_version`, directory permissions, free space, per-replica paths, and shipper rotation checkpoints. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). | +| Gateway OCSF JSONL stops growing, has gaps, or is rejected by a SIEM | File errors, queue pressure, or shipper falling behind retention | Inspect gateway warnings and `openshell_ocsf_log_*` metrics; check `[openshell.gateway.ocsf_log]`, directory permissions, free space, per-replica paths, and shipper rotation checkpoints. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). | | `BatchSpanProcessor.ExportError` repeatedly reports connection refused on `127.0.0.1:4317` | The local gateway started with OTLP configured but the collector forwarding task later stopped, or the config was created manually | Restart `gateway:docker`, `gateway:podman`, or `gateway:vm` so it re-detects the listener; inspect the generated `gateway.toml` for `[openshell.gateway.otlp]` | | Sandbox log shows `OTLP agent trace relay inactive: not enabled by the gateway` at startup, then `agent OTLP export attempted but the agent trace relay is not enabled` when the agent exports | The gateway does not enable `[openshell.gateway.otlp.agent]`, so the supervisor refuses the workload's OTLP relay address | Set `enabled = true` in `[openshell.gateway.otlp.agent]` (Helm: `gatewayConfig."openshell.gateway.otlp.agent".enabled`) with an endpoint from that table or `[openshell.gateway.otlp]`, and recreate the sandbox; `openshell logs --source sandbox` then shows `OTLP agent trace relay enabled` | | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs | diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index f87e06fdf..a950b3a3b 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -922,9 +922,6 @@ openshell settings delete work-session --key ocsf_json_enabled openshell settings get --global --json openshell settings set --global --key ocsf_json_enabled --value true - -# OCSF schema version downgrade for SIEM compatibility (allowed: "1.1", "1.3") -openshell settings set --global --key ocsf_schema_version --value "1.1" ``` Enabling `ocsf_json_enabled` adds compact `OCSF-JSON` records to the supervisor's stderr, alongside shorthand, and to its JSONL file when file storage is available. Collect these records through the supervisor's platform logs (`docker logs`, `podman logs`, or `kubectl logs`); `openshell logs` still serves shorthand. Match the `OCSF-JSON` marker after the timestamp and parse the remainder as JSON. See the published [OCSF JSON export guide](https://docs.nvidia.com/openshell/latest/observability/ocsf-json-export) for delivery and retention limits.