diff --git a/.agents/skills/openshell-cli/SKILL.md b/.agents/skills/openshell-cli/SKILL.md index 462e27f3f2..093f5d2417 100644 --- a/.agents/skills/openshell-cli/SKILL.md +++ b/.agents/skills/openshell-cli/SKILL.md @@ -622,6 +622,9 @@ openshell settings delete work-session --key ocsf_json_enabled openshell settings get --global --json openshell settings set --global --key providers_v2_enabled --value true + +# OCSF schema version downgrade for SIEM compatibility (allowed: "1.1", "1.3") +openshell settings set --global --key ocsf_schema_version --value "1.1" ``` Global mutations prompt for confirmation. Use `--yes` only in reviewed automation. diff --git a/crates/openshell-core/src/settings.rs b/crates/openshell-core/src/settings.rs index 156e4c3845..1adfa95fae 100644 --- a/crates/openshell-core/src/settings.rs +++ b/crates/openshell-core/src/settings.rs @@ -107,6 +107,14 @@ pub const PROPOSAL_APPROVAL_MODE_KEY: &str = "proposal_approval_mode"; /// fail-closes on unknown persisted values for defense in depth. pub const PROPOSAL_APPROVAL_MODE_VALUES: &[&str] = &["manual", "auto"]; +/// Allowed values for `ocsf_schema_version`. +/// +/// Only versions with actual downgrade transforms in +/// `openshell_ocsf::format::downgrade` are accepted. Empty string disables +/// downgrade (equivalent to unsetting the key). Malformed or unsupported +/// versions (e.g. `"banana"`, `"1.6"`) are rejected at configure time. +pub const OCSF_SCHEMA_VERSION_VALUES: &[&str] = &["", "1.1", "1.3"]; + pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[ // Gateway-level opt-in for provider profile policy composition. Defaults // to false when unset. @@ -123,6 +131,14 @@ pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[ kind: SettingValueKind::Bool, allowed_string_values: None, }, + // Target OCSF schema version for JSONL downgrade. When set (e.g. "1.1" + // or "1.3"), the JSONL layer strips fields and profiles that don't exist + // in the target version. Empty or unset means no downgrade. + RegisteredSetting { + key: "ocsf_schema_version", + kind: SettingValueKind::String, + allowed_string_values: Some(OCSF_SCHEMA_VERSION_VALUES), + }, // Sandbox-level opt-in for the agent-driven policy proposal surface. // See AGENT_POLICY_PROPOSALS_ENABLED_KEY for details. Defaults to false. RegisteredSetting { @@ -168,9 +184,9 @@ pub fn parse_bool_like(raw: &str) -> Option { #[cfg(test)] mod tests { use super::{ - PROPOSAL_APPROVAL_MODE_KEY, PROPOSAL_APPROVAL_MODE_VALUES, PROVIDERS_V2_ENABLED_KEY, - REGISTERED_SETTINGS, RegisteredSetting, SettingValueKind, parse_bool_like, - registered_keys_csv, setting_for_key, + OCSF_SCHEMA_VERSION_VALUES, PROPOSAL_APPROVAL_MODE_KEY, PROPOSAL_APPROVAL_MODE_VALUES, + PROVIDERS_V2_ENABLED_KEY, REGISTERED_SETTINGS, RegisteredSetting, SettingValueKind, + parse_bool_like, registered_keys_csv, setting_for_key, }; #[test] @@ -237,6 +253,36 @@ mod tests { } } + // ---- ocsf_schema_version validation ---- + + #[test] + fn ocsf_schema_version_accepts_supported_versions() { + let setting = setting_for_key("ocsf_schema_version") + .expect("ocsf_schema_version should be registered"); + assert_eq!(setting.kind, SettingValueKind::String); + assert_eq!( + setting.allowed_string_values, + Some(OCSF_SCHEMA_VERSION_VALUES) + ); + assert!(setting.validate_string_value("").is_ok()); + assert!(setting.validate_string_value("1.1").is_ok()); + assert!(setting.validate_string_value("1.3").is_ok()); + } + + #[test] + fn ocsf_schema_version_rejects_malformed_and_unsupported() { + let setting = setting_for_key("ocsf_schema_version") + .expect("ocsf_schema_version should be registered"); + for bad in [ + "banana", "1.6", "1.5", "1.7", "1.7.0", "1.1.0", "2.0", " 1.1", "1.1 ", "v1.1", + ] { + let err = setting + .validate_string_value(bad) + .expect_err(&format!("expected '{bad}' to be rejected")); + assert_eq!(err, OCSF_SCHEMA_VERSION_VALUES); + } + } + // ---- parse_bool_like ---- #[test] diff --git a/crates/openshell-ocsf/src/format/downgrade.rs b/crates/openshell-ocsf/src/format/downgrade.rs new file mode 100644 index 0000000000..e75caefe07 --- /dev/null +++ b/crates/openshell-ocsf/src/format/downgrade.rs @@ -0,0 +1,183 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! OCSF schema version downgrade filter. +//! +//! Transforms serialized OCSF JSON events to conform to older schema versions +//! by stripping fields and profiles that don't exist in the target version. + +use serde_json::Value; + +/// Fields to strip when downgrading to v1.3.0 or earlier. +const STRIP_FOR_V1_3: &[&str] = &["ai_model", "container", "observation_point_id"]; + +/// Profile names to remove from `metadata.profiles` when downgrading to v1.3.0 or earlier. +const STRIP_PROFILES_V1_3: &[&str] = &["ai_operation", "container"]; + +/// Downgrade a serialized OCSF event to the target schema version. +/// +/// Modifies the JSON in place: strips fields that don't exist in the target +/// version, removes unknown profile names from `metadata.profiles`, and +/// rewrites `metadata.version` to match. +/// +/// Returns `true` if the event was modified, `false` if no changes were needed +/// (target is current version or newer). +pub fn downgrade_event(event: &mut Value, target_version: &str) -> bool { + let target = parse_version(target_version); + let v1_3 = (1, 3, 0); + + if target >= parse_version(crate::OCSF_VERSION) { + return false; + } + + let Some(obj) = event.as_object_mut() else { + return false; + }; + + let mut modified = false; + + if target <= v1_3 { + for field in STRIP_FOR_V1_3 { + if obj.remove(*field).is_some() { + modified = true; + } + } + + if let Some(profiles) = obj + .get_mut("metadata") + .and_then(Value::as_object_mut) + .and_then(|m| m.get_mut("profiles")) + .and_then(Value::as_array_mut) + { + let before = profiles.len(); + profiles.retain(|p| !p.as_str().is_some_and(|s| STRIP_PROFILES_V1_3.contains(&s))); + if profiles.len() != before { + modified = true; + } + } + } + + if modified && let Some(metadata) = obj.get_mut("metadata").and_then(Value::as_object_mut) { + metadata.insert( + "version".to_string(), + Value::String(target_version.to_string()), + ); + } + + modified +} + +fn parse_version(v: &str) -> (u32, u32, u32) { + let parts: Vec = v.split('.').filter_map(|s| s.parse().ok()).collect(); + ( + parts.first().copied().unwrap_or(0), + parts.get(1).copied().unwrap_or(0), + parts.get(2).copied().unwrap_or(0), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_event() -> Value { + serde_json::json!({ + "class_uid": 4002, + "class_name": "HTTP Activity", + "time": 1_234_567_890, + "severity_id": 1, + "metadata": { + "version": "1.7.0", + "profiles": ["security_control", "network_proxy", "container", "host"] + }, + "device": {"hostname": "sandbox-1"}, + "container": {"name": "test-sandbox"}, + "observation_point_id": 2, + "unmapped": {"key": "value"} + }) + } + + #[test] + fn test_downgrade_to_v1_3_strips_fields() { + let mut event = test_event(); + let modified = downgrade_event(&mut event, "1.3.0"); + + assert!(modified); + assert!(event.get("container").is_none()); + assert!(event.get("observation_point_id").is_none()); + assert!(event.get("device").is_some()); + assert!(event.get("unmapped").is_some()); + } + + #[test] + fn test_downgrade_to_v1_1_strips_fields() { + let mut event = test_event(); + let modified = downgrade_event(&mut event, "1.1.0"); + + assert!(modified); + assert!(event.get("container").is_none()); + assert!(event.get("observation_point_id").is_none()); + } + + #[test] + fn test_downgrade_strips_profiles() { + let mut event = test_event(); + downgrade_event(&mut event, "1.3.0"); + + let profiles = event["metadata"]["profiles"].as_array().unwrap(); + assert!(!profiles.iter().any(|p| p == "container")); + assert!(profiles.iter().any(|p| p == "security_control")); + assert!(profiles.iter().any(|p| p == "host")); + } + + #[test] + fn test_downgrade_rewrites_version() { + let mut event = test_event(); + downgrade_event(&mut event, "1.1.0"); + + assert_eq!(event["metadata"]["version"], "1.1.0"); + } + + #[test] + fn test_no_downgrade_for_current_version() { + let mut event = test_event(); + let modified = downgrade_event(&mut event, "1.7.0"); + + assert!(!modified); + assert_eq!(event["metadata"]["version"], "1.7.0"); + } + + #[test] + fn test_no_downgrade_for_newer_version() { + let mut event = test_event(); + let modified = downgrade_event(&mut event, "1.9.0"); + + assert!(!modified); + } + + #[test] + fn test_downgrade_strips_ai_model_when_present() { + let mut event = serde_json::json!({ + "class_uid": 6003, + "metadata": { + "version": "1.8.0", + "profiles": ["container", "host", "ai_operation"] + }, + "ai_model": {"name": "claude-3-haiku", "ai_provider": "anthropic"}, + "unmapped": {"latency_ms": 701} + }); + let modified = downgrade_event(&mut event, "1.3.0"); + + assert!(modified); + assert!(event.get("ai_model").is_none()); + assert!( + !event["metadata"]["profiles"] + .as_array() + .unwrap() + .iter() + .any(|p| p == "ai_operation") + ); + assert_eq!(event["metadata"]["version"], "1.3.0"); + assert!(event.get("unmapped").is_some()); + } +} diff --git a/crates/openshell-ocsf/src/format/mod.rs b/crates/openshell-ocsf/src/format/mod.rs index 084a013d94..17518a5617 100644 --- a/crates/openshell-ocsf/src/format/mod.rs +++ b/crates/openshell-ocsf/src/format/mod.rs @@ -3,5 +3,6 @@ //! OCSF event formatters: shorthand (human-readable) and JSONL. +pub mod downgrade; pub mod jsonl; pub mod shorthand; diff --git a/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs b/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs index 920483700a..9414c74172 100644 --- a/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs +++ b/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs @@ -12,6 +12,7 @@ use tracing::Subscriber; use tracing_subscriber::Layer; use tracing_subscriber::layer::Context; +use crate::format::downgrade::downgrade_event; use crate::tracing_layers::event_bridge::{OCSF_TARGET, clone_current_event}; /// A tracing `Layer` that intercepts OCSF events and writes JSONL output. @@ -23,9 +24,15 @@ use crate::tracing_layers::event_bridge::{OCSF_TARGET, clone_current_event}; /// `false`, the layer short-circuits without writing. This allows the sandbox /// to hot-toggle OCSF JSONL output at runtime via the `ocsf_json_enabled` /// setting without rebuilding the subscriber. +/// +/// An optional target schema version can be set via +/// [`with_target_version`](Self::with_target_version). When set, events are +/// downgraded to the target version before writing (stripping fields and +/// profiles that don't exist in older schema versions). pub struct OcsfJsonlLayer { writer: Mutex, enabled: Option>, + target_version: Option>>, } impl OcsfJsonlLayer { @@ -35,6 +42,7 @@ impl OcsfJsonlLayer { Self { writer: Mutex::new(writer), enabled: None, + target_version: None, } } @@ -47,6 +55,16 @@ impl OcsfJsonlLayer { self.enabled = Some(flag); self } + + /// Attach a shared target schema version for downgrade filtering. + /// + /// When set, events are downgraded to the target version before writing. + /// The version can be changed at runtime via the shared mutex. + #[must_use] + pub fn with_target_version(mut self, version: Arc>) -> Self { + self.target_version = Some(version); + self + } } impl Layer for OcsfJsonlLayer @@ -67,9 +85,29 @@ where } if let Some(ocsf_event) = clone_current_event() - && let Ok(line) = ocsf_event.to_json_line() && let Ok(mut w) = self.writer.lock() { + let line = if let Some(ref target) = self.target_version + && let Ok(version) = target.lock() + && !version.is_empty() + { + let Ok(mut json) = serde_json::to_value(&ocsf_event) else { + return; + }; + downgrade_event(&mut json, &version); + match serde_json::to_string(&json) { + Ok(mut s) => { + s.push('\n'); + s + } + Err(_) => return, + } + } else { + match ocsf_event.to_json_line() { + Ok(l) => l, + Err(_) => return, + } + }; let _ = w.write_all(line.as_bytes()); } } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 0d26067f6b..e82c1a71fe 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -101,6 +101,7 @@ pub async fn run_sandbox( _health_port: u16, inference_routes: Option, ocsf_enabled: Arc, + ocsf_schema_version: Arc>, network_enabled: bool, process_enabled: bool, upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, @@ -608,6 +609,7 @@ pub async fn run_sandbox( let poll_endpoint = endpoint.to_string(); let poll_engine = engine.clone(); let poll_ocsf_enabled = ocsf_enabled.clone(); + let poll_ocsf_schema_version = ocsf_schema_version.clone(); let poll_pid = entrypoint_pid.clone(); let poll_provider_credentials = provider_credentials.clone(); let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); @@ -623,6 +625,7 @@ pub async fn run_sandbox( entrypoint_pid: poll_pid, interval_secs: poll_interval_secs, ocsf_enabled: poll_ocsf_enabled, + ocsf_schema_version: poll_ocsf_schema_version, provider_credentials: poll_provider_credentials, policy_local_ctx: poll_policy_local, agent_proposals: agent_proposals.clone(), @@ -2767,6 +2770,7 @@ struct PolicyPollLoopContext { entrypoint_pid: Arc, interval_secs: u64, ocsf_enabled: Arc, + ocsf_schema_version: Arc>, provider_credentials: ProviderCredentialState, policy_local_ctx: Option>, agent_proposals: AgentProposals, @@ -3098,6 +3102,7 @@ async fn run_policy_poll_loop_with_client( match initial_poll_disposition(&ctx.loaded_policy_origin, &result) { InitialPollDisposition::Acknowledge(candidate) => { apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); apply_agent_proposals_enabled( &ctx.agent_proposals, agent_proposals_enabled_from_settings(&result.settings), @@ -3123,6 +3128,7 @@ async fn run_policy_poll_loop_with_client( InitialPollDisposition::Reconcile => pending_result = Some(result), InitialPollDisposition::TrackOnly => { apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); apply_agent_proposals_enabled( &ctx.agent_proposals, agent_proposals_enabled_from_settings(&result.settings), @@ -3532,6 +3538,7 @@ async fn run_policy_poll_loop_with_client( // Apply OCSF JSON toggle from the `ocsf_json_enabled` setting. apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); // Apply the agent-proposals feature toggle. On a false→true transition // we lazily install the skill so a sandbox that started with the flag @@ -3586,6 +3593,37 @@ fn extract_bool_setting( }) } +fn apply_ocsf_schema_version_setting( + version: &std::sync::Mutex, + settings: &std::collections::HashMap, +) { + let new_version = extract_string_setting(settings, "ocsf_schema_version").unwrap_or_default(); + if let Ok(mut current) = version.lock() + && *current != new_version + { + info!( + ocsf_schema_version = %new_version, + "OCSF schema version target changed" + ); + *current = new_version; + } +} + +fn extract_string_setting( + settings: &std::collections::HashMap, + key: &str, +) -> Option { + use openshell_core::proto::setting_value; + settings + .get(key) + .and_then(|es| es.value.as_ref()) + .and_then(|sv| sv.value.as_ref()) + .and_then(|v| match v { + setting_value::Value::StringValue(s) => Some(s.clone()), + _ => None, + }) +} + fn agent_proposals_enabled_from_settings( settings: &std::collections::HashMap, ) -> bool { @@ -4135,6 +4173,7 @@ filesystem_policy: entrypoint_pid: Arc::new(AtomicU32::new(0)), interval_secs: 0, ocsf_enabled: Arc::new(AtomicBool::new(false)), + ocsf_schema_version: Arc::new(std::sync::Mutex::new(String::new())), provider_credentials: ProviderCredentialState::from_child_env_snapshot( 0, std::collections::HashMap::new(), diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 98af7f9ea9..2a73075359 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -588,6 +588,7 @@ fn main() -> Result<()> { // `ocsf_json_enabled` setting changes. The JSONL layer checks it // on each event and short-circuits when false. let ocsf_enabled = Arc::new(AtomicBool::new(false)); + let ocsf_schema_version = Arc::new(std::sync::Mutex::new(String::new())); // Keep guards alive for the entire process. When a guard is dropped the // non-blocking writer flushes remaining logs. @@ -606,7 +607,9 @@ fn main() -> Result<()> { .ok() .map(|roller| { let (writer, guard) = tracing_appender::non_blocking(roller); - let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone()); + let layer = OcsfJsonlLayer::new(writer) + .with_enabled_flag(ocsf_enabled.clone()) + .with_target_version(ocsf_schema_version.clone()); (layer, guard) }); let (jsonl_layer, jsonl_guard) = match jsonl_logging { @@ -681,6 +684,7 @@ fn main() -> Result<()> { args.health_port, args.inference_routes, ocsf_enabled, + ocsf_schema_version, args.mode.network, args.mode.process, upstream_proxy_args, diff --git a/docs/images/splunk-cim-v1.1-downgrade.png b/docs/images/splunk-cim-v1.1-downgrade.png new file mode 100644 index 0000000000..feb8e21683 Binary files /dev/null and b/docs/images/splunk-cim-v1.1-downgrade.png differ diff --git a/docs/observability/ocsf-json-export.mdx b/docs/observability/ocsf-json-export.mdx index 5c326d665c..baf09425ca 100644 --- a/docs/observability/ocsf-json-export.mdx +++ b/docs/observability/ocsf-json-export.mdx @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 title: "OCSF JSON Export" sidebar-title: "OCSF JSON Export" -description: "How to enable full OCSF JSON logging for SIEM integration, compliance, and structured analysis." +description: "How to enable full OCSF JSON logging for SIEM integration, compliance, and structured analysis. Includes schema version downgrade for AWS Security Lake, Splunk, and CrowdStrike compatibility." keywords: "Generative AI, Cybersecurity, OCSF, JSON, SIEM, Compliance, Observability" --- @@ -141,14 +141,54 @@ The `class_uid` field identifies the event type: | 5019 | Device Config State Change | `CONFIG:` | | 6002 | Application Lifecycle | `LIFECYCLE:` | +## SIEM Schema Version Compatibility + +OpenShell emits OCSF v1.7.0 events internally, but many SIEMs only support older schema versions. The `ocsf_schema_version` setting tells the JSONL layer to downgrade events before writing, stripping fields and profiles that don't exist in the target version. + +Set the target version globally: + +```shell +openshell settings set --global --key ocsf_schema_version --value "1.1" +``` + +Or per-sandbox: + +```shell +openshell settings set my-sandbox --key ocsf_schema_version --value "1.3" +``` + +The setting takes effect on the next poll cycle, by default every 10 seconds. No sandbox restart is required. + +Supported target versions are `1.1` and `1.3`. When set, the JSONL layer applies the following transformations: + +- Strips fields added after the target version: `ai_model`, `container`, `observation_point_id` +- Removes unknown profile names from `metadata.profiles`: `ai_operation`, `container` +- Rewrites `metadata.version` to match the target version + +The shorthand log output is unaffected. The internal event model stays at v1.7.0; only the serialized JSONL is transformed. + +When unset or empty, no downgrade is applied and events are written at the current schema version. + +| SIEM | Required OCSF Version | `ocsf_schema_version` Value | +|---|---|---| +| AWS Security Lake | v1.1.0 | `1.1` | +| Splunk CIM Add-On | v1.1-v1.3 | `1.1` or `1.3` | +| CrowdStrike FDR | v1.5.0 | Not yet supported | +| Datadog Cloud SIEM | v1.5.0 (selectable) | Not yet supported | + + +The core OCSF event structure (class UIDs, activity IDs, HTTP/network fields) is identical across v1.1 through v1.7. The differences are all profile-gated additions, so downgrading does not lose security-relevant data. + + ## Integration with External Tools The JSONL file can be shipped to any tool that accepts OCSF-formatted data: | Tool | Integration Path | |---|---| -| Splunk | Use the [Splunk OCSF Add-on](https://splunkbase.splunk.com/app/6943) to ingest OCSF JSONL files. | -| Amazon Security Lake | OCSF is the native schema for Security Lake. | +| Splunk | Use the [Splunk OCSF Add-on](https://splunkbase.splunk.com/app/6943) to ingest OCSF JSONL files. Set `ocsf_schema_version` to `1.3` for CIM Add-On compatibility. | +| Amazon Security Lake | OCSF is the native schema for Security Lake. Set `ocsf_schema_version` to `1.1` for v1.1.0 compatibility. | +| CrowdStrike FDR | Ship JSONL files to Falcon Data Replicator. v1.5 downgrade target is not yet supported. | | Elastic | Use Filebeat to ship JSONL files with the OCSF field mappings. | | Custom pipelines | Parse the JSONL file with `jq`, Python, or any JSON-capable tool. |