diff --git a/CHANGELOG.md b/CHANGELOG.md index e526ae42856..37f9a77250c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ---> +# [0.9.2] + +### Security + +- Restrict Kafka client params that execute commands or load libraries (GHSA-g85h-2988-4cv8). +- Reject oversized columnar dictionaries during merges (Tantivy GHSA-7pph-m7rw-cqhv). + # [0.9.1] ### Security diff --git a/docs/configuration/source-config.md b/docs/configuration/source-config.md index 30894d3faad..6b58fc34fc0 100644 --- a/docs/configuration/source-config.md +++ b/docs/configuration/source-config.md @@ -103,6 +103,8 @@ A tutorial is available [here](/docs/ingest-data/kafka.md). The Kafka source consumes a `topic` using the client library [librdkafka](https://github.com/edenhill/librdkafka) and forwards the key-value pairs carried by the parameter `client_params` to the underlying librdkafka consumer. Common `client_params` options are bootstrap servers (`bootstrap.servers`), or security protocol (`security.protocol`). Please, refer to [Kafka](https://kafka.apache.org/documentation/#consumerconfigs) and [librdkafka](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) documentation pages for more advanced options. +For security reasons, `client_params` cannot set `sasl.kerberos.kinit.cmd`, `sasl.kerberos.min.time.before.relogin`, `plugin.library.paths`, `ssl.engine.location`, or `ssl.providers`. By default, librdkafka does not run `kinit` to acquire or refresh Kerberos tickets. To enable it, set the `QW_KAFKA_KINIT_CMD` environment variable to the `kinit` command on every node that creates or runs Kafka sources. The command cannot contain `%{...}` placeholders, so `sasl.kerberos.keytab` and `sasl.kerberos.principal` must be written into the command directly. + | Property | Description | Default value | | --- | --- | --- | | `topic` | Name of the topic to consume. | required | diff --git a/docs/ingest-data/kafka.md b/docs/ingest-data/kafka.md index 11b4757e627..0a2f0761cc9 100644 --- a/docs/ingest-data/kafka.md +++ b/docs/ingest-data/kafka.md @@ -224,6 +224,8 @@ params: sasl.password: your_sasl_password ``` +For Kerberos (`sasl.mechanisms: GSSAPI`), `client_params` cannot set the `kinit` command. Either refresh tickets outside Quickwit, or set the `QW_KAFKA_KINIT_CMD` environment variable on every node that creates or runs Kafka sources, for example `QW_KAFKA_KINIT_CMD="kinit -k -t /path/to/quickwit.keytab quickwit@EXAMPLE.COM"`. See the [source configuration](../configuration/source-config.md) page for details. + :::note If you get the following error: diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index 54ac05c040b..8834a2c8cde 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -3,6 +3,12 @@ title: REST API sidebar_position: 10 --- +## Security + +:::warning +Quickwit does not authenticate clients unless [mutual TLS](../configuration/node-config.md#tls-configuration) is enabled. Anyone who can reach the REST or gRPC API has full access to the cluster. Keep Quickwit on a trusted network or behind an authenticated proxy, and do not expose it directly to the public internet. +::: + ## API version All the API endpoints start with the `api/v1/` prefix. `v1` indicates that we are currently using version 1 of the API. diff --git a/quickwit/Cargo.lock b/quickwit/Cargo.lock index 601594bdcda..336b249b2fb 100644 --- a/quickwit/Cargo.lock +++ b/quickwit/Cargo.lock @@ -4532,7 +4532,7 @@ dependencies = [ [[package]] name = "jitexpr" version = "0.1.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=e1acb3e0112f2b4819946aa4ea5742bf429853d8#e1acb3e0112f2b4819946aa4ea5742bf429853d8" dependencies = [ "cranelift", "cranelift-jit", @@ -5995,7 +5995,7 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "ownedbytes" version = "0.9.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=e1acb3e0112f2b4819946aa4ea5742bf429853d8#e1acb3e0112f2b4819946aa4ea5742bf429853d8" dependencies = [ "stable_deref_trait", ] @@ -7103,7 +7103,7 @@ dependencies = [ [[package]] name = "quickwit-actors" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "async-trait", @@ -7123,7 +7123,7 @@ dependencies = [ [[package]] name = "quickwit-aws" -version = "0.9.1" +version = "0.9.2" dependencies = [ "aws-config", "aws-runtime", @@ -7143,7 +7143,7 @@ dependencies = [ [[package]] name = "quickwit-cli" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "backtrace", @@ -7195,7 +7195,7 @@ dependencies = [ [[package]] name = "quickwit-cluster" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "async-trait", @@ -7222,7 +7222,7 @@ dependencies = [ [[package]] name = "quickwit-codegen" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "futures", @@ -7238,7 +7238,7 @@ dependencies = [ [[package]] name = "quickwit-codegen-example" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "async-trait", @@ -7265,7 +7265,7 @@ dependencies = [ [[package]] name = "quickwit-common" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "async-speed-limit", @@ -7316,7 +7316,7 @@ dependencies = [ [[package]] name = "quickwit-compaction" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "async-trait", @@ -7340,7 +7340,7 @@ dependencies = [ [[package]] name = "quickwit-config" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "bytes", @@ -7373,7 +7373,7 @@ dependencies = [ [[package]] name = "quickwit-control-plane" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "async-trait", @@ -7404,7 +7404,7 @@ dependencies = [ [[package]] name = "quickwit-datetime" -version = "0.9.1" +version = "0.9.2" dependencies = [ "itertools 0.15.0", "serde", @@ -7416,7 +7416,7 @@ dependencies = [ [[package]] name = "quickwit-directories" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "async-trait", @@ -7433,7 +7433,7 @@ dependencies = [ [[package]] name = "quickwit-doc-mapper" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "base64 0.23.0", @@ -7464,7 +7464,7 @@ dependencies = [ [[package]] name = "quickwit-index-management" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "futures", @@ -7485,7 +7485,7 @@ dependencies = [ [[package]] name = "quickwit-indexing" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "arc-swap", @@ -7551,7 +7551,7 @@ dependencies = [ [[package]] name = "quickwit-ingest" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "async-trait", @@ -7593,7 +7593,7 @@ dependencies = [ [[package]] name = "quickwit-integration-tests" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "aws-sdk-sqs", @@ -7632,7 +7632,7 @@ dependencies = [ [[package]] name = "quickwit-jaeger" -version = "0.9.1" +version = "0.9.2" dependencies = [ "async-trait", "itertools 0.15.0", @@ -7665,7 +7665,7 @@ dependencies = [ [[package]] name = "quickwit-janitor" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "async-trait", @@ -7698,7 +7698,7 @@ dependencies = [ [[package]] name = "quickwit-lambda-client" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "async-trait", @@ -7725,7 +7725,7 @@ dependencies = [ [[package]] name = "quickwit-lambda-server" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "base64 0.23.0", @@ -7749,7 +7749,7 @@ dependencies = [ [[package]] name = "quickwit-macros" -version = "0.9.1" +version = "0.9.2" dependencies = [ "proc-macro2", "quote", @@ -7758,7 +7758,7 @@ dependencies = [ [[package]] name = "quickwit-metastore" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "async-trait", @@ -7804,7 +7804,7 @@ dependencies = [ [[package]] name = "quickwit-metrics" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "atomic_float", @@ -7823,14 +7823,14 @@ dependencies = [ [[package]] name = "quickwit-metrics-inventory" -version = "0.9.1" +version = "0.9.2" dependencies = [ "quickwit-metrics", ] [[package]] name = "quickwit-opentelemetry" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "async-trait", @@ -7852,7 +7852,7 @@ dependencies = [ [[package]] name = "quickwit-proto" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "async-trait", @@ -7890,7 +7890,7 @@ dependencies = [ [[package]] name = "quickwit-query" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "base64 0.23.0", @@ -7918,7 +7918,7 @@ dependencies = [ [[package]] name = "quickwit-rest-client" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "bytes", @@ -7944,7 +7944,7 @@ dependencies = [ [[package]] name = "quickwit-search" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "assert-json-diff", @@ -7989,7 +7989,7 @@ dependencies = [ [[package]] name = "quickwit-serve" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "assert-json-diff", @@ -8067,7 +8067,7 @@ dependencies = [ [[package]] name = "quickwit-storage" -version = "0.9.1" +version = "0.9.2" dependencies = [ "ahash", "anyhow", @@ -8127,7 +8127,7 @@ dependencies = [ [[package]] name = "quickwit-telemetry-exporters" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "metrics", @@ -8150,7 +8150,7 @@ dependencies = [ [[package]] name = "quickwit-transport" -version = "0.9.1" +version = "0.9.2" dependencies = [ "anyhow", "arc-swap", @@ -10306,7 +10306,7 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tantivy" version = "0.27.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=e1acb3e0112f2b4819946aa4ea5742bf429853d8#e1acb3e0112f2b4819946aa4ea5742bf429853d8" dependencies = [ "aho-corasick", "arc-swap", @@ -10324,6 +10324,7 @@ dependencies = [ "fs4", "futures-channel", "futures-util", + "hashbrown 0.17.1", "htmlescape", "itertools 0.14.0", "jitexpr", @@ -10347,7 +10348,7 @@ dependencies = [ "tantivy-columnar", "tantivy-common", "tantivy-fst", - "tantivy-query-grammar 0.26.0 (git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe)", + "tantivy-query-grammar 0.26.0 (git+https://github.com/quickwit-oss/tantivy/?rev=e1acb3e0112f2b4819946aa4ea5742bf429853d8)", "tantivy-sstable", "tantivy-stacker", "tantivy-tokenizer-api", @@ -10364,7 +10365,7 @@ dependencies = [ [[package]] name = "tantivy-bitpacker" version = "0.10.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=e1acb3e0112f2b4819946aa4ea5742bf429853d8#e1acb3e0112f2b4819946aa4ea5742bf429853d8" dependencies = [ "bitpacking", ] @@ -10372,7 +10373,7 @@ dependencies = [ [[package]] name = "tantivy-columnar" version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=e1acb3e0112f2b4819946aa4ea5742bf429853d8#e1acb3e0112f2b4819946aa4ea5742bf429853d8" dependencies = [ "downcast-rs", "fastdivide", @@ -10388,7 +10389,7 @@ dependencies = [ [[package]] name = "tantivy-common" version = "0.11.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=e1acb3e0112f2b4819946aa4ea5742bf429853d8#e1acb3e0112f2b4819946aa4ea5742bf429853d8" dependencies = [ "async-trait", "byteorder", @@ -10424,7 +10425,7 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" version = "0.26.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=e1acb3e0112f2b4819946aa4ea5742bf429853d8#e1acb3e0112f2b4819946aa4ea5742bf429853d8" dependencies = [ "fnv", "nom 7.1.3", @@ -10436,7 +10437,7 @@ dependencies = [ [[package]] name = "tantivy-sstable" version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=e1acb3e0112f2b4819946aa4ea5742bf429853d8#e1acb3e0112f2b4819946aa4ea5742bf429853d8" dependencies = [ "futures-util", "itertools 0.14.0", @@ -10449,7 +10450,7 @@ dependencies = [ [[package]] name = "tantivy-stacker" version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=e1acb3e0112f2b4819946aa4ea5742bf429853d8#e1acb3e0112f2b4819946aa4ea5742bf429853d8" dependencies = [ "murmurhash32", "tantivy-common", @@ -10458,7 +10459,7 @@ dependencies = [ [[package]] name = "tantivy-tokenizer-api" version = "0.7.0" -source = "git+https://github.com/quickwit-oss/tantivy/?rev=2e87c2eb0c8577c96c82a777f345bbf236a985fe#2e87c2eb0c8577c96c82a777f345bbf236a985fe" +source = "git+https://github.com/quickwit-oss/tantivy/?rev=e1acb3e0112f2b4819946aa4ea5742bf429853d8#e1acb3e0112f2b4819946aa4ea5742bf429853d8" dependencies = [ "serde", ] diff --git a/quickwit/Cargo.toml b/quickwit/Cargo.toml index a58d5f3797f..af613e7e89d 100644 --- a/quickwit/Cargo.toml +++ b/quickwit/Cargo.toml @@ -42,7 +42,7 @@ members = [ ] [workspace.package] -version = "0.9.1" +version = "0.9.2" edition = "2024" homepage = "https://quickwit.io/" documentation = "https://quickwit.io/docs/" @@ -379,7 +379,7 @@ quickwit-telemetry-exporters = { path = "quickwit-telemetry-exporters" } quickwit-transport = { path = "quickwit-transport" } # This is the tip of tantivy main. -tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "2e87c2eb0c8577c96c82a777f345bbf236a985fe", default-features = false, features = [ +tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "e1acb3e0112f2b4819946aa4ea5742bf429853d8", default-features = false, features = [ "jitexpr", "lz4-compression", "mmap", @@ -388,7 +388,7 @@ tantivy = { git = "https://github.com/quickwit-oss/tantivy/", rev = "2e87c2eb0c8 "columnar-zstd-compression", ] } # Must stay on the same rev as `tantivy`. -tantivy-common = { git = "https://github.com/quickwit-oss/tantivy/", rev = "2e87c2eb0c8577c96c82a777f345bbf236a985fe" } +tantivy-common = { git = "https://github.com/quickwit-oss/tantivy/", rev = "e1acb3e0112f2b4819946aa4ea5742bf429853d8" } tantivy-fst = "0.5" # This is actually not used directly the goal is to fix the version diff --git a/quickwit/quickwit-indexing/src/source/kafka_source.rs b/quickwit/quickwit-indexing/src/source/kafka_source.rs index a6769fc169d..e72e26b023a 100644 --- a/quickwit/quickwit-indexing/src/source/kafka_source.rs +++ b/quickwit/quickwit-indexing/src/source/kafka_source.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; use std::fmt; +use std::sync::LazyLock; use std::time::{Duration, Instant}; use anyhow::{Context, anyhow, bail}; @@ -707,14 +708,47 @@ fn parse_client_log_level(client_log_level: Option) -> anyhow::Result anyhow::Result { + static KINIT_CMD: LazyLock> = + LazyLock::new(|| quickwit_common::get_from_env_opt(QW_KAFKA_KINIT_CMD_ENV_KEY, true)); + parse_client_params_inner(client_params, KINIT_CMD.as_deref()) +} + +fn parse_client_params_inner( + client_params: JsonValue, + kinit_cmd_opt: Option<&str>, +) -> anyhow::Result { let params = if let JsonValue::Object(params) = client_params { params } else { bail!("failed to parse Kafka client parameters. `client_params` must be a JSON object"); }; let mut client_config = ClientConfig::new(); + if let Some(kinit_cmd) = kinit_cmd_opt.filter(|kinit_cmd| !kinit_cmd.trim().is_empty()) { + if kinit_cmd.contains("%{") { + bail!( + "failed to parse Kafka client parameters. `{QW_KAFKA_KINIT_CMD_ENV_KEY}` must not \ + contain `%{{...}}` placeholders" + ); + } + client_config.set("sasl.kerberos.kinit.cmd", kinit_cmd); + } else { + client_config.set("sasl.kerberos.min.time.before.relogin", "0"); + } for (key, value_json) in params { + if FORBIDDEN_CLIENT_PARAMS.contains(&key.as_str()) { + bail!("failed to parse Kafka client parameters. `client_params.{key}` is not allowed"); + } let value = match value_json { JsonValue::Bool(value_bool) => value_bool.to_string(), JsonValue::Number(value_number) => value_number.to_string(), @@ -756,6 +790,56 @@ fn message_payload_to_doc(message: &BorrowedMessage) -> Option { None } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_client_params_rejects_forbidden_params() { + for key in [ + "plugin.library.paths", + "sasl.kerberos.kinit.cmd", + "sasl.kerberos.min.time.before.relogin", + "ssl.engine.location", + "ssl.providers", + ] { + parse_client_params_inner(json!({ key: "value" }), Some("kinit")).unwrap_err(); + } + } + + #[test] + fn test_parse_client_params_kinit_cmd() { + let client_config = parse_client_params_inner(json!({}), None).unwrap(); + assert_eq!( + client_config.get("sasl.kerberos.min.time.before.relogin"), + Some("0") + ); + assert_eq!(client_config.get("sasl.kerberos.kinit.cmd"), None); + + for empty_kinit_cmd in ["", " "] { + let client_config = + parse_client_params_inner(json!({}), Some(empty_kinit_cmd)).unwrap(); + assert_eq!( + client_config.get("sasl.kerberos.min.time.before.relogin"), + Some("0") + ); + assert_eq!(client_config.get("sasl.kerberos.kinit.cmd"), None); + } + + let client_config = parse_client_params_inner(json!({}), Some("kinit -k")).unwrap(); + assert_eq!( + client_config.get("sasl.kerberos.kinit.cmd"), + Some("kinit -k") + ); + assert_eq!( + client_config.get("sasl.kerberos.min.time.before.relogin"), + None + ); + + parse_client_params_inner(json!({}), Some("kinit %{sasl.kerberos.principal}")).unwrap_err(); + } +} + #[cfg(all(test, feature = "kafka-broker-tests"))] mod kafka_broker_tests { use std::num::NonZeroUsize; diff --git a/quickwit/quickwit-ui/package.json b/quickwit/quickwit-ui/package.json index e31ee71252d..a77b43a4938 100644 --- a/quickwit/quickwit-ui/package.json +++ b/quickwit/quickwit-ui/package.json @@ -1,6 +1,6 @@ { "name": "quickwit-ui", - "version": "0.9.1", + "version": "0.9.2", "license": "Apache-2.0", "private": true, "packageManager": "yarn@1.22.22",