diff --git a/docs/server-runtime.md b/docs/server-runtime.md index d35b6b34ab..bff7c51783 100644 --- a/docs/server-runtime.md +++ b/docs/server-runtime.md @@ -28,6 +28,36 @@ uses G1 with an occupancy threshold of 45 when `JAVA_OPTIONS` is empty. Do not combine explicit `-g` selection with a different collector in `JAVA_OPTIONS` or `-j`; the JVM rejects conflicting collector selections. +### Launcher memory budget + +The default launcher heap uses half the smaller of host available memory and the +remaining budget under visible cgroup v1/v2 hard limits, with a 512 MiB minimum and a +32 GiB maximum. It refuses startup when the effective budget is below 512 MiB. +If a known hard limit has missing, unreadable or invalid usage, the budget is zero; +the launcher refuses startup instead of using host memory. For cgroup v1, numeric +unlimited values are recognized only when `uname -m` reports `x86_64`, `aarch64` or +`riscv64` and `getconf PAGESIZE` returns a positive integer. The threshold is the +signed 64-bit maximum rounded down to that page size. Recognized unlimited levels +are skipped so readable finite ancestors still apply; without a finite limit, +host sizing remains available. Unknown or masked machine names, or an unavailable +or invalid page size, keep numeric limits on the finite-limit path. + +Inactive file cache can contribute to this budget on a warm restart. For a finite +cgroup v2 limit, the launcher checks the immediate cgroup children of that reclaim +target. If any child's `memory.min` is nonzero, missing, invalid or unreadable, it +excludes cache credit at that limit: child cache may be protected from ancestor reclaim. +If the target directory cannot be enumerated, or a child cannot be searched, cache +credit is also excluded because child protection cannot be determined. Ordinary +searchable directories without cgroup controller files are skipped. +Unprotected children retain the existing warm-cache budget, including inherited and +nested hard limits. A cgroup's own `memory.min` does not block cache credit for its +own limit; another ancestor can still constrain it through child protection. +The cgroup v1 cache calculation is unchanged. + +This is a startup estimate, not a guarantee against runtime OOM. Native allocations +and concurrent processes also consume memory. Explicit heap options retain their +existing precedence; see [container heap sizing](../docker/README.md#server-heap-sizing). + ### Serializer configuration Use the Gremlin configuration files shipped with the upgraded Server. Serializer classes diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 5641a52bdd..65720021eb 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -497,15 +497,38 @@ function cgroup_free_memory() { dir="${dir%/}" fi local limit usage inactive available smallest="" + local child child_min local cache_key=inactive_file - [[ "$version" != cgroup ]] || cache_key=total_inactive_file + local unlimited="" kernel_machine="" page_size="" + if [[ "$version" == cgroup ]]; then + cache_key=total_inactive_file + # v1 exposes PAGE_COUNTER_MAX * PAGE_SIZE instead of "max". + # LONG_BIT describes userspace, not the kernel. Personality can also + # mask uname as 32-bit, so infer unlimited only from known 64-bit names. + kernel_machine=$(uname -m 2>/dev/null) || true + page_size=$(getconf PAGESIZE 2>/dev/null) || true + if [[ "$page_size" =~ ^[1-9][0-9]*$ ]]; then + case "$kernel_machine" in + x86_64|aarch64|riscv64) + unlimited=$((9223372036854775807 / page_size * page_size)) ;; + esac + fi + fi while :; do limit="" - usage=0 + usage="" [[ ! -r "$dir/$limit_file" ]] || read -r limit < "$dir/$limit_file" || true - [[ ! -r "$dir/$usage_file" ]] || read -r usage < "$dir/$usage_file" || true + if [[ -n "$unlimited" && "$limit" =~ ^[0-9]+$ ]] && + awk -v limit="$limit" -v unlimited="$unlimited" 'BEGIN {exit !(limit >= unlimited)}'; then + limit="" + fi if [[ "$limit" =~ ^[0-9]+$ ]]; then - [[ "$usage" =~ ^[0-9]+$ ]] || usage=0 + # Unknown usage under a known limit must not fall back to host memory. + if [[ ! -r "$dir/$usage_file" ]] || ! read -r usage < "$dir/$usage_file" || + [[ ! "$usage" =~ ^[0-9]+$ ]]; then + echo 0 + return 0 + fi # Inactive file cache can be reclaimed on a warm restart. For v1, # use the hierarchical total so ancestor budgets include children. inactive=0 @@ -513,6 +536,30 @@ function cgroup_free_memory() { inactive=$(awk -v key="$cache_key" '$1 == key {print $2; exit}' "$dir/memory.stat") fi [[ "$inactive" =~ ^[0-9]+$ ]] || inactive=0 + if [[ "$version" == cgroup2 && "$inactive" != 0 ]]; then + # A child's protected cache cannot be reclaimed by this limit. + # The target's own memory.min does not protect against its own reclaim. + if [[ ! -r "$dir" || ! -x "$dir" ]]; then + # Known files can be readable while child enumeration is denied. + inactive=0 + else + for child in "$dir"/* "$dir"/.[!.]* "$dir"/..?*; do + [[ -d "$child" ]] || continue + # An unsearchable directory can hide a protected cgroup. + if [[ ! -x "$child" ]]; then + inactive=0 + break + fi + [[ -e "$child/memory.current" && -e "$child/memory.max" ]] || continue + child_min="" + [[ ! -r "$child/memory.min" ]] || read -r child_min < "$child/memory.min" || true + if [[ "$child_min" != 0 ]]; then + inactive=0 + break + fi + done + fi + fi # Inconsistent snapshots receive no cache credit. awk also avoids # shell integer overflow for v1's unlimited sentinel. available=$(awk -v limit="$limit" -v usage="$usage" -v inactive="$inactive" 'BEGIN { diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-cgroup-memory.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-cgroup-memory.sh index 9e570e2537..860bbffe81 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-cgroup-memory.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-cgroup-memory.sh @@ -22,11 +22,12 @@ UTIL=${1:?Usage: $0 UTIL_SH} # shellcheck disable=SC1090,SC2329 source "$UTIL" FIXTURE=$(mktemp -d) -trap 'rm -rf "$FIXTURE"' EXIT +trap 'chmod -R u+rwX "$FIXTURE"; rm -rf "$FIXTURE"' EXIT ROOT="$FIXTURE/cgroup" MEMBERSHIP="$FIXTURE/membership" MOUNTINFO="$FIXTURE/mountinfo" mkdir -p "$ROOT" +printf '0\n' > "$ROOT/memory.min" assert_memory() { local expected=$1 actual actual=$(cgroup_free_memory "$ROOT" "$MEMBERSHIP" "$MOUNTINFO") @@ -35,6 +36,45 @@ assert_memory() { exit 1 } } +# Verify unknown usage and its zero-budget path through the actual launcher helpers. +assert_unknown_usage() { + local usage_file=$1 valid_usage=$2 + printf 'invalid\n' > "$usage_file" + assert_memory 0 + : > "$usage_file" + assert_memory 0 + rm "$usage_file" + assert_memory 0 + ( + # Only supply host data; the cgroup helper still reads the real fixture. + # shellcheck disable=SC2329 + uname() { [[ "${1:-}" != -m ]] && echo Linux || echo x86_64; } + # shellcheck disable=SC2329 + cat() { + if [[ "$1" == /proc/meminfo ]]; then + printf 'MemFree: 1048576 kB\nBuffers: 0 kB\nCached: 0 kB\n' + else + command cat "$@" + fi + } + # shellcheck disable=SC2329 + cgroup_free_memory() { + bash -c 'source "$1"; cgroup_free_memory "$2" "$3" "$4"' fixture \ + "$UTIL" "$ROOT" "$MEMBERSHIP" "$MOUNTINFO" + } + [[ $(free_memory) == 0 ]] + if (calc_xmx 512 32768); then + echo 'unknown cgroup usage fell back to host memory' >&2 + exit 1 + fi + ) + printf '%s\n' "$valid_usage" > "$usage_file" + chmod 000 "$usage_file" + if [[ ! -r "$usage_file" ]]; then + assert_memory 0 + fi + chmod 600 "$usage_file" +} # No controller keeps the original host fallback. assert_memory "" printf '0::/\n' > "$MEMBERSHIP" @@ -45,6 +85,8 @@ assert_memory "" printf '4294967296\n' > "$ROOT/memory.max" printf '1073741824\n' > "$ROOT/memory.current" assert_memory 3072 +assert_unknown_usage "$ROOT/memory.current" 1073741824 +assert_memory 3072 # Usage at/over limit must not become a negative or wrapped number. printf '4294967296\n' > "$ROOT/memory.current" assert_memory 0 @@ -52,6 +94,8 @@ printf '5368709120\n' > "$ROOT/memory.current" assert_memory 0 # A finite ancestor constrains an unlimited child. mkdir -p "$ROOT/team/service" +printf '0\n' > "$ROOT/team/memory.min" +printf '0\n' > "$ROOT/team/service/memory.min" printf '0::/team/service\n' > "$MEMBERSHIP" printf 'max\n' > "$ROOT/team/service/memory.max" printf '0\n' > "$ROOT/team/service/memory.current" @@ -69,10 +113,120 @@ assert_memory 3072 # Malformed limits do not introduce arithmetic errors or an artificial zero. printf 'invalid\n' > "$ROOT/memory.max" assert_memory "" -# v1 has different names and a numeric unlimited sentinel. +# The v1 fixture represents a known 64-bit Linux kernel, independently of the +# test host's userspace or machine name (the checks also run on macOS). +# shellcheck disable=SC2329 +uname() { [[ "${1:-}" != -m ]] && command uname "$@" || echo x86_64; } +export -f uname +PAGE_SIZE=$(getconf PAGESIZE) +V1_UNLIMITED=$((9223372036854775807 / PAGE_SIZE * PAGE_SIZE)) mkdir -p "$ROOT/memory/job" printf '5:cpu,cpuacct:/job\n6:memory:/job\n' > "$MEMBERSHIP" -printf '9223372036854771712\n' > "$ROOT/memory/memory.limit_in_bytes" +printf '%s\n' "$V1_UNLIMITED" > "$ROOT/memory/memory.limit_in_bytes" +printf '0\n' > "$ROOT/memory/memory.usage_in_bytes" +printf '2147483648\n' > "$ROOT/memory/job/memory.limit_in_bytes" +printf '536870912\n' > "$ROOT/memory/job/memory.usage_in_bytes" +assert_memory 1536 +assert_unknown_usage "$ROOT/memory/job/memory.usage_in_bytes" 536870912 +assert_memory 1536 +# A v1 unlimited child must not reject unreadable usage or hide a finite ancestor. +printf '%s\n' "$V1_UNLIMITED" > "$ROOT/memory/job/memory.limit_in_bytes" +printf '2147483648\n' > "$ROOT/memory/memory.limit_in_bytes" +printf '536870912\n' > "$ROOT/memory/memory.usage_in_bytes" +printf 'invalid\n' > "$ROOT/memory/job/memory.usage_in_bytes" +assert_memory 1536 +# The existing 64-bit/4-KiB sentinel is also an unlimited boundary. +printf '9223372036854771712\n' > "$ROOT/memory/job/memory.limit_in_bytes" +assert_memory 1536 +printf '%s\n' "$V1_UNLIMITED" > "$ROOT/memory/job/memory.limit_in_bytes" +rm "$ROOT/memory/job/memory.usage_in_bytes" +assert_memory 1536 +printf '536870912\n' > "$ROOT/memory/job/memory.usage_in_bytes" +chmod 000 "$ROOT/memory/job/memory.usage_in_bytes" +assert_memory 1536 +chmod 600 "$ROOT/memory/job/memory.usage_in_bytes" +# Native-query failures must keep unknown numeric-limit usage conservative. +rm "$ROOT/memory/job/memory.usage_in_bytes" +# The page immediately below the sentinel is finite and still needs usage. +printf '%s\n' "$((V1_UNLIMITED - PAGE_SIZE))" > "$ROOT/memory/job/memory.limit_in_bytes" +assert_memory 0 +printf '%s\n' "$V1_UNLIMITED" > "$ROOT/memory/job/memory.limit_in_bytes" +( + # shellcheck disable=SC2329 + getconf() { return 127; } + assert_memory 0 +) +( + # shellcheck disable=SC2329 + getconf() { echo invalid; } + assert_memory 0 +) +( + # shellcheck disable=SC2329 + getconf() { [[ "$1" != PAGESIZE ]] || return 0; } + assert_memory 0 +) +# A 32-bit userspace cannot turn an 8-TiB finite limit into unlimited. +( + # shellcheck disable=SC2329 + getconf() { [[ "$1" != LONG_BIT ]] && echo 4096 || echo 32; } + printf '8796093022208\n' > "$ROOT/memory/memory.limit_in_bytes" + printf '8795824586752\n' > "$ROOT/memory/memory.usage_in_bytes" + printf '9223372036854771712\n' > "$ROOT/memory/job/memory.limit_in_bytes" + for machine in x86_64 aarch64 riscv64; do + # shellcheck disable=SC2329 + uname() { echo "$machine"; } + assert_memory 256 + done + # Unknown/32-bit-looking names, including personality-masked uname, must + # retain finite limits rather than infer a 32-bit unlimited sentinel. + printf '8796093022208\n' > "$ROOT/memory/job/memory.limit_in_bytes" + printf '8795824586752\n' > "$ROOT/memory/job/memory.usage_in_bytes" + for machine in unknown i686 armv8l; do + assert_memory 256 + rm "$ROOT/memory/job/memory.usage_in_bytes" + assert_memory 0 + printf '9223372036854771712\n' > "$ROOT/memory/job/memory.limit_in_bytes" + assert_memory 0 + printf '8796093022208\n' > "$ROOT/memory/job/memory.limit_in_bytes" + printf '8795824586752\n' > "$ROOT/memory/job/memory.usage_in_bytes" + done + # shellcheck disable=SC2329 + uname() { return 127; } + assert_memory 256 + rm "$ROOT/memory/job/memory.usage_in_bytes" + assert_memory 0 + printf '9223372036854771712\n' > "$ROOT/memory/job/memory.limit_in_bytes" + assert_memory 0 +) +# Restore the original finite-ancestor/unlimited-child fixture. +printf '2147483648\n' > "$ROOT/memory/memory.limit_in_bytes" +printf '536870912\n' > "$ROOT/memory/memory.usage_in_bytes" +printf '%s\n' "$V1_UNLIMITED" > "$ROOT/memory/job/memory.limit_in_bytes" +# With no finite limits, unknown unlimited usage still permits host sizing. +printf '%s\n' "$V1_UNLIMITED" > "$ROOT/memory/memory.limit_in_bytes" +rm "$ROOT/memory/memory.usage_in_bytes" +assert_memory "" +( + # Supply host data; the actual helper reads the unlimited fixture. + # shellcheck disable=SC2329 + uname() { [[ "${1:-}" != -m ]] && echo Linux || echo x86_64; } + # shellcheck disable=SC2329 + cat() { + if [[ "$1" == /proc/meminfo ]]; then + printf 'MemFree: 1048576 kB\nBuffers: 0 kB\nCached: 0 kB\n' + else + command cat "$@" + fi + } + # shellcheck disable=SC2329 + cgroup_free_memory() { + bash -c 'source "$1"; cgroup_free_memory "$2" "$3" "$4"' fixture \ + "$UTIL" "$ROOT" "$MEMBERSHIP" "$MOUNTINFO" + } + [[ $(free_memory) == 1024 ]] + [[ $(calc_xmx 512 32768) == 512 ]] +) printf '0\n' > "$ROOT/memory/memory.usage_in_bytes" printf '2147483648\n' > "$ROOT/memory/job/memory.limit_in_bytes" printf '536870912\n' > "$ROOT/memory/job/memory.usage_in_bytes" @@ -100,6 +254,7 @@ assert_memory 1536 # Skip unrelated bind mounts before applying namespace fallback. UNRELATED="$FIXTURE/unrelated" mkdir -p "$UNRELATED" +printf '0\n' > "$UNRELATED/memory.min" printf '17179869184\n' > "$UNRELATED/memory.max" printf '0\n' > "$UNRELATED/memory.current" printf '0::/team/service\n' > "$MEMBERSHIP" @@ -146,6 +301,76 @@ assert_memory 1024 printf '6:memory:/job\n' > "$MEMBERSHIP" printf 'inactive_file 0\ntotal_inactive_file 536870912\n' > "$ROOT/memory/job/memory.stat" assert_memory 2048 +# A separate v2 hierarchy preserves ordinary dual/inherited warm limits. +ROOT="$FIXTURE/v2-warm" +mkdir -p "$ROOT/target" "$ROOT/sibling" "$ROOT/notes" +for dir in "$ROOT" "$ROOT/target" "$ROOT/sibling"; do + printf '0\n' > "$dir/memory.min" + printf 'max\n' > "$dir/memory.max" + printf '0\n' > "$dir/memory.current" + printf 'inactive_file 0\n' > "$dir/memory.stat" +done +printf '0::/target\n' > "$MEMBERSHIP" +: > "$MOUNTINFO" +printf '2147483648\n' > "$ROOT/memory.max" +printf '1610612736\n' > "$ROOT/memory.current" +printf 'inactive_file 1610612736\n' > "$ROOT/memory.stat" +printf '2147483648\n' > "$ROOT/target/memory.max" +printf '1610612736\n' > "$ROOT/target/memory.current" +printf 'inactive_file 1610612736\n' > "$ROOT/target/memory.stat" +# A directory without controller files is not a cgroup child. +printf '1610612736\n' > "$ROOT/notes/memory.min" +assert_memory 2048 +printf 'max\n' > "$ROOT/target/memory.max" +assert_memory 2048 +# Protection is relative to the reclaim target: its own min is ignored. +printf 'max\n' > "$ROOT/memory.max" +printf '2147483648\n' > "$ROOT/target/memory.max" +printf '1610612736\n' > "$ROOT/target/memory.min" +assert_memory 2048 +printf '0\n' > "$ROOT/target/memory.min" +printf '2147483648\n' > "$ROOT/memory.max" +# Move the warm cache to a protected sibling; the ancestor must not credit it. +printf '0\n' > "$ROOT/target/memory.current" +printf 'inactive_file 0\n' > "$ROOT/target/memory.stat" +printf '1610612736\n' > "$ROOT/sibling/memory.current" +printf 'inactive_file 1610612736\n' > "$ROOT/sibling/memory.stat" +printf '1610612736\n' > "$ROOT/sibling/memory.min" +assert_memory 512 +printf 'invalid\n' > "$ROOT/sibling/memory.min" +assert_memory 512 +rm "$ROOT/sibling/memory.min" +assert_memory 512 +printf '0\n' > "$ROOT/sibling/memory.min" +chmod 000 "$ROOT/sibling/memory.min" +if [[ ! -r "$ROOT/sibling/memory.min" ]]; then + assert_memory 512 +fi +chmod 600 "$ROOT/sibling/memory.min" +assert_memory 2048 +# Hidden child cgroups (as used by systemd) have the same protection semantics. +mv "$ROOT/sibling" "$ROOT/.sibling" +printf '1610612736\n' > "$ROOT/.sibling/memory.min" +assert_memory 512 +# Actual filesystem permissions must not disguise a cgroup as an ordinary dir. +printf '0\n' > "$ROOT/.sibling/memory.min" +chmod 000 "$ROOT/.sibling" +if [[ ! -x "$ROOT/.sibling" ]]; then + assert_memory 512 +else + # Root with DAC override can still read the actual zero protection value. + assert_memory 2048 +fi +chmod 700 "$ROOT/.sibling" +# Search permission alone permits known files but cannot enumerate children. +chmod 111 "$ROOT" +if [[ ! -r "$ROOT" ]]; then + assert_memory 512 +else + # Root with DAC override can enumerate the unprotected children. + assert_memory 2048 +fi +chmod 700 "$ROOT" # shellcheck disable=SC2329 free_memory() { echo 3072; } [[ $(calc_xmx 512 32768) == 1536 ]]