diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 266971f45d9..8a7e579a25f 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -41,7 +41,7 @@ jobs: - name: Install dependencies run: | sudo apt update - sudo apt install libmagick++-dev libncurses-dev libpcre2-dev libbrotli-dev libluajit-5.1-dev luajit libjansson-dev libcjose-dev libmaxminddb-dev libgeoip-dev ninja-build cmake libpcre3-dev + sudo apt install libmagick++-dev libncurses-dev libpcre2-dev libbrotli-dev libluajit-5.1-dev luajit libjansson-dev libcjose-dev libmaxminddb-dev libgeoip-dev ninja-build cmake libpcre3-dev libzstd-dev liblz4-dev # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v4 diff --git a/CMakeLists.txt b/CMakeLists.txt index 897cf30df74..fef88b2d529 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -489,33 +489,41 @@ set(TS_USE_MALLOC_ALLOCATOR ${ENABLE_MALLOC_ALLOCATOR}) set(TS_USE_ALLOCATOR_METRICS ${ENABLE_ALLOCATOR_METRICS}) find_package(ZLIB REQUIRED) -find_package(zstd CONFIG QUIET) -if(zstd_FOUND) - - # Provide a compatibility target name if the upstream package does not export it - # Our code links against `zstd::zstd`; upstream zstd usually exports - # `zstd::libzstd_shared`/`zstd::libzstd_static`. Create an alias if needed. - if(NOT TARGET zstd::zstd) - if(TARGET zstd::libzstd_shared) - set(_zstd_target zstd::libzstd_shared) - elseif(TARGET zstd::libzstd_static) - set(_zstd_target zstd::libzstd_static) - elseif(TARGET zstd::libzstd) - set(_zstd_target zstd::libzstd) - endif() - if(DEFINED _zstd_target) +# 1.4.0 stabilized the advanced one-shot API (ZSTD_compress2 et al.) used by +# the RAM cache; plugins/compress already requires the same floor for +# ZSTD_compressStream2, so this is not specific to the cache. +find_package(ZSTD 1.4.0) +set(HAVE_ZSTD_H ${ZSTD_FOUND}) + +# cmake/FindZSTD.cmake creates the zstd::zstd target this tree links against, +# but with CMAKE_FIND_PACKAGE_PREFER_CONFIG the lookup can resolve through +# zstd's own config package instead: on a case-insensitive filesystem the +# ZSTDConfig.cmake CMake searches for matches the zstdConfig.cmake that zstd +# installs. That package exports zstd::libzstd_shared/_static, so alias +# whichever it gave us or the four targets linking zstd::zstd fail at generate +# time. +if(ZSTD_FOUND AND NOT TARGET zstd::zstd) + foreach(_zstd_target zstd::libzstd_shared zstd::libzstd_static zstd::libzstd) + if(TARGET ${_zstd_target}) add_library(zstd_zstd INTERFACE) target_link_libraries(zstd_zstd INTERFACE ${_zstd_target}) add_library(zstd::zstd ALIAS zstd_zstd) - set(HAVE_ZSTD_H TRUE) - else() - set(HAVE_ZSTD_H FALSE) + break() endif() + endforeach() + unset(_zstd_target) + if(NOT TARGET zstd::zstd) + message(WARNING "zstd found but it exports no target this build can use; building without zstd") + set(HAVE_ZSTD_H FALSE) endif() -else() - set(HAVE_ZSTD_H FALSE) endif() +# 1.7.0 (r129) introduced the current compression API, including +# LZ4_compress_default(), which the RAM cache uses; 1.7.5 is the floor for +# LZ4_versionString(), which traffic_layout reports. +find_package(LZ4 1.7.5) +set(HAVE_LZ4_H ${LZ4_FOUND}) + # ncurses is used in traffic_top find_package(Curses) set(HAVE_CURSES_H ${CURSES_HAVE_CURSES_H}) diff --git a/ci/docker/deb/Dockerfile b/ci/docker/deb/Dockerfile index 85815163e2f..cd7d4bb89bf 100644 --- a/ci/docker/deb/Dockerfile +++ b/ci/docker/deb/Dockerfile @@ -56,7 +56,7 @@ RUN apt-get update; apt-get -y dist-upgrade; \ libhwloc-dev libunwind8 libunwind-dev zlib1g-dev \ tcl-dev tcl8.6-dev libjemalloc-dev libluajit-5.1-dev liblzma-dev \ libhiredis-dev libbrotli-dev libncurses-dev libgeoip-dev libmagick++-dev \ - libzstd-dev; \ + libzstd-dev liblz4-dev; \ # Optional: This is for the OpenSSH server, and Jenkins account + access (comment out if not needed) apt-get -y install openssh-server openjdk-8-jre && mkdir /run/sshd; \ groupadd -g 665 jenkins && \ diff --git a/ci/docker/yum/Dockerfile b/ci/docker/yum/Dockerfile index 4f53cadea60..c97237b21ab 100644 --- a/ci/docker/yum/Dockerfile +++ b/ci/docker/yum/Dockerfile @@ -52,7 +52,7 @@ RUN yum -y update; \ # Devel packages that ATS needs yum -y install openssl-devel expat-devel pcre-devel libcap-devel hwloc-devel libunwind-devel \ xz-devel libcurl-devel ncurses-devel jemalloc-devel GeoIP-devel luajit-devel brotli-devel \ - ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel \ + ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel zstd-devel lz4-devel \ perl-ExtUtils-MakeMaker perl-Digest-SHA perl-URI; \ # This is for autest stuff yum -y install python3 httpd-tools procps-ng nmap-ncat \ diff --git a/cmake/FindLZ4.cmake b/cmake/FindLZ4.cmake new file mode 100644 index 00000000000..b9128664c68 --- /dev/null +++ b/cmake/FindLZ4.cmake @@ -0,0 +1,74 @@ +####################### +# +# Licensed to the Apache Software Foundation (ASF) under one or more contributor license +# agreements. See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# +####################### + +# FindLZ4.cmake +# +# This will define the following variables +# +# LZ4_FOUND +# LZ4_LIBRARY +# LZ4_INCLUDE_DIRS +# LZ4_VERSION +# +# and the following imported target +# +# LZ4::LZ4 +# + +find_library(LZ4_LIBRARY NAMES lz4 liblz4) +find_path(LZ4_INCLUDE_DIR NAMES lz4.h) + +mark_as_advanced(LZ4_FOUND LZ4_LIBRARY LZ4_INCLUDE_DIR) + +# The version lives in three separate macros in lz4.h; a config package would +# supply it, but this module has to read them out to satisfy a version request. +if(LZ4_INCLUDE_DIR AND EXISTS "${LZ4_INCLUDE_DIR}/lz4.h") + set(_LZ4_version_parts "") + foreach(_LZ4_part MAJOR MINOR RELEASE) + file(STRINGS "${LZ4_INCLUDE_DIR}/lz4.h" _LZ4_line REGEX "^#define[ \t]+LZ4_VERSION_${_LZ4_part}[ \t]+[0-9]+") + # The value may be followed by a comment, so capture it rather than + # anchoring on the end of the line. + if(_LZ4_line MATCHES "^#define[ \t]+LZ4_VERSION_${_LZ4_part}[ \t]+([0-9]+)") + list(APPEND _LZ4_version_parts "${CMAKE_MATCH_1}") + endif() + endforeach() + list(LENGTH _LZ4_version_parts _LZ4_version_count) + if(_LZ4_version_count EQUAL 3) + list(JOIN _LZ4_version_parts "." LZ4_VERSION) + endif() + unset(_LZ4_line) + unset(_LZ4_part) + unset(_LZ4_version_parts) + unset(_LZ4_version_count) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + LZ4 + REQUIRED_VARS LZ4_LIBRARY LZ4_INCLUDE_DIR + VERSION_VAR LZ4_VERSION +) + +if(LZ4_FOUND) + set(LZ4_INCLUDE_DIRS "${LZ4_INCLUDE_DIR}") +endif() + +if(LZ4_FOUND AND NOT TARGET LZ4::LZ4) + add_library(LZ4::LZ4 INTERFACE IMPORTED) + target_include_directories(LZ4::LZ4 INTERFACE ${LZ4_INCLUDE_DIRS}) + target_link_libraries(LZ4::LZ4 INTERFACE "${LZ4_LIBRARY}") +endif() diff --git a/cmake/FindZSTD.cmake b/cmake/FindZSTD.cmake new file mode 100644 index 00000000000..d08facc4052 --- /dev/null +++ b/cmake/FindZSTD.cmake @@ -0,0 +1,74 @@ +####################### +# +# Licensed to the Apache Software Foundation (ASF) under one or more contributor license +# agreements. See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# +####################### + +# FindZSTD.cmake +# +# This will define the following variables +# +# ZSTD_FOUND +# ZSTD_LIBRARY +# ZSTD_INCLUDE_DIRS +# ZSTD_VERSION +# +# and the following imported target +# +# zstd::zstd +# + +find_library(ZSTD_LIBRARY NAMES zstd libzstd) +find_path(ZSTD_INCLUDE_DIR NAMES zstd.h) + +mark_as_advanced(ZSTD_FOUND ZSTD_LIBRARY ZSTD_INCLUDE_DIR) + +# The version lives in three separate macros in zstd.h; a config package would +# supply it, but this module has to read them out to satisfy a version request. +if(ZSTD_INCLUDE_DIR AND EXISTS "${ZSTD_INCLUDE_DIR}/zstd.h") + set(_ZSTD_version_parts "") + foreach(_ZSTD_part MAJOR MINOR RELEASE) + file(STRINGS "${ZSTD_INCLUDE_DIR}/zstd.h" _ZSTD_line REGEX "^#define[ \t]+ZSTD_VERSION_${_ZSTD_part}[ \t]+[0-9]+") + # The value may be followed by a comment, so capture it rather than + # anchoring on the end of the line. + if(_ZSTD_line MATCHES "^#define[ \t]+ZSTD_VERSION_${_ZSTD_part}[ \t]+([0-9]+)") + list(APPEND _ZSTD_version_parts "${CMAKE_MATCH_1}") + endif() + endforeach() + list(LENGTH _ZSTD_version_parts _ZSTD_version_count) + if(_ZSTD_version_count EQUAL 3) + list(JOIN _ZSTD_version_parts "." ZSTD_VERSION) + endif() + unset(_ZSTD_line) + unset(_ZSTD_part) + unset(_ZSTD_version_parts) + unset(_ZSTD_version_count) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + ZSTD + REQUIRED_VARS ZSTD_LIBRARY ZSTD_INCLUDE_DIR + VERSION_VAR ZSTD_VERSION +) + +if(ZSTD_FOUND) + set(ZSTD_INCLUDE_DIRS "${ZSTD_INCLUDE_DIR}") +endif() + +if(ZSTD_FOUND AND NOT TARGET zstd::zstd) + add_library(zstd::zstd INTERFACE IMPORTED) + target_include_directories(zstd::zstd INTERFACE ${ZSTD_INCLUDE_DIRS}) + target_link_libraries(zstd::zstd INTERFACE "${ZSTD_LIBRARY}") +endif() diff --git a/contrib/docker/ubuntu/noble/Dockerfile b/contrib/docker/ubuntu/noble/Dockerfile index d5c90227dfa..d8f4ef3627e 100644 --- a/contrib/docker/ubuntu/noble/Dockerfile +++ b/contrib/docker/ubuntu/noble/Dockerfile @@ -50,6 +50,7 @@ RUN apt update \ hwloc \ libbrotli-dev \ libzstd-dev \ + liblz4-dev \ luajit \ libluajit-5.1-dev \ libcap-dev \ diff --git a/contrib/docker/ubuntu/resolute/Dockerfile b/contrib/docker/ubuntu/resolute/Dockerfile index f0c8db3447d..67377c024a4 100644 --- a/contrib/docker/ubuntu/resolute/Dockerfile +++ b/contrib/docker/ubuntu/resolute/Dockerfile @@ -49,6 +49,7 @@ RUN apt update \ hwloc \ libbrotli-dev \ libzstd-dev \ + liblz4-dev \ luajit \ libluajit-5.1-dev \ libcap-dev \ diff --git a/doc/admin-guide/files/records.yaml.en.rst b/doc/admin-guide/files/records.yaml.en.rst index 5d75b68bafc..2bbc7016f83 100644 --- a/doc/admin-guide/files/records.yaml.en.rst +++ b/doc/admin-guide/files/records.yaml.en.rst @@ -3178,11 +3178,18 @@ RAM Cache Value Description ======== =================================================================== ``0`` No compression - ``1`` Fastlz (extremely fast, relatively low compression) - ``2`` Libz (moderate speed, reasonable compression) + ``1`` Fastlz (extremely fast, relatively low compression) - prefer lz4 + ``2`` Libz (moderate speed, reasonable compression) - prefer zstd ``3`` Liblzma (very slow, high compression) + ``4`` lz4 (extremely fast, relatively low compression) + ``5`` zstd (fast speed, reasonable compression) ======== =================================================================== + ``3``, ``4`` and ``5`` require that |TS| was built with liblzma, lz4 or + libzstd respectively; configuring one that was not compiled in is a fatal + error at startup. ``traffic_layout info`` reports which are available as + ``TS_HAS_LZ4`` and ``TS_HAS_ZSTD``. + Compression runs on task threads. To use more cores for RAM cache compression, increase :ts:cv:`proxy.config.task_threads`. diff --git a/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst b/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst index c6e39eb7449..ed4b0711d5e 100644 --- a/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/cache-volume.en.rst @@ -132,6 +132,16 @@ a configuration with only one cache volume: :literal:`0`. Accumulates the number of misses to the LRU RAM cache for this volume. Note that this count includes hits to the other memory caches, including the last open read and aggregation buffer caches, so it may not represent the total number of cache accesses that go to disk. +.. ts:stat:: global proxy.process.cache.volume_0.ram_cache.compress.failure integer + :type: counter + + Accumulates the number of RAM cache entries the compression library could not compress, for this volume. Objects that simply did not shrink enough to be worth compressing are not counted, since that is the ordinary outcome for already-compressed content. + +.. ts:stat:: global proxy.process.cache.volume_0.ram_cache.decompress.failure integer + :type: counter + + Accumulates the number of RAM cache entries that failed to decompress on read, for this volume. A failed entry is dropped from the RAM cache and the read is treated as a miss. A nonzero value indicates data corruption or a compression library error, not ordinary cache churn. + .. ts:stat:: global proxy.process.cache.volume_0.last_open_read.hits integer :type: counter diff --git a/doc/admin-guide/monitoring/statistics/core/cache.en.rst b/doc/admin-guide/monitoring/statistics/core/cache.en.rst index bb2f7d17f42..27dc8fba579 100644 --- a/doc/admin-guide/monitoring/statistics/core/cache.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/cache.en.rst @@ -95,6 +95,16 @@ Cache Accumulates the number of misses to the LRU RAM cache for all volumes. Note that this includes hits to the other memory caches, including the last open read and aggregation buffer caches, so it may not represent the total number of cache accesses that go to disk. +.. ts:stat:: global proxy.process.cache.ram_cache.compress.failure integer + :type: counter + + Accumulates the number of RAM cache entries the compression library could not compress, for all volumes. Objects that simply did not shrink enough to be worth compressing are not counted, since that is the ordinary outcome for already-compressed content. + +.. ts:stat:: global proxy.process.cache.ram_cache.decompress.failure integer + :type: counter + + Accumulates the number of RAM cache entries that failed to decompress on read, for all volumes. A failed entry is dropped from the RAM cache and the read is treated as a miss. A nonzero value indicates data corruption or a compression library error, not ordinary cache churn. + .. ts:stat:: global proxy.process.cache.last_open_read.hits integer :type: counter diff --git a/doc/admin-guide/storage/index.en.rst b/doc/admin-guide/storage/index.en.rst index af6d23de065..0b000223e93 100644 --- a/doc/admin-guide/storage/index.en.rst +++ b/doc/admin-guide/storage/index.en.rst @@ -98,18 +98,8 @@ images). This should not be confused with ``Content-Encoding: gzip``, this feature is only present to save space internally in the RAM cache itself. As such, it is completely transparent to the User-Agent. The RAM cache compression is enabled with the option -:ts:cv:`proxy.config.cache.ram_cache.compress`. - -Possible values are: - -======= ============================= -Value Meaning -======= ============================= -0 No compression (*default*) -1 *fastlz* compression -2 *libz* compression -3 *liblzma* compression -======= ============================= +:ts:cv:`proxy.config.cache.ram_cache.compress`, which documents the available +codecs and which of them a given build supports. .. _changing-the-size-of-the-ram-cache: diff --git a/doc/developer-guide/cache-architecture/ram-cache.en.rst b/doc/developer-guide/cache-architecture/ram-cache.en.rst index bb6851e2e2d..a28d7f148b8 100644 --- a/doc/developer-guide/cache-architecture/ram-cache.en.rst +++ b/doc/developer-guide/cache-architecture/ram-cache.en.rst @@ -37,7 +37,7 @@ following features: * Is Scan Resistant and extracts robust hit rates even when the working set does not fit in the RAM Cache. -* Supports compression at 3 levels: fastlz, gzip (libz), and xz (liblzma). +* Supports compression at 5 levels: fastlz, gzip (libz), xz (liblzma), lz4 and zstd. Compression can be moved to another thread. * Has very low CPU overhead, only slightly more than a basic LRU. Rather than @@ -72,9 +72,9 @@ len Length of the object, which differs from *size* because of compression and padding). compressed_len Compressed length of the object. compressed Compression type, or ``none`` if no compression. Possible types - are: *fastlz*, *libz*, and *liblzma*. -uncompressible Flag indicating that content cannot be compressed (true), or that - it mat be compressed (false). + are: *fastlz*, *libz*, *liblzma*, *lz4* and *zstd*. +incompressible Flag indicating that content cannot be compressed (true), or that + it may be compressed (false). copy Whether or not this object should be copied in and copied out (e.g. HTTP HDR). LRU link @@ -147,18 +147,28 @@ since we need to make a copy anyway. Those not tagged ``copy`` are inserted uncompressed in the hope that they can be reused in uncompressed form. This is a compile time option and may be something we want to change. -There are 3 algorithms and levels of compression (speed on an Intel i7 920 -series processor using one thread): - -======= ================ ================== ==================================== -Method Compression Rate Decompression Rate Notes -======= ================ ================== ==================================== -fastlz 173 MB/sec 442 MB/sec Basically free since disk or network - will limit first; ~53% final size. -libz 55 MB/sec 234 MB/sec Almost free, particularly - decompression; ~37% final size. -liblzma 3 MB/sec 50 MB/sec Expensive; ~27% final size. -======= ================ ================== ==================================== +There are 5 algorithms and levels of compression (speed on an Intel Xeon Gold +6338 processor using lzbench and the silesia XML corpus): + +======= ===== ================= ================== ==================================== +Method Level Compression Rate Decompression Rate Notes +======= ===== ================= ================== ==================================== +fastlz 1/2 452 MB/sec 913 MB/sec Effectively obsolete; prefer lz4. + fastlz_compress() selects + level 2 at 64 KiB and above, + so most objects use it; the + figures here are level 1. + Basically free since disk or network + will limit first; ~26% final size. +libz 6 54 MB/sec 536 MB/sec Effectively obsolete; prefer zstd. + Almost free, particularly + decompression; ~13% final size. +liblzma 6 5 MB/sec 291 MB/sec Expensive; ~8% final size. +lz4 1 727 MB/sec 3458 MB/sec Basically free since disk or network + will limit first; ~23% final size. +zstd 3 508 MB/sec 1690 MB/sec Basically free since disk or network + will limit first; ~12% final size. +======= ===== ================= ================== ==================================== These are ballpark numbers, and your millage will vary enormously. JPEG, for example, will not compress with any of these (or at least will only do so at diff --git a/include/iocore/cache/Cache.h b/include/iocore/cache/Cache.h index 9a0fe5d430b..a7eea390971 100644 --- a/include/iocore/cache/Cache.h +++ b/include/iocore/cache/Cache.h @@ -44,8 +44,31 @@ static constexpr ts::ModuleVersion CACHE_MODULE_VERSION(1, 0); #define CACHE_COMPRESSION_FASTLZ 1 #define CACHE_COMPRESSION_LIBZ 2 #define CACHE_COMPRESSION_LIBLZMA 3 +#define CACHE_COMPRESSION_LZ4 4 +#define CACHE_COMPRESSION_ZSTD 5 + +enum { + RAM_HIT_COMPRESS_NONE = 1, + RAM_HIT_COMPRESS_FASTLZ, + RAM_HIT_COMPRESS_LIBZ, + RAM_HIT_COMPRESS_LIBLZMA, + RAM_HIT_COMPRESS_LZ4, + RAM_HIT_COMPRESS_ZSTD, + RAM_HIT_LAST_ENTRY +}; -enum { RAM_HIT_COMPRESS_NONE = 1, RAM_HIT_COMPRESS_FASTLZ, RAM_HIT_COMPRESS_LIBZ, RAM_HIT_COMPRESS_LIBLZMA, RAM_HIT_LAST_ENTRY }; +// The RAM_HIT_COMPRESS_* values are the CACHE_COMPRESSION_* values offset by +// one; keep the two sequences from silently desyncing when a codec is added. +// The pairwise asserts catch a reordering of either sequence; the count assert +// catches a new CACHE_COMPRESSION_* that was never given a RAM_HIT_COMPRESS_* +// enumerator (or vice versa). +static_assert(RAM_HIT_LAST_ENTRY == CACHE_COMPRESSION_ZSTD + 2); +static_assert(RAM_HIT_COMPRESS_NONE == CACHE_COMPRESSION_NONE + 1); +static_assert(RAM_HIT_COMPRESS_FASTLZ == CACHE_COMPRESSION_FASTLZ + 1); +static_assert(RAM_HIT_COMPRESS_LIBZ == CACHE_COMPRESSION_LIBZ + 1); +static_assert(RAM_HIT_COMPRESS_LIBLZMA == CACHE_COMPRESSION_LIBLZMA + 1); +static_assert(RAM_HIT_COMPRESS_LZ4 == CACHE_COMPRESSION_LZ4 + 1); +static_assert(RAM_HIT_COMPRESS_ZSTD == CACHE_COMPRESSION_ZSTD + 1); struct CacheVC; class CacheEvacuateDocVC; diff --git a/include/tscore/ink_config.h.cmake.in b/include/tscore/ink_config.h.cmake.in index 40b52686c7c..681c99581cc 100644 --- a/include/tscore/ink_config.h.cmake.in +++ b/include/tscore/ink_config.h.cmake.in @@ -47,6 +47,7 @@ #cmakedefine HAVE_POSIX_FALLOCATE 1 #cmakedefine HAVE_POSIX_MADVISE 1 #cmakedefine HAVE_ZSTD_H 1 +#cmakedefine HAVE_LZ4_H 1 #cmakedefine HAVE_PTHREAD_GETNAME_NP 1 #cmakedefine HAVE_PTHREAD_GET_NAME_NP 1 diff --git a/src/iocore/cache/CMakeLists.txt b/src/iocore/cache/CMakeLists.txt index c9ba5bd2fde..a997dada83f 100644 --- a/src/iocore/cache/CMakeLists.txt +++ b/src/iocore/cache/CMakeLists.txt @@ -58,6 +58,14 @@ if(HAVE_LZMA_H) target_link_libraries(inkcache PRIVATE LibLZMA::LibLZMA) endif() +if(HAVE_LZ4_H) + target_link_libraries(inkcache PRIVATE LZ4::LZ4) +endif() + +if(HAVE_ZSTD_H) + target_link_libraries(inkcache PRIVATE zstd::zstd) +endif() + if(BUILD_TESTING) # Unit Tests with unit_tests/main.cc macro(add_cache_test name) @@ -93,6 +101,7 @@ if(BUILD_TESTING) add_cache_test(Update_Header unit_tests/test_Update_header.cc) add_cache_test(CacheStripe unit_tests/test_Stripe.cc) add_cache_test(CacheAggregateWriteBuffer unit_tests/test_AggregateWriteBuffer.cc) + add_cache_test(RamCacheCLFUS unit_tests/test_RamCacheCLFUS.cc) add_cache_test(RamCacheCompressEntries unit_tests/test_RamCacheCompressEntries.cc) # Only the shutdown test attaches a live segment; the rest need no shm syscall. add_cache_test(CacheShm unit_tests/test_CacheShm.cc) diff --git a/src/iocore/cache/Cache.cc b/src/iocore/cache/Cache.cc index d5dcf3df76b..1fad04027b5 100644 --- a/src/iocore/cache/Cache.cc +++ b/src/iocore/cache/Cache.cc @@ -866,6 +866,34 @@ ink_cache_init(ts::ModuleVersion v) RecEstablishStaticConfigInt32(cache_config_ram_cache_algorithm, "proxy.config.cache.ram_cache.algorithm"); RecEstablishStaticConfigInt32(cache_config_ram_cache_compress, "proxy.config.cache.ram_cache.compress"); + // Validate here, where the value is read: this runs before any stripe exists + // and the record is RECU_RESTART_TS, so a bad codec cannot reach the RAM + // cache and nothing downstream needs to re-check it. + switch (cache_config_ram_cache_compress) { + case CACHE_COMPRESSION_NONE: + case CACHE_COMPRESSION_FASTLZ: + case CACHE_COMPRESSION_LIBZ: + break; + case CACHE_COMPRESSION_LIBLZMA: +#ifndef HAVE_LZMA_H + Fatal("proxy.config.cache.ram_cache.compress is %d (liblzma), but this build has no liblzma support", + cache_config_ram_cache_compress); +#endif + break; + case CACHE_COMPRESSION_LZ4: +#ifndef HAVE_LZ4_H + Fatal("proxy.config.cache.ram_cache.compress is %d (lz4), but this build has no lz4 support", cache_config_ram_cache_compress); +#endif + break; + case CACHE_COMPRESSION_ZSTD: +#ifndef HAVE_ZSTD_H + Fatal("proxy.config.cache.ram_cache.compress is %d (zstd), but this build has no zstd support", + cache_config_ram_cache_compress); +#endif + break; + default: + Fatal("proxy.config.cache.ram_cache.compress has unknown value %d", cache_config_ram_cache_compress); + } RecEstablishStaticConfigInt32(cache_config_ram_cache_compress_percent, "proxy.config.cache.ram_cache.compress_percent"); cache_config_ram_cache_use_seen_filter = RecGetRecordInt("proxy.config.cache.ram_cache.use_seen_filter").value_or(0); diff --git a/src/iocore/cache/CacheProcessor.cc b/src/iocore/cache/CacheProcessor.cc index c178aeb959d..299025735ba 100644 --- a/src/iocore/cache/CacheProcessor.cc +++ b/src/iocore/cache/CacheProcessor.cc @@ -1183,43 +1183,45 @@ register_cache_stats(CacheStatsBlock *rsb, const std::string &prefix) rsb->fragment_document_count[2] = ts::Metrics::Counter::createPtr(prefix + ".frags_per_doc.3+"); // And then everything else - rsb->bytes_used = ts::Metrics::Gauge::createPtr(prefix + ".bytes_used"); - rsb->bytes_total = ts::Metrics::Gauge::createPtr(prefix + ".bytes_total"); - rsb->stripes = ts::Metrics::Gauge::createPtr(prefix + ".stripes"); - rsb->ram_cache_bytes_total = ts::Metrics::Gauge::createPtr(prefix + ".ram_cache.total_bytes"); - rsb->ram_cache_bytes = ts::Metrics::Gauge::createPtr(prefix + ".ram_cache.bytes_used"); - rsb->ram_cache_hits = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.hits"); - rsb->last_open_read_hits = ts::Metrics::Counter::createPtr(prefix + ".last_open_read.hits"); - rsb->agg_buffer_hits = ts::Metrics::Counter::createPtr(prefix + ".aggregation_buffer.hits"); - rsb->ram_cache_misses = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.misses"); - rsb->all_mem_misses = ts::Metrics::Counter::createPtr(prefix + ".all_memory_caches.misses"); - rsb->pread_count = ts::Metrics::Counter::createPtr(prefix + ".pread_count"); - rsb->percent_full = ts::Metrics::Gauge::createPtr(prefix + ".percent_full"); - rsb->read_seek_fail = ts::Metrics::Counter::createPtr(prefix + ".read.seek.failure"); - rsb->read_invalid = ts::Metrics::Counter::createPtr(prefix + ".read.invalid"); - rsb->write_backlog_failure = ts::Metrics::Counter::createPtr(prefix + ".write.backlog.failure"); - rsb->direntries_total = ts::Metrics::Gauge::createPtr(prefix + ".direntries.total"); - rsb->direntries_used = ts::Metrics::Gauge::createPtr(prefix + ".direntries.used"); - rsb->directory_collision = ts::Metrics::Counter::createPtr(prefix + ".directory_collision"); - rsb->read_busy_success = ts::Metrics::Counter::createPtr(prefix + ".read_busy.success"); - rsb->read_busy_failure = ts::Metrics::Counter::createPtr(prefix + ".read_busy.failure"); - rsb->write_bytes = ts::Metrics::Counter::createPtr(prefix + ".write_bytes_stat"); - rsb->hdr_vector_marshal = ts::Metrics::Counter::createPtr(prefix + ".vector_marshals"); - rsb->hdr_marshal = ts::Metrics::Counter::createPtr(prefix + ".hdr_marshals"); - rsb->hdr_marshal_bytes = ts::Metrics::Counter::createPtr(prefix + ".hdr_marshal_bytes"); - rsb->gc_bytes_evacuated = ts::Metrics::Counter::createPtr(prefix + ".gc_bytes_evacuated"); - rsb->gc_frags_evacuated = ts::Metrics::Counter::createPtr(prefix + ".gc_frags_evacuated"); - rsb->directory_wrap = ts::Metrics::Counter::createPtr(prefix + ".wrap_count"); - rsb->directory_sync_count = ts::Metrics::Counter::createPtr(prefix + ".sync.count"); - rsb->directory_sync_bytes = ts::Metrics::Counter::createPtr(prefix + ".sync.bytes"); - rsb->directory_sync_time = ts::Metrics::Counter::createPtr(prefix + ".sync.time"); - rsb->span_errors_read = ts::Metrics::Counter::createPtr(prefix + ".span.errors.read"); - rsb->span_errors_write = ts::Metrics::Counter::createPtr(prefix + ".span.errors.write"); - rsb->span_failing = ts::Metrics::Gauge::createPtr(prefix + ".span.failing"); - rsb->span_offline = ts::Metrics::Gauge::createPtr(prefix + ".span.offline"); - rsb->span_online = ts::Metrics::Gauge::createPtr(prefix + ".span.online"); - rsb->stripe_lock_contention = ts::Metrics::Counter::createPtr(prefix + ".stripe.lock_contention"); - rsb->writer_lock_contention = ts::Metrics::Counter::createPtr(prefix + ".writer.lock_contention"); + rsb->bytes_used = ts::Metrics::Gauge::createPtr(prefix + ".bytes_used"); + rsb->bytes_total = ts::Metrics::Gauge::createPtr(prefix + ".bytes_total"); + rsb->stripes = ts::Metrics::Gauge::createPtr(prefix + ".stripes"); + rsb->ram_cache_bytes_total = ts::Metrics::Gauge::createPtr(prefix + ".ram_cache.total_bytes"); + rsb->ram_cache_bytes = ts::Metrics::Gauge::createPtr(prefix + ".ram_cache.bytes_used"); + rsb->ram_cache_hits = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.hits"); + rsb->last_open_read_hits = ts::Metrics::Counter::createPtr(prefix + ".last_open_read.hits"); + rsb->agg_buffer_hits = ts::Metrics::Counter::createPtr(prefix + ".aggregation_buffer.hits"); + rsb->ram_cache_misses = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.misses"); + rsb->ram_cache_compress_failures = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.compress.failure"); + rsb->ram_cache_decompress_failures = ts::Metrics::Counter::createPtr(prefix + ".ram_cache.decompress.failure"); + rsb->all_mem_misses = ts::Metrics::Counter::createPtr(prefix + ".all_memory_caches.misses"); + rsb->pread_count = ts::Metrics::Counter::createPtr(prefix + ".pread_count"); + rsb->percent_full = ts::Metrics::Gauge::createPtr(prefix + ".percent_full"); + rsb->read_seek_fail = ts::Metrics::Counter::createPtr(prefix + ".read.seek.failure"); + rsb->read_invalid = ts::Metrics::Counter::createPtr(prefix + ".read.invalid"); + rsb->write_backlog_failure = ts::Metrics::Counter::createPtr(prefix + ".write.backlog.failure"); + rsb->direntries_total = ts::Metrics::Gauge::createPtr(prefix + ".direntries.total"); + rsb->direntries_used = ts::Metrics::Gauge::createPtr(prefix + ".direntries.used"); + rsb->directory_collision = ts::Metrics::Counter::createPtr(prefix + ".directory_collision"); + rsb->read_busy_success = ts::Metrics::Counter::createPtr(prefix + ".read_busy.success"); + rsb->read_busy_failure = ts::Metrics::Counter::createPtr(prefix + ".read_busy.failure"); + rsb->write_bytes = ts::Metrics::Counter::createPtr(prefix + ".write_bytes_stat"); + rsb->hdr_vector_marshal = ts::Metrics::Counter::createPtr(prefix + ".vector_marshals"); + rsb->hdr_marshal = ts::Metrics::Counter::createPtr(prefix + ".hdr_marshals"); + rsb->hdr_marshal_bytes = ts::Metrics::Counter::createPtr(prefix + ".hdr_marshal_bytes"); + rsb->gc_bytes_evacuated = ts::Metrics::Counter::createPtr(prefix + ".gc_bytes_evacuated"); + rsb->gc_frags_evacuated = ts::Metrics::Counter::createPtr(prefix + ".gc_frags_evacuated"); + rsb->directory_wrap = ts::Metrics::Counter::createPtr(prefix + ".wrap_count"); + rsb->directory_sync_count = ts::Metrics::Counter::createPtr(prefix + ".sync.count"); + rsb->directory_sync_bytes = ts::Metrics::Counter::createPtr(prefix + ".sync.bytes"); + rsb->directory_sync_time = ts::Metrics::Counter::createPtr(prefix + ".sync.time"); + rsb->span_errors_read = ts::Metrics::Counter::createPtr(prefix + ".span.errors.read"); + rsb->span_errors_write = ts::Metrics::Counter::createPtr(prefix + ".span.errors.write"); + rsb->span_failing = ts::Metrics::Gauge::createPtr(prefix + ".span.failing"); + rsb->span_offline = ts::Metrics::Gauge::createPtr(prefix + ".span.offline"); + rsb->span_online = ts::Metrics::Gauge::createPtr(prefix + ".span.online"); + rsb->stripe_lock_contention = ts::Metrics::Counter::createPtr(prefix + ".stripe.lock_contention"); + rsb->writer_lock_contention = ts::Metrics::Counter::createPtr(prefix + ".writer.lock_contention"); } // Copy the per-volume tuning fields from the volume config onto the CacheVol. @@ -1666,21 +1668,6 @@ CacheProcessor::cacheInitialized() used_direntries += vol_used_direntries; } - switch (cache_config_ram_cache_compress) { - default: - Fatal("unknown RAM cache compression type: %d", cache_config_ram_cache_compress); - case CACHE_COMPRESSION_NONE: - case CACHE_COMPRESSION_FASTLZ: - break; - case CACHE_COMPRESSION_LIBZ: - break; - case CACHE_COMPRESSION_LIBLZMA: -#ifndef HAVE_LZMA_H - Fatal("lzma not available for RAM cache compression"); -#endif - break; - } - ts::Metrics::Gauge::store(cache_rsb.ram_cache_bytes_total, total_ram_cache_bytes); ts::Metrics::Gauge::store(cache_rsb.bytes_total, total_cache_bytes); ts::Metrics::Gauge::store(cache_rsb.direntries_total, total_direntries); diff --git a/src/iocore/cache/P_CacheStats.h b/src/iocore/cache/P_CacheStats.h index 7673cf56e0e..809ce20645a 100644 --- a/src/iocore/cache/P_CacheStats.h +++ b/src/iocore/cache/P_CacheStats.h @@ -37,41 +37,43 @@ struct CacheStatsBlock { ts::Metrics::Counter::AtomicType *fragment_document_count[3] = {nullptr, nullptr, nullptr}; // For 1, 2 and 3+ fragments - ts::Metrics::Gauge::AtomicType *bytes_used = nullptr; - ts::Metrics::Gauge::AtomicType *bytes_total = nullptr; - ts::Metrics::Gauge::AtomicType *stripes = nullptr; - ts::Metrics::Gauge::AtomicType *ram_cache_bytes = nullptr; - ts::Metrics::Gauge::AtomicType *ram_cache_bytes_total = nullptr; - ts::Metrics::Gauge::AtomicType *direntries_total = nullptr; - ts::Metrics::Gauge::AtomicType *direntries_used = nullptr; - ts::Metrics::Counter::AtomicType *ram_cache_hits = nullptr; - ts::Metrics::Counter::AtomicType *last_open_read_hits = nullptr; - ts::Metrics::Counter::AtomicType *agg_buffer_hits = nullptr; - ts::Metrics::Counter::AtomicType *ram_cache_misses = nullptr; - ts::Metrics::Counter::AtomicType *all_mem_misses = nullptr; - ts::Metrics::Counter::AtomicType *pread_count = nullptr; - ts::Metrics::Gauge::AtomicType *percent_full = nullptr; - ts::Metrics::Counter::AtomicType *read_seek_fail = nullptr; - ts::Metrics::Counter::AtomicType *read_invalid = nullptr; - ts::Metrics::Counter::AtomicType *write_backlog_failure = nullptr; - ts::Metrics::Counter::AtomicType *directory_collision = nullptr; - ts::Metrics::Counter::AtomicType *read_busy_success = nullptr; - ts::Metrics::Counter::AtomicType *read_busy_failure = nullptr; - ts::Metrics::Counter::AtomicType *gc_bytes_evacuated = nullptr; - ts::Metrics::Counter::AtomicType *gc_frags_evacuated = nullptr; - ts::Metrics::Counter::AtomicType *write_bytes = nullptr; - ts::Metrics::Counter::AtomicType *hdr_vector_marshal = nullptr; - ts::Metrics::Counter::AtomicType *hdr_marshal = nullptr; - ts::Metrics::Counter::AtomicType *hdr_marshal_bytes = nullptr; - ts::Metrics::Counter::AtomicType *directory_wrap = nullptr; - ts::Metrics::Counter::AtomicType *directory_sync_count = nullptr; - ts::Metrics::Counter::AtomicType *directory_sync_time = nullptr; - ts::Metrics::Counter::AtomicType *directory_sync_bytes = nullptr; - ts::Metrics::Counter::AtomicType *span_errors_read = nullptr; - ts::Metrics::Counter::AtomicType *span_errors_write = nullptr; - ts::Metrics::Gauge::AtomicType *span_offline = nullptr; - ts::Metrics::Gauge::AtomicType *span_online = nullptr; - ts::Metrics::Gauge::AtomicType *span_failing = nullptr; - ts::Metrics::Counter::AtomicType *stripe_lock_contention = nullptr; - ts::Metrics::Counter::AtomicType *writer_lock_contention = nullptr; + ts::Metrics::Gauge::AtomicType *bytes_used = nullptr; + ts::Metrics::Gauge::AtomicType *bytes_total = nullptr; + ts::Metrics::Gauge::AtomicType *stripes = nullptr; + ts::Metrics::Gauge::AtomicType *ram_cache_bytes = nullptr; + ts::Metrics::Gauge::AtomicType *ram_cache_bytes_total = nullptr; + ts::Metrics::Gauge::AtomicType *direntries_total = nullptr; + ts::Metrics::Gauge::AtomicType *direntries_used = nullptr; + ts::Metrics::Counter::AtomicType *ram_cache_hits = nullptr; + ts::Metrics::Counter::AtomicType *last_open_read_hits = nullptr; + ts::Metrics::Counter::AtomicType *agg_buffer_hits = nullptr; + ts::Metrics::Counter::AtomicType *ram_cache_misses = nullptr; + ts::Metrics::Counter::AtomicType *ram_cache_compress_failures = nullptr; + ts::Metrics::Counter::AtomicType *ram_cache_decompress_failures = nullptr; + ts::Metrics::Counter::AtomicType *all_mem_misses = nullptr; + ts::Metrics::Counter::AtomicType *pread_count = nullptr; + ts::Metrics::Gauge::AtomicType *percent_full = nullptr; + ts::Metrics::Counter::AtomicType *read_seek_fail = nullptr; + ts::Metrics::Counter::AtomicType *read_invalid = nullptr; + ts::Metrics::Counter::AtomicType *write_backlog_failure = nullptr; + ts::Metrics::Counter::AtomicType *directory_collision = nullptr; + ts::Metrics::Counter::AtomicType *read_busy_success = nullptr; + ts::Metrics::Counter::AtomicType *read_busy_failure = nullptr; + ts::Metrics::Counter::AtomicType *gc_bytes_evacuated = nullptr; + ts::Metrics::Counter::AtomicType *gc_frags_evacuated = nullptr; + ts::Metrics::Counter::AtomicType *write_bytes = nullptr; + ts::Metrics::Counter::AtomicType *hdr_vector_marshal = nullptr; + ts::Metrics::Counter::AtomicType *hdr_marshal = nullptr; + ts::Metrics::Counter::AtomicType *hdr_marshal_bytes = nullptr; + ts::Metrics::Counter::AtomicType *directory_wrap = nullptr; + ts::Metrics::Counter::AtomicType *directory_sync_count = nullptr; + ts::Metrics::Counter::AtomicType *directory_sync_time = nullptr; + ts::Metrics::Counter::AtomicType *directory_sync_bytes = nullptr; + ts::Metrics::Counter::AtomicType *span_errors_read = nullptr; + ts::Metrics::Counter::AtomicType *span_errors_write = nullptr; + ts::Metrics::Gauge::AtomicType *span_offline = nullptr; + ts::Metrics::Gauge::AtomicType *span_online = nullptr; + ts::Metrics::Gauge::AtomicType *span_failing = nullptr; + ts::Metrics::Counter::AtomicType *stripe_lock_contention = nullptr; + ts::Metrics::Counter::AtomicType *writer_lock_contention = nullptr; }; diff --git a/src/iocore/cache/RamCacheCLFUS.cc b/src/iocore/cache/RamCacheCLFUS.cc index dcabd4c85c3..82146e2753e 100644 --- a/src/iocore/cache/RamCacheCLFUS.cc +++ b/src/iocore/cache/RamCacheCLFUS.cc @@ -33,10 +33,62 @@ #include "fastlz/fastlz.h" #include "tscore/CryptoHash.h" #include "tscore/Regression.h" + #include #ifdef HAVE_LZMA_H #include #endif +#ifdef HAVE_LZ4_H +#include +#endif +#ifdef HAVE_ZSTD_H +#include +// ZSTD_getErrorCode() and the ZSTD_error_* codes live here, not in zstd.h. +#include +#include +constexpr int CLFUS_ZSTD_LEVEL = 3; + +namespace +{ + +// One-shot ZSTD_compress/ZSTD_decompress allocate and free a context on every +// call, so reuse a per-thread context instead. May return nullptr if zstd +// fails to allocate one; that failure is sticky for the life of the thread, so +// warn when it happens. The compression level is a sticky parameter set once +// here; no explicit ZSTD_CCtx_reset() is needed because ZSTD_compress2() +// starts a new session on every call (resets are only for interrupting the +// streaming API or changing sticky parameters). +ZSTD_CCtx * +zstd_cctx() +{ + thread_local std::unique_ptr ctx = [] { + std::unique_ptr c{ZSTD_createCCtx(), ZSTD_freeCCtx}; + if (c && ZSTD_isError(ZSTD_CCtx_setParameter(c.get(), ZSTD_c_compressionLevel, CLFUS_ZSTD_LEVEL))) { + c.reset(); + } + if (!c) { + Warning("unable to allocate zstd compression context; RAM cache entries will not be compressed on this thread"); + } + return c; + }(); + return ctx.get(); +} + +ZSTD_DCtx * +zstd_dctx() +{ + thread_local std::unique_ptr ctx = [] { + std::unique_ptr c{ZSTD_createDCtx(), ZSTD_freeDCtx}; + if (!c) { + Warning("unable to allocate zstd decompression context; compressed RAM cache entries will miss on this thread"); + } + return c; + }(); + return ctx.get(); +} + +} // end anonymous namespace +#endif // #define CHECK_ACOUNTING 1 // very expensive double checking of all sizes @@ -111,21 +163,7 @@ class RamCacheCLFUSCompressor : public Continuation int RamCacheCLFUSCompressor::mainEvent(int /* event ATS_UNUSED */, Event *e) { - switch (cache_config_ram_cache_compress) { - default: - Warning("unknown RAM cache compression type: %d", cache_config_ram_cache_compress); - case CACHE_COMPRESSION_NONE: - case CACHE_COMPRESSION_FASTLZ: - break; - case CACHE_COMPRESSION_LIBZ: - Warning("libz not available for RAM cache compression"); - break; - case CACHE_COMPRESSION_LIBLZMA: -#ifndef HAVE_LZMA_H - Warning("lzma not available for RAM cache compression"); -#endif - break; - } + // The codec is validated once in ink_cache_init(), before any cache exists. if (cache_config_ram_cache_compress_percent) { rc->compress_entries(e->ethread); } @@ -138,6 +176,44 @@ static const int bucket_sizes[] = {127, 251, 509, 1021, 203 65521, 131071, 262139, 524287, 1048573, 2097143, 4194301, 8388593, 16777213, 33554393, 67108859, 134217689, 268435399, 536870909, 1073741789, 2147483647}; +// Only safe when init() did not schedule the background compressor, i.e. when +// cache_config_ram_cache_compress was CACHE_COMPRESSION_NONE at init() time. +// That scheduled RamCacheCLFUSCompressor holds a raw back-pointer to this +// object and nothing cancels it, so destroying a cache that has one would +// leave it dangling. Cancelling the event here would not be enough either: the +// continuation carries no mutex, so EventProcessor::schedule leaves the event +// with none and it can be running compress_entries() on an ET_TASK thread +// while this destructor runs. Making that safe means giving the compressor its +// own ProxyMutex and cancelling the retained Event under it -- not the stripe +// mutex, because Mutex_unlock() only decrements nthread_holding, so a +// continuation dispatched holding stripe->mutex would keep the stripe locked +// across the codec call and defeat the lock drop in compress_entries(). Not +// worth doing while production never destroys a RamCacheCLFUS -- these live +// for the lifetime of their StripeSM. Unit tests that construct one directly +// must init() with compression off and drive compress_entries() synchronously. +RamCacheCLFUS::~RamCacheCLFUS() +{ + // Entries are pool-allocated without running their destructor, so release the + // data reference explicitly before returning each one to the allocator, then + // free the hash table and the seen filter. + // History entries (lru[1]) hold no data and were never counted. + while (RamCacheCLFUSEntry *e = this->_lru[0].dequeue()) { + this->_bytes -= e->size + entry_overhead; + ts::Metrics::Gauge::decrement(cache_rsb.ram_cache_bytes, e->size); + ts::Metrics::Gauge::decrement(stripe->cache_vol->vol_rsb.ram_cache_bytes, e->size); + this->_objects--; + e->data = nullptr; + THREAD_FREE(e, ramCacheCLFUSEntryAllocator, this_thread()); + } + while (RamCacheCLFUSEntry *e = this->_lru[1].dequeue()) { + this->_history--; + e->data = nullptr; + THREAD_FREE(e, ramCacheCLFUSEntryAllocator, this_thread()); + } + ats_free(this->_bucket); + ats_free(this->_seen); +} + void RamCacheCLFUS::_resize_hashtable() { @@ -205,6 +281,27 @@ check_accounting(RamCacheCLFUS *c) #define check_accounting(_c) #endif +namespace +{ + +// Record a RAM cache decompression failure. This is data corruption or a codec +// error rather than an ordinary miss, so it has to be visible outside a debug +// build: a warning carrying the codec's own diagnosis, which tells a corrupt +// frame apart from a bookkeeping error in e->len, plus the global and +// per-volume counters. Call while the entry is still intact. Throttled through +// the site-wide facility so it honors proxy.config.log.throttling_interval_msec +// and reports its own suppression count. +void +note_decompress_failure(StripeSM *stripe, const CryptoHash *key, const RamCacheCLFUSEntry *e, const char *detail) +{ + SiteThrottledWarning("RAM cache decompression failed: type %d len %u compressed_len %u key %X: %s; entry dropped", + static_cast(e->flag_bits.compressed), e->len, e->compressed_len, key->slice32(3), detail); + ts::Metrics::Counter::increment(cache_rsb.ram_cache_decompress_failures); + ts::Metrics::Counter::increment(stripe->cache_vol->vol_rsb.ram_cache_decompress_failures); +} + +} // end anonymous namespace + int RamCacheCLFUS::get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey) { @@ -228,19 +325,27 @@ RamCacheCLFUS::get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey b = static_cast(ats_malloc(e->len)); switch (e->flag_bits.compressed) { default: + note_decompress_failure(stripe, key, e, "no decoder for this compression type"); goto Lfailed; case CACHE_COMPRESSION_FASTLZ: { - int l = static_cast(e->len); - if ((l != fastlz_decompress(e->data->data(), e->compressed_len, b, l))) { + int l = static_cast(e->len); + int rc = fastlz_decompress(e->data->data(), e->compressed_len, b, l); + if (l != rc) { + char detail[128]; + snprintf(detail, sizeof(detail), "fastlz_decompress produced %d bytes, expected %d", rc, l); + note_decompress_failure(stripe, key, e, detail); goto Lfailed; } ram_hit_state = RAM_HIT_COMPRESS_FASTLZ; break; } case CACHE_COMPRESSION_LIBZ: { - uLongf l = e->len; - if (Z_OK != - uncompress(reinterpret_cast(b), &l, reinterpret_cast(e->data->data()), e->compressed_len)) { + uLongf l = e->len; + int rc = uncompress(reinterpret_cast(b), &l, reinterpret_cast(e->data->data()), e->compressed_len); + if (Z_OK != rc) { + char detail[128]; + snprintf(detail, sizeof(detail), "uncompress: %s", zError(rc)); + note_decompress_failure(stripe, key, e, detail); goto Lfailed; } ram_hit_state = RAM_HIT_COMPRESS_LIBZ; @@ -250,13 +355,61 @@ RamCacheCLFUS::get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey case CACHE_COMPRESSION_LIBLZMA: { size_t l = static_cast(e->len), ipos = 0, opos = 0; uint64_t memlimit = e->len * 2 + lzma_base_memlimit; - if (LZMA_OK != lzma_stream_buffer_decode(&memlimit, 0, nullptr, reinterpret_cast(e->data->data()), &ipos, - e->compressed_len, reinterpret_cast(b), &opos, l)) { + lzma_ret rc = lzma_stream_buffer_decode(&memlimit, 0, nullptr, reinterpret_cast(e->data->data()), &ipos, + e->compressed_len, reinterpret_cast(b), &opos, l); + if (LZMA_OK != rc) { + char detail[128]; + snprintf(detail, sizeof(detail), "lzma_stream_buffer_decode returned %d, wrote %zu of %zu output bytes", + static_cast(rc), opos, l); + note_decompress_failure(stripe, key, e, detail); goto Lfailed; } ram_hit_state = RAM_HIT_COMPRESS_LIBLZMA; break; } +#endif +#ifdef HAVE_LZ4_H + case CACHE_COMPRESSION_LZ4: { + int l = static_cast(e->len); + int rc = LZ4_decompress_safe(e->data->data(), b, e->compressed_len, l); + if (l != rc) { + // A negative return is a malformed frame; a smaller non-negative + // one means e->len disagrees with the frame's content. + char detail[128]; + snprintf(detail, sizeof(detail), "LZ4_decompress_safe returned %d, expected %d", rc, l); + note_decompress_failure(stripe, key, e, detail); + goto Lfailed; + } + ram_hit_state = RAM_HIT_COMPRESS_LZ4; + break; + } +#endif +#ifdef HAVE_ZSTD_H + case CACHE_COMPRESSION_ZSTD: { + size_t l = static_cast(e->len); + ZSTD_DCtx *dctx = zstd_dctx(); + if (dctx == nullptr) { + // This thread can't decompress, but the entry itself is fine: + // miss instead of evicting it. + ats_free(b); + goto Lerror; + } + size_t ll = ZSTD_decompressDCtx(dctx, b, l, e->data->data(), e->compressed_len); + if (ZSTD_isError(ll)) { + char detail[128]; + snprintf(detail, sizeof(detail), "ZSTD_decompressDCtx: %s", ZSTD_getErrorName(ll)); + note_decompress_failure(stripe, key, e, detail); + goto Lfailed; + } + if (l != ll) { + char detail[128]; + snprintf(detail, sizeof(detail), "ZSTD_decompressDCtx produced %zu bytes, expected %zu", ll, l); + note_decompress_failure(stripe, key, e, detail); + goto Lfailed; + } + ram_hit_state = RAM_HIT_COMPRESS_ZSTD; + break; + } #endif } IOBufferData *data = new_xmalloc_IOBufferData(b, e->len); @@ -301,6 +454,8 @@ RamCacheCLFUS::get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey return 0; Lfailed: + // Every branch above reported the failure through note_decompress_failure() + // while the entry was still intact; this only tears it down. ats_free(b); this->_destroy(e); DDbg(dbg_ctl_ram_cache, "get %X %" PRId64 " Z_ERR", key->slice32(3), auxkey); @@ -390,6 +545,22 @@ RamCacheCLFUS::compress_entries(EThread *thread, int do_at_most) return; } ink_assert(stripe != nullptr); +#ifdef HAVE_ZSTD_H + if (cache_config_ram_cache_compress == CACHE_COMPRESSION_ZSTD && zstd_cctx() == nullptr) { + // The per-thread context failed to allocate, and that failure is sticky + // for the life of the thread this cache's compressor is pinned to, so no + // entry can be compressed on this pass or any later one. Skip the pass + // rather than walking every entry -- dropping and retaking the stripe lock + // and allocating a compressBound()-sized buffer for each -- only to fail + // every time. The entries are left untouched: this says nothing about the + // data, so they stay eligible. Deliberately not counted in + // ram_cache.compress.failure: that counter means entries the codec could + // not compress, and incrementing here once per pass would make it climb + // once a second per stripe for the life of the thread. The one-time + // Warning in zstd_cctx() is what reports this condition. + return; + } +#endif MUTEX_TAKE_LOCK(stripe->mutex, thread); if (!this->_compressed) { this->_compressed = this->_lru[0].head; @@ -428,7 +599,17 @@ RamCacheCLFUS::compress_entries(EThread *thread, int do_at_most) break; #ifdef HAVE_LZMA_H case CACHE_COMPRESSION_LIBLZMA: - l = e->len; + l = static_cast(lzma_stream_buffer_bound(e->len)); + break; +#endif +#ifdef HAVE_LZ4_H + case CACHE_COMPRESSION_LZ4: + l = static_cast(LZ4_compressBound(e->len)); + break; +#endif +#ifdef HAVE_ZSTD_H + case CACHE_COMPRESSION_ZSTD: + l = static_cast(ZSTD_compressBound(e->len)); break; #endif } @@ -439,6 +620,10 @@ RamCacheCLFUS::compress_entries(EThread *thread, int do_at_most) MUTEX_UNTAKE_LOCK(stripe->mutex, thread); b = static_cast(ats_malloc(l)); bool failed = false; + // Distinguishes a transient, data-independent failure (the codec could + // not allocate its working memory) from the codec rejecting this data. + // Only the latter says anything about the entry. + bool transient = false; switch (ctype) { default: // The bound switch above filtered unknown types; this is unreachable, @@ -452,8 +637,10 @@ RamCacheCLFUS::compress_entries(EThread *thread, int do_at_most) break; case CACHE_COMPRESSION_LIBZ: { uLongf ll = l; - if ((Z_OK != compress(reinterpret_cast(b), &ll, reinterpret_cast(edata->data()), elen))) { - failed = true; + int rc = compress(reinterpret_cast(b), &ll, reinterpret_cast(edata->data()), elen); + if (Z_OK != rc) { + failed = true; + transient = (rc == Z_MEM_ERROR); } l = static_cast(ll); break; @@ -469,6 +656,34 @@ RamCacheCLFUS::compress_entries(EThread *thread, int do_at_most) l = static_cast(pos); break; } +#endif +#ifdef HAVE_LZ4_H + case CACHE_COMPRESSION_LZ4: { + int ll = l; + if ((l = LZ4_compress_default(edata->data(), b, elen, ll)) == 0) { + failed = true; + } + break; + } +#endif +#ifdef HAVE_ZSTD_H + case CACHE_COMPRESSION_ZSTD: { + // The pass-level check above already proved this thread has a context, + // and the context is thread_local while the pass never changes thread. + ZSTD_CCtx *cctx = zstd_cctx(); + ink_assert(cctx != nullptr); + size_t zret = ZSTD_compress2(cctx, b, l, edata->data(), elen); + if (ZSTD_isError(zret)) { + failed = true; + // ZSTD_createCCtx() allocates only the context struct; the much + // larger working buffers are allocated on first use and grow with + // the input, so this is the realistic out-of-memory path. + transient = (ZSTD_getErrorCode(zret) == ZSTD_error_memory_allocation); + } else { + l = static_cast(zret); + } + break; + } #endif } MUTEX_TAKE_LOCK(stripe->mutex, thread); @@ -495,6 +710,15 @@ RamCacheCLFUS::compress_entries(EThread *thread, int do_at_most) } } if (failed) { + ts::Metrics::Counter::increment(cache_rsb.ram_cache_compress_failures); + ts::Metrics::Counter::increment(stripe->cache_vol->vol_rsb.ram_cache_compress_failures); + if (transient) { + // An allocation failure inside the codec is not a property of the + // data, so do not record it as permanently incompressible; leave the + // entry eligible for a later pass. + ats_free(b); + goto Lcontinue; + } goto Lfailed; } if (l > required_compression * e->len) { diff --git a/src/iocore/cache/RamCacheCLFUS.h b/src/iocore/cache/RamCacheCLFUS.h index 037e0cfb159..d4c13836d02 100644 --- a/src/iocore/cache/RamCacheCLFUS.h +++ b/src/iocore/cache/RamCacheCLFUS.h @@ -27,6 +27,7 @@ #include "P_RamCache.h" +#include "iocore/cache/Cache.h" #include "iocore/eventsystem/IOBuffer.h" #include "tscore/CryptoHash.h" #include "tscore/List.h" @@ -46,13 +47,18 @@ struct RamCacheCLFUSEntry { uint32_t compressed_len; union { struct { - uint32_t compressed : 3; // compression type + uint32_t compressed : 3; // compression type, a CACHE_COMPRESSION_* value uint32_t incompressible : 1; uint32_t lru : 1; uint32_t copy : 1; // copy-in-copy-out } flag_bits; uint32_t flags; }; + // The compression type is stored in the 3-bit flag_bits.compressed field + // above, so a newly added codec value must still fit. Checked here rather + // than beside the codec, so that a build without that codec still evaluates + // it. + static_assert(CACHE_COMPRESSION_ZSTD < (1 << 3)); LINK(RamCacheCLFUSEntry, lru_link); LINK(RamCacheCLFUSEntry, hash_link); Ptr data; @@ -62,6 +68,12 @@ class RamCacheCLFUS : public RamCache { public: RamCacheCLFUS() {} + ~RamCacheCLFUS() override; + + // Owns raw _bucket/_seen allocations and the pool-allocated entries, so + // copying one would double free all three. + RamCacheCLFUS(const RamCacheCLFUS &) = delete; + RamCacheCLFUS &operator=(const RamCacheCLFUS &) = delete; // returns 1 on found/stored, 0 on not found/stored, if provided auxkey1 and auxkey2 must match int get(CryptoHash *key, Ptr *ret_data, uint64_t auxkey = 0) override; @@ -90,6 +102,11 @@ class RamCacheCLFUS : public RamCache int _ncompressed = 0; RamCacheCLFUSEntry *_compressed = nullptr; // first uncompressed lru[0] entry + // Lets the unit tests reach a stored entry so the decompression failure + // paths in get() can be exercised; see unit_tests/test_RamCacheCLFUS.cc. + // Nothing in the product uses this. + friend struct RamCacheCLFUSTestAccess; + void _resize_hashtable(); void _victimize(RamCacheCLFUSEntry *e); void _move_compressed(RamCacheCLFUSEntry *e); diff --git a/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc b/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc new file mode 100644 index 00000000000..46d1026151c --- /dev/null +++ b/src/iocore/cache/unit_tests/test_RamCacheCLFUS.cc @@ -0,0 +1,377 @@ +/** @file + + Catch-based unit tests for RAM cache (CLFUS) compression roundtrips. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "main.h" + +#include "../RamCacheCLFUS.h" +#include "../P_CacheInternal.h" + +#include "iocore/cache/Cache.h" +#include "tscore/ink_config.h" + +#include +#include +#include +#include +#include + +// Required by main.h +int cache_vols = 1; +bool reuse_existing_cache = false; + +// Reaches into RamCacheCLFUS to find a stored entry; declared a friend there. +struct RamCacheCLFUSTestAccess { + static RamCacheCLFUSEntry * + find_entry(RamCacheCLFUS &rc, const CryptoHash &key) + { + for (RamCacheCLFUSEntry *e = rc._bucket[key.slice32(3) % rc._nbuckets].head; e != nullptr; e = e->hash_link.next) { + if (e->key == key) { + return e; + } + } + return nullptr; + } +}; + +namespace +{ + +// A compression backend to exercise, along with the RAM_HIT_* state get() +// should report once a compressible object has been stored compressed. +struct CompressionCase { + int config; // CACHE_COMPRESSION_* + int expected_hit; // RAM_HIT_COMPRESS_* reported by get() for compressible data + const char *name; +}; + +std::vector +compression_cases() +{ + std::vector cases{ + {CACHE_COMPRESSION_NONE, RAM_HIT_COMPRESS_NONE, "none" }, + {CACHE_COMPRESSION_FASTLZ, RAM_HIT_COMPRESS_FASTLZ, "fastlz"}, + {CACHE_COMPRESSION_LIBZ, RAM_HIT_COMPRESS_LIBZ, "libz" }, + }; +#ifdef HAVE_LZMA_H + cases.push_back({CACHE_COMPRESSION_LIBLZMA, RAM_HIT_COMPRESS_LIBLZMA, "liblzma"}); +#endif +#ifdef HAVE_LZ4_H + cases.push_back({CACHE_COMPRESSION_LZ4, RAM_HIT_COMPRESS_LZ4, "lz4"}); +#endif +#ifdef HAVE_ZSTD_H + cases.push_back({CACHE_COMPRESSION_ZSTD, RAM_HIT_COMPRESS_ZSTD, "zstd"}); +#endif + return cases; +} + +// Minimal CacheDisk wiring needed to construct a StripeSM. Mirrors the helper +// in test_Stripe.cc. +void +init_disk(CacheDisk &disk) +{ + disk.path = static_cast(ats_malloc(1)); + disk.path[0] = '\0'; + disk.disk_stripes = static_cast(ats_malloc(sizeof(DiskStripe *))); + disk.disk_stripes[0] = nullptr; + disk.header = static_cast(ats_malloc(sizeof(DiskHeader))); + disk.header->num_volumes = 0; +} + +// The CLFUS get/put/compress paths touch only these metrics and the stripe +// mutex, so that is all the stripe needs for these tests. +void +wire_stripe(StripeSM &stripe, CacheVol &cache_vol) +{ + stripe.cache_vol = &cache_vol; + + cache_rsb.ram_cache_bytes = ts::Metrics::Gauge::createPtr("unit_test.clfus.ram_cache.bytes"); + cache_rsb.ram_cache_hits = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.hits"); + cache_rsb.ram_cache_misses = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.misses"); + cache_rsb.ram_cache_compress_failures = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.compress.failure"); + cache_rsb.ram_cache_decompress_failures = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.decompress.failure"); + cache_vol.vol_rsb.ram_cache_bytes = ts::Metrics::Gauge::createPtr("unit_test.clfus.vol.ram_cache.bytes"); + cache_vol.vol_rsb.ram_cache_hits = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.hits"); + cache_vol.vol_rsb.ram_cache_misses = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.misses"); + cache_vol.vol_rsb.ram_cache_compress_failures = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.compress.failure"); + cache_vol.vol_rsb.ram_cache_decompress_failures = + ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.decompress.failure"); +} + +Ptr +make_buffer(const std::vector &bytes) +{ + int64_t idx = iobuffer_size_to_index(bytes.size(), MAX_BUFFER_SIZE_INDEX); + Ptr data{make_ptr(new_IOBufferData(idx, MEMALIGNED))}; + std::memcpy(data->data(), bytes.data(), bytes.size()); + return data; +} + +// Highly compressible: a short repeating pattern. +std::vector +compressible_bytes(std::size_t len) +{ + std::vector bytes(len); + for (std::size_t i = 0; i < len; i++) { + bytes[i] = static_cast('A' + (i % 26)); + } + return bytes; +} + +// Effectively incompressible: a deterministic xorshift byte stream. +std::vector +incompressible_bytes(std::size_t len) +{ + std::vector bytes(len); + uint32_t state = 0x9e3779b9; + for (std::size_t i = 0; i < len; i++) { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + bytes[i] = static_cast(state & 0xff); + } + return bytes; +} + +// Comparing two 256 KB vectors with CHECK() makes Catch2 stringify both +// operands, which throws before it can report anything useful. Report the +// offset of the first difference instead, so a genuine round-trip failure +// names the byte. +std::size_t +first_difference(const std::vector &lhs, const std::vector &rhs) +{ + std::size_t common = std::min(lhs.size(), rhs.size()); + for (std::size_t i = 0; i < common; i++) { + if (lhs[i] != rhs[i]) { + return i; + } + } + return common; +} + +struct RoundtripResult { + int hit = 0; + int64_t size_before = 0; // rc.size() after put, before the compression pass + int64_t size_after = 0; // rc.size() after the compression pass + std::vector out; +}; + +// Store payload under a fresh key, force a synchronous compression pass with +// `config`, then read it back. +RoundtripResult +store_compress_get(StripeSM &stripe, int config, const std::vector &payload) +{ + // Initialize with compression disabled so init() does not schedule the + // background compressor (which would retain a pointer to this stack object). + cache_config_ram_cache_compress = CACHE_COMPRESSION_NONE; + cache_config_ram_cache_compress_percent = 100; + cache_config_ram_cache_use_seen_filter = 0; + + RamCacheCLFUS rc; + rc.init(1 << 20, &stripe); + + Ptr in = make_buffer(payload); + uint32_t len = static_cast(payload.size()); + + // A fresh RamCacheCLFUS per call, so one fixed key cannot collide with + // anything. + CryptoHash key; + key.u64[0] = 0xc0ffee00; + key.u64[1] = 0xdeadbeef; + + REQUIRE(rc.put(&key, in.get(), len) == 1); + + RoundtripResult r; + + r.size_before = rc.size(); + cache_config_ram_cache_compress = config; + rc.compress_entries(this_ethread()); + r.size_after = rc.size(); + + Ptr ret; + + r.hit = rc.get(&key, &ret); + REQUIRE(ret.get() != nullptr); + r.out.assign(ret->data(), ret->data() + len); + return r; +} + +} // namespace + +TEST_CASE("CLFUS compressible objects roundtrip cleanly", "[cache][ramcache][compress]") +{ + CacheDisk disk; + init_disk(disk); + StripeSM stripe{&disk, 10, 0}; + CacheVol cache_vol; + wire_stripe(stripe, cache_vol); + + // Large enough to exercise the *_compressBound() arithmetic and uint32_t + // casts in compress_entries(), not just small-buffer paths. + auto payload = compressible_bytes(256 * 1024); + const CompressionCase c = GENERATE(from_range(compression_cases())); + INFO("compression backend: " << c.name); + + RoundtripResult r = store_compress_get(stripe, c.config, payload); + + CHECK(r.hit == c.expected_hit); + CHECK(r.out.size() == payload.size()); + CHECK(first_difference(r.out, payload) == payload.size()); + if (c.config != CACHE_COMPRESSION_NONE) { + // The feature's contract is that compression saves memory, not merely + // that the entry is tagged compressed. + CHECK(r.size_after < r.size_before); + } +} + +TEST_CASE("CLFUS incompressible objects fall back to uncompressed storage", "[cache][ramcache][compress]") +{ + CacheDisk disk; + init_disk(disk); + StripeSM stripe{&disk, 10, 0}; + CacheVol cache_vol; + wire_stripe(stripe, cache_vol); + + auto payload = incompressible_bytes(256 * 1024); + + // Only the backends that actually attempt compression are interesting here. + std::vector cases; + for (auto const &candidate : compression_cases()) { + if (candidate.config != CACHE_COMPRESSION_NONE) { + cases.push_back(candidate); + } + } + const CompressionCase c = GENERATE_REF(from_range(cases)); + INFO("compression backend: " << c.name); + + RoundtripResult r = store_compress_get(stripe, c.config, payload); + + // Incompressible data is kept verbatim, so a read reports no compression. + CHECK(r.hit == RAM_HIT_COMPRESS_NONE); + CHECK(r.out.size() == payload.size()); + CHECK(first_difference(r.out, payload) == payload.size()); + // And the pass never grows the entry: a regression that stored the expanded + // "compressed" blob would still read back correctly but would cost memory. + // Not an equality check, because re-storing an incompressible entry tightly + // legitimately shrinks a payload that carried buffer padding. + CHECK(r.size_after <= r.size_before); +} + +TEST_CASE("CLFUS single-byte payload roundtrips", "[cache][ramcache][compress]") +{ + CacheDisk disk; + init_disk(disk); + StripeSM stripe{&disk, 10, 0}; + CacheVol cache_vol; + wire_stripe(stripe, cache_vol); + + auto payload = compressible_bytes(1); + const CompressionCase c = GENERATE(from_range(compression_cases())); + INFO("compression backend: " << c.name); + + RoundtripResult r = store_compress_get(stripe, c.config, payload); + + // Every codec emits a frame larger than a single byte, so a one-byte object + // can never shrink; whichever way a backend declines it -- an explicit + // too-small guard, the incompressible marking, or storing the bytes verbatim + // -- the object must survive and read back uncompressed. + CHECK(r.hit == RAM_HIT_COMPRESS_NONE); + CHECK(r.out.size() == payload.size()); + CHECK(first_difference(r.out, payload) == payload.size()); +} + +TEST_CASE("CLFUS reports a corrupted compressed entry rather than serving it", "[cache][ramcache][compress]") +{ + CacheDisk disk; + init_disk(disk); + StripeSM stripe{&disk, 10, 0}; + CacheVol cache_vol; + wire_stripe(stripe, cache_vol); + + // Only backends that actually store a compressed blob can have one corrupted. + std::vector cases; + for (auto const &candidate : compression_cases()) { + if (candidate.config != CACHE_COMPRESSION_NONE) { + cases.push_back(candidate); + } + } + const CompressionCase c = GENERATE_REF(from_range(cases)); + INFO("compression backend: " << c.name); + + cache_config_ram_cache_compress = CACHE_COMPRESSION_NONE; + cache_config_ram_cache_compress_percent = 100; + cache_config_ram_cache_use_seen_filter = 0; + + RamCacheCLFUS rc; + rc.init(1 << 20, &stripe); + + auto payload = compressible_bytes(256 * 1024); + uint32_t len = static_cast(payload.size()); + Ptr in = make_buffer(payload); + + CryptoHash key; + key.u64[0] = 0xc0ffee00; + key.u64[1] = 0xdeadbeef; + + REQUIRE(rc.put(&key, in.get(), len) == 1); + + cache_config_ram_cache_compress = c.config; + rc.compress_entries(this_ethread()); + + RamCacheCLFUSEntry *e = RamCacheCLFUSTestAccess::find_entry(rc, key); + REQUIRE(e != nullptr); + // The pass must actually have compressed it, or there is nothing to corrupt. + REQUIRE(e->flag_bits.compressed != 0); + REQUIRE(e->compressed_len > 0); + + // Overwrite the whole stored blob. Every codec here rejects this either + // outright or by producing the wrong length, which is what get() checks. + std::memset(e->data->data(), 0xff, e->compressed_len); + + int64_t before = ts::Metrics::Counter::load(cache_rsb.ram_cache_decompress_failures); + + Ptr ret; + int hit = rc.get(&key, &ret); + + // A corrupted entry is a miss, is counted, and is dropped rather than + // handed to the caller. + CHECK(hit == 0); + CHECK(ts::Metrics::Counter::load(cache_rsb.ram_cache_decompress_failures) == before + 1); + CHECK(RamCacheCLFUSTestAccess::find_entry(rc, key) == nullptr); +} + +// A backend that is not compiled in silently disappears from the parametrized +// cases above; make that visible in the test output rather than shipping an +// untested backend behind a green run. +TEST_CASE("CLFUS compression backends compiled in", "[cache][ramcache][compress]") +{ +#ifndef HAVE_LZMA_H + WARN("liblzma is not compiled in; the liblzma RAM cache compression backend is NOT tested"); +#endif +#ifndef HAVE_LZ4_H + WARN("lz4 is not compiled in; the lz4 RAM cache compression backend is NOT tested"); +#endif +#ifndef HAVE_ZSTD_H + WARN("zstd is not compiled in; the zstd RAM cache compression backend is NOT tested"); +#endif + CHECK(compression_cases().size() >= 3); +} diff --git a/src/iocore/cache/unit_tests/test_RamCacheCompressEntries.cc b/src/iocore/cache/unit_tests/test_RamCacheCompressEntries.cc index 9644f5acf0b..18973a7e8ed 100644 --- a/src/iocore/cache/unit_tests/test_RamCacheCompressEntries.cc +++ b/src/iocore/cache/unit_tests/test_RamCacheCompressEntries.cc @@ -58,12 +58,17 @@ wire_stripe(StripeSM &stripe, CacheVol &cache_vol) { stripe.cache_vol = &cache_vol; - cache_rsb.ram_cache_bytes = ts::Metrics::Gauge::createPtr("unit_test.clfus.ram_cache.bytes"); - cache_rsb.ram_cache_hits = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.hits"); - cache_rsb.ram_cache_misses = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.misses"); - cache_vol.vol_rsb.ram_cache_bytes = ts::Metrics::Gauge::createPtr("unit_test.clfus.vol.ram_cache.bytes"); - cache_vol.vol_rsb.ram_cache_hits = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.hits"); - cache_vol.vol_rsb.ram_cache_misses = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.misses"); + cache_rsb.ram_cache_bytes = ts::Metrics::Gauge::createPtr("unit_test.clfus.ram_cache.bytes"); + cache_rsb.ram_cache_hits = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.hits"); + cache_rsb.ram_cache_misses = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.misses"); + cache_rsb.ram_cache_compress_failures = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.compress.failure"); + cache_rsb.ram_cache_decompress_failures = ts::Metrics::Counter::createPtr("unit_test.clfus.ram_cache.decompress.failure"); + cache_vol.vol_rsb.ram_cache_bytes = ts::Metrics::Gauge::createPtr("unit_test.clfus.vol.ram_cache.bytes"); + cache_vol.vol_rsb.ram_cache_hits = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.hits"); + cache_vol.vol_rsb.ram_cache_misses = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.misses"); + cache_vol.vol_rsb.ram_cache_compress_failures = ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.compress.failure"); + cache_vol.vol_rsb.ram_cache_decompress_failures = + ts::Metrics::Counter::createPtr("unit_test.clfus.vol.ram_cache.decompress.failure"); } RamCacheCLFUS * @@ -72,8 +77,9 @@ make_cache(StripeSM &stripe) // Initialize with compression disabled so init() does not schedule the // background compressor continuation, which would retain a pointer to the // cache; compression is driven synchronously by the tests instead. The - // caches are kept reachable for the life of the process because the policy - // has no destructor (entries are pool-allocated). + // caches are intentionally leaked rather than destroyed: ~RamCacheCLFUS() + // is only safe for a cache that never scheduled that continuation, and + // leaking keeps these tests independent of that constraint. cache_config_ram_cache_compress = CACHE_COMPRESSION_NONE; cache_config_ram_cache_compress_percent = 100; cache_config_ram_cache_use_seen_filter = 0; diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc index e429ac4680b..35c0d119447 100644 --- a/src/records/RecordsConfig.cc +++ b/src/records/RecordsConfig.cc @@ -886,7 +886,7 @@ static constexpr RecordElement RecordsConfig[] = , {RECT_CONFIG, "proxy.config.cache.ram_cache.use_seen_filter", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-9]", RECA_NULL} , - {RECT_CONFIG, "proxy.config.cache.ram_cache.compress", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-3]", RECA_NULL} + {RECT_CONFIG, "proxy.config.cache.ram_cache.compress", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_INT, "[0-5]", RECA_NULL} , {RECT_CONFIG, "proxy.config.cache.ram_cache.compress_percent", RECD_INT, "90", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} , diff --git a/src/records/unit_tests/test_RecUtils.cc b/src/records/unit_tests/test_RecUtils.cc index 68f772ad7bd..6b723df558c 100644 --- a/src/records/unit_tests/test_RecUtils.cc +++ b/src/records/unit_tests/test_RecUtils.cc @@ -26,6 +26,8 @@ #include "../P_RecUtils.h" #include "records/RecordsConfig.h" +#include + TEST_CASE("recordRangeCheck via RecordValidityCheck", "[librecords][RecUtils]") { SECTION("valid ranges") @@ -213,3 +215,24 @@ TEST_CASE("search_default_domains accepts documented values", "[librecords][RecU REQUIRE(RecordValidityCheck("2", record->check, record->regex)); REQUIRE_FALSE(RecordValidityCheck("3", record->check, record->regex)); } + +TEST_CASE("ram_cache.compress accepts every compression backend", "[librecords][RecUtils]") +{ + const auto *record = GetRecordElementByName("proxy.config.cache.ram_cache.compress"); + + REQUIRE(record != nullptr); + REQUIRE(record->check == RECC_INT); + REQUIRE(record->regex != nullptr); + + // The validity range must cover every CACHE_COMPRESSION_* value, otherwise a + // documented backend is rejected at load time and silently falls back to the + // default of 0 (no compression). Kept as a literal rather than including + // iocore/cache/Cache.h, which would point a records test at the cache layer; + // extend it when a codec is added there. + constexpr int largest_compression_type = 5; // CACHE_COMPRESSION_ZSTD + for (int i = 0; i <= largest_compression_type; i++) { + INFO("CACHE_COMPRESSION_* value: " << i); + REQUIRE(RecordValidityCheck(std::to_string(i).c_str(), record->check, record->regex)); + } + REQUIRE_FALSE(RecordValidityCheck(std::to_string(largest_compression_type + 1).c_str(), record->check, record->regex)); +} diff --git a/src/traffic_layout/CMakeLists.txt b/src/traffic_layout/CMakeLists.txt index b86d4f2573a..ddc9c2674d8 100644 --- a/src/traffic_layout/CMakeLists.txt +++ b/src/traffic_layout/CMakeLists.txt @@ -35,6 +35,10 @@ if(HAVE_ZSTD_H) target_link_libraries(traffic_layout PRIVATE zstd::zstd) endif() +if(HAVE_LZ4_H) + target_link_libraries(traffic_layout PRIVATE LZ4::LZ4) +endif() + install(TARGETS traffic_layout) clang_tidy_check(traffic_layout) diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc index 91b0677e042..d15fb3011b5 100644 --- a/src/traffic_layout/info.cc +++ b/src/traffic_layout/info.cc @@ -54,6 +54,10 @@ #include #endif +#if HAVE_LZ4_H +#include +#endif + #if HAVE_SSL_CTX_ADD_CERT_COMPRESSION_ALG static constexpr int ts_has_cert_compression_callbacks = 1; #else @@ -133,6 +137,11 @@ produce_features(bool json) print_feature("TS_HAS_ZSTD", 1, json); #else print_feature("TS_HAS_ZSTD", 0, json); +#endif +#ifdef HAVE_LZ4_H + print_feature("TS_HAS_LZ4", 1, json); +#else + print_feature("TS_HAS_LZ4", 0, json); #endif print_feature("TS_HAS_CERT_COMPRESSION", ts_has_cert_compression, json); print_feature("TS_HAS_CERT_COMPRESSION_CALLBACKS", ts_has_cert_compression_callbacks, json); @@ -259,6 +268,12 @@ produce_versions(bool json) #else print_var("zstd", undef, json); #endif +#ifdef HAVE_LZ4_H + // Runtime version, matching what the zstd line above reports. + print_var("lz4", LBW().print("{}", LZ4_versionString()).view(), json); +#else + print_var("lz4", undef, json); +#endif // This should always be last print_var("traffic-server", LBW().print(TS_VERSION_STRING).view(), json, true); diff --git a/tools/package/trafficserver.spec b/tools/package/trafficserver.spec index a20951fa72e..dbd2b8502fc 100755 --- a/tools/package/trafficserver.spec +++ b/tools/package/trafficserver.spec @@ -35,6 +35,7 @@ URL: https://trafficserver.apache.org/ Source0: http://www.apache.org/dist/%{name}/%{name}-%{version}.tar.bz2 BuildRequires: expat-devel hwloc-devel openssl-devel pcre-devel zlib-devel xz-devel +BuildRequires: libzstd-devel lz4-devel BuildRequires: libcurl-devel ncurses-devel BuildRequires: gcc gcc-c++ perl-ExtUtils-MakeMaker BuildRequires: libcap-devel