From 755edef29be309bdebcee708dbfb11485dd09721 Mon Sep 17 00:00:00 2001 From: Dmitry Werner Date: Tue, 15 Sep 2026 18:01:54 +0500 Subject: [PATCH] IGNITE-29055 Add a metric for detected blocked system threads to WorkersRegistry --- .../_docs/monitoring-metrics/new-metrics.adoc | 1 + .../processors/metric/GridMetricManager.java | 7 +++++ .../internal/worker/WorkersRegistry.java | 23 +++++++++++++++ .../failure/SystemWorkersBlockingTest.java | 28 +++++++++++++++++++ 4 files changed, 59 insertions(+) diff --git a/docs/_docs/monitoring-metrics/new-metrics.adoc b/docs/_docs/monitoring-metrics/new-metrics.adoc index b07cb629b4c26..6ad2a1e06c8e4 100644 --- a/docs/_docs/monitoring-metrics/new-metrics.adoc +++ b/docs/_docs/monitoring-metrics/new-metrics.adoc @@ -37,6 +37,7 @@ Register name: `sys` |ThreadCount |integer| ThreadMXBean.getThreadCount |TotalExecutedTasks |long| Total executed tasks. |TotalStartedThreadCount |long| ThreadMXBean.getTotalStartedThreadCount +|BlockedSystemThreadsCount |long| Total number of blocked system-critical threads. |UpTime| long | RuntimeMxBean.getUptime() |memory.heap.committed| long| MemoryUsage.getHeapMemoryUsage().getCommitted() |memory.heap.init | long| MemoryUsage.getHeapMemoryUsage().getInit() diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/GridMetricManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/GridMetricManager.java index b7d5d0fd54771..9dd8c7cf104ad 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/GridMetricManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/metric/GridMetricManager.java @@ -54,6 +54,7 @@ import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.worker.WorkersRegistry; import org.apache.ignite.metric.IgniteMetrics; import org.apache.ignite.metric.MetricRegistry; import org.apache.ignite.spi.metric.HistogramMetric; @@ -244,6 +245,12 @@ public GridMetricManager(GridKernalContext ctx) { sysreg.register("CurrentThreadCpuTime", threads::getCurrentThreadCpuTime, null); sysreg.register("CurrentThreadUserTime", threads::getCurrentThreadUserTime, null); + WorkersRegistry workersRegistry = ctx.workersRegistry(); + + // Workers registry can be unavailable in standalone (WAL reader) and test contexts. + if (workersRegistry != null) + workersRegistry.registerMetrics(sysreg); + MetricRegistryImpl pmeReg = registry(PME_METRICS); long[] pmeBounds = new long[] {500, 1000, 5000, 30000}; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java index 7af557dc45433..ffe775d24389e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java @@ -26,6 +26,8 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.ignite.IgniteLogger; import org.apache.ignite.failure.FailureType; +import org.apache.ignite.internal.processors.metric.MetricRegistryImpl; +import org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.util.worker.GridWorker; import org.apache.ignite.internal.util.worker.GridWorkerListener; @@ -43,6 +45,12 @@ public class WorkersRegistry implements GridWorkerListener { /** */ private static final long DFLT_CHECK_INTERVAL = 3_000; + /** Blocked system-critical threads count metric name. */ + public static final String BLOCKED_SYSTEM_WORKERS_CNT = "BlockedSystemThreadsCount"; + + /** Blocked system-critical threads count metric description. */ + private static final String BLOCKED_SYSTEM_WORKERS_CNT_DESC = "Total number of blocked system-critical threads."; + /** Registered workers. */ private final ConcurrentMap registeredWorkers = new ConcurrentHashMap<>(); @@ -72,6 +80,9 @@ public class WorkersRegistry implements GridWorkerListener { /** Logger. */ private final IgniteLogger log; + /** Blocked system-critical threads count metric. */ + private volatile AtomicLongMetric blockedWorkersCntMetric; + /** * @param workerFailedHnd Closure to invoke on worker failure. * @param sysWorkerBlockedTimeout Maximum allowed worker heartbeat interval in milliseconds, non-positive value denotes @@ -158,6 +169,15 @@ public void setSystemWorkerBlockedTimeout(long val) { sysWorkerBlockedTimeout = U.ensurePositive(val, Long.MAX_VALUE); } + /** + * Registers workers registry metrics. + * + * @param mreg Metric registry to register metrics in. + */ + public void registerMetrics(MetricRegistryImpl mreg) { + blockedWorkersCntMetric = mreg.longMetric(BLOCKED_SYSTEM_WORKERS_CNT, BLOCKED_SYSTEM_WORKERS_CNT_DESC); + } + /** {@inheritDoc} */ @Override public void onStarted(GridWorker w) { register(w); @@ -229,6 +249,9 @@ public void setSystemWorkerBlockedTimeout(long val) { "[workerName=" + worker.name() + ", threadName=" + runner.getName() + ", blockedFor=" + heartbeatDelay / 1000 + "s]"); + if (blockedWorkersCntMetric != null) + blockedWorkersCntMetric.increment(); + workerFailedHnd.apply(worker, SYSTEM_WORKER_BLOCKED); } diff --git a/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java b/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java index 57495daec0a7a..7794af4372e06 100644 --- a/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java @@ -30,14 +30,18 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.processors.metric.GridMetricManager; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.util.worker.GridWorker; import org.apache.ignite.internal.worker.WorkersRegistry; +import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.GridAbstractTest; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; +import static org.apache.ignite.internal.worker.WorkersRegistry.BLOCKED_SYSTEM_WORKERS_CNT; + /** * Tests the handling of long blocking operations in system-critical workers. */ @@ -127,6 +131,30 @@ public void testBlockingWorker() throws Exception { e -> LatchingGridWorker.class.getName().equals(e.getClassName()))); } + /** + * Tests that the {@code BlockedSystemThreadsCount} metric is registered in the system metric registry + * and is incremented when a system-critical thread is detected as blocked. + * + * @throws Exception If failed. + */ + @Test + public void testBlockedSystemThreadsCountMetric() throws Exception { + IgniteEx ignite = startGrid(0); + + GridWorker worker = new LatchingGridWorker(ignite); + + runWorker(worker); + + ignite.context().workersRegistry().register(worker); + + assertTrue(hndLatch.await(ignite.configuration().getFailureDetectionTimeout() * 2, TimeUnit.MILLISECONDS)); + + LongMetric metric = ignite.context().metric().registry(GridMetricManager.SYS_METRICS).findMetric(BLOCKED_SYSTEM_WORKERS_CNT); + + assertNotNull("Metric is not registered", metric); + assertTrue("Metric is not incremented", metric.value() > 0); + } + /** * @throws Exception If failed. */