IGNITE-29055 Add a metric for detected blocked system threads to WorkersRegistry

docs/_docs/monitoring-metrics/new-metrics.adoc

@@ -37,6 +37,7 @@ Register name: `sys`
 |ThreadCount |integer| ThreadMXBean.getThreadCount
 |TotalExecutedTasks  |long|    Total executed tasks.
 |TotalStartedThreadCount |long|    ThreadMXBean.getTotalStartedThreadCount
+|BlockedSystemThreadsCount |long| Total number of blocked system-critical threads.
 |UpTime|  long  |  RuntimeMxBean.getUptime()
 |memory.heap.committed|   long|    MemoryUsage.getHeapMemoryUsage().getCommitted()
 |memory.heap.init |   long|    MemoryUsage.getHeapMemoryUsage().getInit()

modules/core/src/main/java/org/apache/ignite/internal/processors/metric/GridMetricManager.java

@@ -54,6 +54,7 @@
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.worker.WorkersRegistry;
 import org.apache.ignite.metric.IgniteMetrics;
 import org.apache.ignite.metric.MetricRegistry;
 import org.apache.ignite.spi.metric.HistogramMetric;
@@ -244,6 +245,12 @@ public GridMetricManager(GridKernalContext ctx) {
         sysreg.register("CurrentThreadCpuTime", threads::getCurrentThreadCpuTime, null);
         sysreg.register("CurrentThreadUserTime", threads::getCurrentThreadUserTime, null);
 
+        WorkersRegistry workersRegistry = ctx.workersRegistry();
+
+        // Workers registry can be unavailable in standalone (WAL reader) and test contexts.
+        if (workersRegistry != null)
+            workersRegistry.registerMetrics(sysreg);
+
         MetricRegistryImpl pmeReg = registry(PME_METRICS);
 
         long[] pmeBounds = new long[] {500, 1000, 5000, 30000};

modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java

@@ -26,6 +26,8 @@
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.failure.FailureType;
+import org.apache.ignite.internal.processors.metric.MetricRegistryImpl;
+import org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.internal.util.worker.GridWorkerListener;
@@ -43,6 +45,12 @@ public class WorkersRegistry implements GridWorkerListener {
     /** */
     private static final long DFLT_CHECK_INTERVAL = 3_000;
 
+    /** Blocked system-critical threads count metric name. */
+    public static final String BLOCKED_SYSTEM_WORKERS_CNT = "BlockedSystemThreadsCount";
+
+    /** Blocked system-critical threads count metric description. */
+    private static final String BLOCKED_SYSTEM_WORKERS_CNT_DESC = "Total number of blocked system-critical threads.";
+
     /** Registered workers. */
     private final ConcurrentMap<String, GridWorker> registeredWorkers = new ConcurrentHashMap<>();
 
@@ -72,6 +80,9 @@ public class WorkersRegistry implements GridWorkerListener {
     /** Logger. */
     private final IgniteLogger log;
 
+    /** Blocked system-critical threads count metric. */
+    private volatile AtomicLongMetric blockedWorkersCntMetric;
+
     /**
      * @param workerFailedHnd Closure to invoke on worker failure.
      * @param sysWorkerBlockedTimeout Maximum allowed worker heartbeat interval in milliseconds, non-positive value denotes
@@ -158,6 +169,15 @@ public void setSystemWorkerBlockedTimeout(long val) {
         sysWorkerBlockedTimeout = U.ensurePositive(val, Long.MAX_VALUE);
     }
 
+    /**
+     * Registers workers registry metrics.
+     *
+     * @param mreg Metric registry to register metrics in.
+     */
+    public void registerMetrics(MetricRegistryImpl mreg) {
+        blockedWorkersCntMetric = mreg.longMetric(BLOCKED_SYSTEM_WORKERS_CNT, BLOCKED_SYSTEM_WORKERS_CNT_DESC);
+    }
+
     /** {@inheritDoc} */
     @Override public void onStarted(GridWorker w) {
         register(w);
@@ -229,6 +249,9 @@ public void setSystemWorkerBlockedTimeout(long val) {
                                     "[workerName=" + worker.name() + ", threadName=" + runner.getName() +
                                     ", blockedFor=" + heartbeatDelay / 1000 + "s]");
 
+                            if (blockedWorkersCntMetric != null)
+                                blockedWorkersCntMetric.increment();
+
                             workerFailedHnd.apply(worker, SYSTEM_WORKER_BLOCKED);
                         }
 

modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersBlockingTest.java

@@ -30,14 +30,18 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.processors.metric.GridMetricManager;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.internal.worker.WorkersRegistry;
+import org.apache.ignite.spi.metric.LongMetric;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.GridAbstractTest;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.junit.Test;
 
+import static org.apache.ignite.internal.worker.WorkersRegistry.BLOCKED_SYSTEM_WORKERS_CNT;
+
 /**
  * Tests the handling of long blocking operations in system-critical workers.
  */
@@ -127,6 +131,30 @@ public void testBlockingWorker() throws Exception {
             e -> LatchingGridWorker.class.getName().equals(e.getClassName())));
     }
 
+    /**
+     * Tests that the {@code BlockedSystemThreadsCount} metric is registered in the system metric registry
+     * and is incremented when a system-critical thread is detected as blocked.
+     *
+     * @throws Exception If failed.
+     */
+    @Test
+    public void testBlockedSystemThreadsCountMetric() throws Exception {
+        IgniteEx ignite = startGrid(0);
+
+        GridWorker worker = new LatchingGridWorker(ignite);
+
+        runWorker(worker);
+
+        ignite.context().workersRegistry().register(worker);
+
+        assertTrue(hndLatch.await(ignite.configuration().getFailureDetectionTimeout() * 2, TimeUnit.MILLISECONDS));
+
+        LongMetric metric = ignite.context().metric().registry(GridMetricManager.SYS_METRICS).findMetric(BLOCKED_SYSTEM_WORKERS_CNT);
+
+        assertNotNull("Metric is not registered", metric);
+        assertTrue("Metric is not incremented", metric.value() > 0);
+    }
+
     /**
      * @throws Exception If failed.
      */