diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h
index 1d857a0ed5..9665ebf76d 100644
--- a/include/pybind11/cast.h
+++ b/include/pybind11/cast.h
@@ -259,22 +259,26 @@ struct type_caster<T, enable_if_t<std::is_arithmetic<T>::value && !is_std_char_t
             return false;
         } else {
             handle src_or_index = src;
-            // PyPy: 7.3.7's 3.8 does not implement PyLong_*'s __index__ calls.
-#if defined(PYPY_VERSION)
             object index;
-            // If not a PyLong, we need to call PyNumber_Index explicitly on PyPy.
+            // If not a PyLong, call PyNumber_Index explicitly where the PyLong_As* call below
+            // does not: PyLong_AsUnsignedLong[Long] never calls __index__, and PyPy 7.3.7's 3.8
+            // does not implement PyLong_*'s __index__ calls at all.
             // When convert is false, we only reach here if PYBIND11_INDEX_CHECK passed above.
+#if defined(PYPY_VERSION)
             if (!PYBIND11_LONG_CHECK(src.ptr())) {
+#else
+            if (std::is_unsigned<py_type>::value && !PYBIND11_LONG_CHECK(src.ptr())) {
+#endif
                 index = reinterpret_steal<object>(PyNumber_Index(src.ptr()));
                 if (!index) {
                     PyErr_Clear();
-                    if (!convert)
+                    if (!convert) {
                         return false;
+                    }
                 } else {
                     src_or_index = index;
                 }
             }
-#endif
             if (std::is_unsigned<py_type>::value) {
                 py_value = as_unsigned<py_type>(src_or_index.ptr());
             } else { // signed integer:
diff --git a/tests/test_enum.cpp b/tests/test_enum.cpp
index 2a7fa0fda0..186bc922cb 100644
--- a/tests/test_enum.cpp
+++ b/tests/test_enum.cpp
@@ -59,6 +59,12 @@ TEST_SUBMODULE(enums, m) {
         .value("ESecondMode", ClassWithUnscopedEnum::ESecondMode)
         .export_values();
 
+    // test_scalar_comparison_with_index_object
+    enum UnscopedInt32Enum : std::int32_t { I32Three = 3 };
+    py::enum_<UnscopedInt32Enum>(m, "UnscopedInt32Enum").value("Three", I32Three);
+    enum UnscopedUInt32Enum : std::uint32_t { U32Three = 3 };
+    py::enum_<UnscopedUInt32Enum>(m, "UnscopedUInt32Enum").value("Three", U32Three);
+
     // test_enum_to_int
     m.def("test_enum_to_int", [](int) {});
     m.def("test_enum_to_uint", [](uint32_t) {});
diff --git a/tests/test_enum.py b/tests/test_enum.py
index 81170c91dd..451807365c 100644
--- a/tests/test_enum.py
+++ b/tests/test_enum.py
@@ -230,6 +230,26 @@ def test_enum_to_int():
     m.test_enum_to_long_long(m.ScopedBoolEnum.TRUE)
 
 
+@pytest.mark.parametrize("enum_name", ["UnscopedInt32Enum", "UnscopedUInt32Enum"])
+def test_scalar_comparison_with_index_object(enum_name):
+    # An unsigned underlying type must compare like a signed one against objects
+    # that only implement __index__ (e.g. numpy integers).
+    class Index:
+        def __index__(self):
+            return 3
+
+    three = getattr(m, enum_name).Three
+    assert three == 3
+    assert three == Index()
+    assert not three != Index()
+    assert three != 4
+    np = pytest.importorskip("numpy")
+    assert three == np.int32(3)
+    assert three == np.uint64(3)
+    assert not three != np.int64(3)
+    assert three != np.int32(4)
+
+
 def test_duplicate_enum_name():
     with pytest.raises(ValueError) as excinfo:
         m.register_bad_enum()