diff --git a/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/core/GestureHandlerOrchestrator.kt b/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/core/GestureHandlerOrchestrator.kt
index 71ace5fd8b..af1f97c856 100644
--- a/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/core/GestureHandlerOrchestrator.kt
+++ b/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/core/GestureHandlerOrchestrator.kt
@@ -56,6 +56,7 @@ class GestureHandlerOrchestrator(
     private set
   private var handlingChangeSemaphore = 0
   private var finishedHandlersCleanupScheduled = false
+  private var interceptedViewsCancelScheduled = false
   private var activationIndex = 0
 
   /**
@@ -74,6 +75,11 @@ class GestureHandlerOrchestrator(
       cancelAll()
     }
     deliverEventToGestureHandlers(event)
+    if (interceptedViewsCancelScheduled) {
+      interceptedViewsCancelScheduled = false
+      cancelTouchesInInterceptedViews(event)
+    }
+    releasePointerDownPoints(event)
     isHandlingTouch = false
     if (finishedHandlersCleanupScheduled && handlingChangeSemaphore == 0) {
       cleanupFinishedHandlers()
@@ -759,18 +765,26 @@ class GestureHandlerOrchestrator(
   }
 
   private fun trackPointerDownPoints(event: MotionEvent) {
-    val index = event.actionIndex
+    val action = event.actionMasked
+    if (action == MotionEvent.ACTION_DOWN || action == MotionEvent.ACTION_POINTER_DOWN) {
+      val index = event.actionIndex
+      pointerDownPoints.put(event.getPointerId(index), PointF(event.getX(index), event.getY(index)))
+    }
+  }
+
+  // Runs after the event is handled, so a sweep started by a lifting pointer still sees it.
+  private fun releasePointerDownPoints(event: MotionEvent) {
     when (event.actionMasked) {
-      MotionEvent.ACTION_DOWN, MotionEvent.ACTION_POINTER_DOWN ->
-        pointerDownPoints.put(event.getPointerId(index), PointF(event.getX(index), event.getY(index)))
-      MotionEvent.ACTION_POINTER_UP ->
-        pointerDownPoints.remove(event.getPointerId(index))
-      MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL ->
-        pointerDownPoints.clear()
+      MotionEvent.ACTION_POINTER_UP -> pointerDownPoints.remove(event.getPointerId(event.actionIndex))
+      MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> pointerDownPoints.clear()
     }
   }
 
-  fun cancelTouchesInInterceptedViews(event: MotionEvent) {
+  fun scheduleInterceptedViewsCancel() {
+    interceptedViewsCancelScheduled = true
+  }
+
+  private fun cancelTouchesInInterceptedViews(event: MotionEvent) {
     viewsToCancel.clear()
     for (i in 0 until pointerDownPoints.size()) {
       val point = pointerDownPoints.valueAt(i)
diff --git a/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/react/RNGestureHandlerRootHelper.kt b/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/react/RNGestureHandlerRootHelper.kt
index baba360c10..0467c2ef47 100644
--- a/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/react/RNGestureHandlerRootHelper.kt
+++ b/packages/react-native-gesture-handler/android/src/main/java/com/swmansion/gesturehandler/react/RNGestureHandlerRootHelper.kt
@@ -20,7 +20,6 @@ class RNGestureHandlerRootHelper(private val context: ReactContext, wrappedView:
   private val jsGestureHandler: GestureHandler?
   val rootView: ViewGroup
   private var shouldIntercept = false
-  private var wasIntercepting = false
   private var passingTouch = false
   private var passingNativeTouch = false
   private var nativeTouchGrabRequested = false
@@ -115,6 +114,11 @@ class RNGestureHandlerRootHelper(private val context: ReactContext, wrappedView:
     override fun onHandleHover(event: MotionEvent, sourceEvent: MotionEvent) = handleEvent(event)
 
     override fun onCancel() {
+      // On the transition into interception, cancel the native views the pointers landed on - the
+      // framework's ACTION_CANCEL never reaches them since RNGH ignores `onInterceptTouchEvent`.
+      if (!shouldIntercept) {
+        orchestrator?.scheduleInterceptedViewsCancel()
+      }
       shouldIntercept = true
     }
   }
@@ -185,13 +189,6 @@ class RNGestureHandlerRootHelper(private val context: ReactContext, wrappedView:
     orchestrator!!.onTouchEvent(event)
     passingTouch = false
 
-    // On the transition into interception, cancel the native views the pointers landed on - the
-    // framework's ACTION_CANCEL never reaches them since RNGH ignores `onInterceptTouchEvent`
-    if (shouldIntercept && !wasIntercepting) {
-      orchestrator!!.cancelTouchesInInterceptedViews(event)
-    }
-    wasIntercepting = shouldIntercept
-
     return shouldIntercept
   }