Android: Allow virtual joystick and touch events at the same time

jme3-android/src/main/java/com/jme3/input/android/AndroidInputHandler.java

@@ -63,6 +63,11 @@ public class AndroidInputHandler implements View.OnTouchListener,
     protected AndroidTouchInput touchInput;
     protected AndroidJoyInput joyInput;
     protected MouseInput mouseInput;
+    /**
+     * Pointer ids owned by the on-screen virtual joystick for the duration of the MotionEvent
+     * being dispatched - see {@link #isPointerCapturedByJoystick(int)}.
+     */
+    private long joystickPointerMask;
 
     public AndroidInputHandler() {
         touchInput = new AndroidTouchInput(this);
@@ -208,17 +213,42 @@ public boolean onTouch(View view, MotionEvent event) {
 //        logger.log(Level.INFO, "onTouch source: {0}, isTouch: {1}",
 //                new Object[]{source, isTouch});
 
-        if (isTouch && joyInput != null && joyInput.onTouch(event)) {
-            return true;
+        boolean joyConsumed = false;
+        joystickPointerMask = 0L;
+        if (isTouch && joyInput != null) {
+            // The union of the captures before and after the event, so that the pointer a
+            // DOWN captures (only in the "after" set) and the pointer an UP releases (only
+            // in the "before" set) are both hidden from touchInput below.
+            long capturedBefore = joyInput.getCapturedPointerMask();
+            joyConsumed = joyInput.onTouch(event);
+            joystickPointerMask = capturedBefore | joyInput.getCapturedPointerMask();
         }
 
         if (isTouch && touchInput != null) {
-            // send the event to the touch processor
+            // The virtual joystick doesn't get to swallow the whole MotionEvent: it only owns
+            // the pointers it has captured, and touchInput skips exactly those. Dropping the
+            // event outright would mean a finger resting on the on-screen stick blocked every
+            // other finger from being reported at all - so no looking around while moving, and
+            // a jump in accumulated drag as soon as the stick was released.
             consumed = touchInput.onTouch(event);
         }
 
-        return consumed;
+        return consumed || joyConsumed;
+
+    }
 
+    /**
+     * Returns whether the given pointer is driving an on-screen virtual joystick control in
+     * the MotionEvent currently being dispatched, and so must not also be reported as a touch
+     * (or emulated mouse) event. Only meaningful while {@link #onTouch(View, MotionEvent)} is
+     * on the stack.
+     *
+     * @param pointerId the Android pointer id to test
+     * @return true if the virtual joystick owns this pointer
+     */
+    public boolean isPointerCapturedByJoystick(int pointerId) {
+        return pointerId >= 0 && pointerId < Long.SIZE
+                && (joystickPointerMask & (1L << pointerId)) != 0;
     }
 
     @Override

jme3-android/src/main/java/com/jme3/input/android/AndroidJoyInput.java

@@ -225,6 +225,17 @@ public Joystick[] loadJoysticks(InputManager inputManager) {
         return joystickList.toArray( new Joystick[joystickList.size()] );
     }
 
+    /**
+     * Returns a bit mask of the pointer ids currently captured by the on-screen virtual
+     * joystick - see {@link VirtualJoystick#getCapturedPointerMask()}.
+     *
+     * @return the captured pointer ids as a bit mask, 0 if there is no virtual joystick
+     */
+    public long getCapturedPointerMask() {
+        VirtualJoystick joystick = virtualJoystick;
+        return joystick == null ? 0L : joystick.getCapturedPointerMask();
+    }
+
     public boolean onTouch(MotionEvent event) {
         VirtualJoystick joystick = virtualJoystick;
         if (joystick == null || inputHandler.getView() == null) {
@@ -240,6 +251,9 @@ public boolean onTouch(MotionEvent event) {
         switch (action) {
             case MotionEvent.ACTION_POINTER_DOWN:
             case MotionEvent.ACTION_DOWN:
+                // Touch is back in use, so a keyboard/gamepad seen earlier no longer
+                // justifies keeping the AUTO-mode virtual joystick suppressed.
+                keyboardSuppressedAutoJoystick = false;
                 consumed = joystick.onPointerDown(event.getPointerId(pointerIndex),
                         toJmeX(event.getX(pointerIndex)), toJmeY(event.getY(pointerIndex)), time);
                 break;

jme3-android/src/main/java/com/jme3/input/android/AndroidTouchInput.java

@@ -170,13 +170,16 @@ public boolean onTouch(MotionEvent event) {
         float jmeX;
         float jmeY;
 
-        numPointers = event.getPointerCount();
+        numPointers = countReportedPointers(event);
 
         // final int historySize = event.getHistorySize();
         //final int pointerCount = event.getPointerCount();
         switch (getAction(event)) {
             case MotionEvent.ACTION_POINTER_DOWN:
             case MotionEvent.ACTION_DOWN:
+                if (isJoystickPointer(pointerId)) {
+                    break;
+                }
                 jmeX = getJmeX(event.getX(pointerIndex));
                 jmeY = invertY(getJmeY(event.getY(pointerIndex)));
                 touch = getFreeTouchEvent();
@@ -193,17 +196,26 @@ public boolean onTouch(MotionEvent event) {
 
                 bWasHandled = true;
                 break;
-            case MotionEvent.ACTION_POINTER_UP:
             case MotionEvent.ACTION_CANCEL:
+                // A cancelled gesture ends every pointer at once, not just the one the event
+                // names - which, with the virtual joystick owning some of them, may well be a
+                // pointer that was never reported here to begin with.
+                bWasHandled = releaseAllPointers(event);
+                break;
+            case MotionEvent.ACTION_POINTER_UP:
             case MotionEvent.ACTION_UP:
+                if (lastPositions.remove(pointerId) == null) {
+                    // Never reported as DOWN - eg. it went to the virtual joystick instead -
+                    // so releasing it here would be an UP with no matching press.
+                    break;
+                }
                 jmeX = getJmeX(event.getX(pointerIndex));
                 jmeY = invertY(getJmeY(event.getY(pointerIndex)));
                 touch = getFreeTouchEvent();
                 touch.set(TouchEvent.Type.UP, jmeX, jmeY, 0, 0);
                 touch.setPointerId(pointerId);
                 touch.setTime(event.getEventTime());
                 touch.setPressure(event.getPressure(pointerIndex));
-                lastPositions.remove(pointerId);
 
                 addEvent(touch);
                 addEvent(generateMouseEvent(touch));
@@ -213,6 +225,9 @@ public boolean onTouch(MotionEvent event) {
             case MotionEvent.ACTION_MOVE:
                 // Convert all pointers into events
                 for (int p = 0; p < event.getPointerCount(); p++) {
+                    if (isJoystickPointer(event.getPointerId(p))) {
+                        continue;
+                    }
                     jmeX = getJmeX(event.getX(p));
                     jmeY = invertY(getJmeY(event.getY(p)));
                     lastPos = lastPositions.get(event.getPointerId(p));
@@ -243,17 +258,73 @@ public boolean onTouch(MotionEvent event) {
 
         }
 
-        // Try to detect gestures
-        if (gestureDetector != null) {
-            gestureDetector.onTouchEvent(event);
-        }
-        if (scaleDetector != null) {
-            scaleDetector.onTouchEvent(event);
+        // Try to detect gestures - but not for events the virtual joystick has a hand in,
+        // where a gesture spanning both the stick and another finger would be meaningless
+        // (and the detectors have no way to be told to ignore individual pointers).
+        if (numPointers == event.getPointerCount()) {
+            if (gestureDetector != null) {
+                gestureDetector.onTouchEvent(event);
+            }
+            if (scaleDetector != null) {
+                scaleDetector.onTouchEvent(event);
+            }
         }
 
         return bWasHandled;
     }
 
+    /**
+     * Emits an UP for every pointer currently being tracked, used when Android cancels the
+     * whole gesture.
+     *
+     * @param event the cancelling MotionEvent
+     * @return true if at least one pointer was released
+     */
+    private boolean releaseAllPointers(MotionEvent event) {
+        boolean released = false;
+        for (int p = 0; p < event.getPointerCount(); p++) {
+            int pointerId = event.getPointerId(p);
+            if (lastPositions.remove(pointerId) == null) {
+                continue;
+            }
+            TouchEvent touch = getFreeTouchEvent();
+            touch.set(TouchEvent.Type.UP, getJmeX(event.getX(p)), invertY(getJmeY(event.getY(p))), 0, 0);
+            touch.setPointerId(pointerId);
+            touch.setTime(event.getEventTime());
+            touch.setPressure(event.getPressure(p));
+
+            addEvent(touch);
+            addEvent(generateMouseEvent(touch));
+
+            released = true;
+        }
+        return released;
+    }
+
+    /**
+     * Counts the pointers in the event that this class actually reports, ie. those not being
+     * used to drive an on-screen virtual joystick control. Those are hidden from touch and
+     * mouse emulation entirely, so they mustn't count towards the multi-touch check in
+     * {@link #generateMouseEvent(TouchEvent)} either - otherwise holding the on-screen stick
+     * would suppress the emulated mouse events of the finger looking around.
+     *
+     * @param event the MotionEvent being dispatched
+     * @return the number of pointers reported by this class
+     */
+    private int countReportedPointers(MotionEvent event) {
+        int count = 0;
+        for (int p = 0; p < event.getPointerCount(); p++) {
+            if (!isJoystickPointer(event.getPointerId(p))) {
+                count++;
+            }
+        }
+        return count;
+    }
+
+    private boolean isJoystickPointer(int pointerId) {
+        return androidInput != null && androidInput.isPointerCapturedByJoystick(pointerId);
+    }
+
     // TODO: Ring Buffer for mouse events?
     public InputEvent generateMouseEvent(TouchEvent event) {
         InputEvent inputEvent = null;

jme3-core/src/main/java/com/jme3/input/virtual/VirtualJoystick.java

@@ -245,6 +245,30 @@ public boolean onPointerUp(int pointerId, float x, float y, long time) {
         }
     }
 
+    /**
+     * Returns a bit mask of the pointer ids currently captured by an on-screen control,
+     * bit <em>n</em> being set when pointer id <em>n</em> is captured.
+     *
+     * <p>Backends use this to tell apart the pointers this joystick owns from the ones that
+     * should still reach the rest of the input pipeline, so that a finger on the on-screen
+     * stick doesn't stop a second finger from being reported as an ordinary touch. Pointer
+     * ids of 64 or above cannot be represented and are omitted; no platform jME supports
+     * produces them.
+     *
+     * @return the captured pointer ids as a bit mask, 0 if none are captured
+     */
+    public long getCapturedPointerMask() {
+        synchronized (inputLock) {
+            long mask = 0L;
+            for (Integer pointerId : captures.keySet()) {
+                if (pointerId >= 0 && pointerId < Long.SIZE) {
+                    mask |= 1L << pointerId;
+                }
+            }
+            return mask;
+        }
+    }
+
     /**
      * Releases all active pointer captures.
      *