From 37606982664f56d4892768a12e69354ea39bd60c Mon Sep 17 00:00:00 2001
From: Eliaazzz <Elialiu760317@outlook.com>
Date: Fri, 24 Jul 2026 00:36:50 +1000
Subject: [PATCH 01/16] [Python] Refactor MatchContinuously onto the Watch
 transform

Route MatchContinuously through Watch when deduplication is enabled. The
polling loop and the set of already-matched file ids now live in the
splittable DoFn restriction, replacing the per-key state DoFns.

Because the matched ids are part of the restriction, a runner with
checkpointing enabled restores them after a restart and does not reprocess
files. The docstring is updated accordingly. Verified on Flink 1.20: after
killing the TaskManager mid stream the job restored from a checkpoint and
every file was still emitted exactly once.

has_deduplication=False keeps the previous PeriodicImpulse behaviour.
---
 sdks/python/apache_beam/io/fileio.py      | 255 +++++++++++++++-------
 sdks/python/apache_beam/io/fileio_test.py | 120 +++++++++-
 2 files changed, 294 insertions(+), 81 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index a333b7c89775..32f1b05edfd3 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -93,28 +93,37 @@
 import random
 import uuid
 from collections import namedtuple
-from functools import partial
 from typing import Any
 from typing import BinaryIO  # pylint: disable=unused-import
 from typing import Callable
 from typing import Iterable
+from typing import Optional
 from typing import Union
 
 import apache_beam as beam
+from apache_beam.coders.coders import FloatCoder
+from apache_beam.coders.coders import StrUtf8Coder
+from apache_beam.coders.coders import TupleCoder
+from apache_beam.coders.coders import VarIntCoder
 from apache_beam.io import filesystem
 from apache_beam.io import filesystems
 from apache_beam.io.filesystem import BeamIOError
 from apache_beam.io.filesystem import CompressionTypes
+from apache_beam.io.watch import PollFn
+from apache_beam.io.watch import PollResult
+from apache_beam.io.watch import TerminationCondition
+from apache_beam.io.watch import Watch
+from apache_beam.io.watch import never
 from apache_beam.options.pipeline_options import GoogleCloudOptions
 from apache_beam.options.value_provider import StaticValueProvider
 from apache_beam.options.value_provider import ValueProvider
 from apache_beam.transforms.periodicsequence import PeriodicImpulse
-from apache_beam.transforms.userstate import CombiningValueStateSpec
 from apache_beam.transforms.window import BoundedWindow
 from apache_beam.transforms.window import FixedWindows
 from apache_beam.transforms.window import GlobalWindow
 from apache_beam.transforms.window import IntervalWindow
 from apache_beam.utils.timestamp import MAX_TIMESTAMP
+from apache_beam.utils.timestamp import Duration
 from apache_beam.utils.timestamp import Timestamp
 
 __all__ = [
@@ -251,6 +260,98 @@ def process(
     yield ReadableFile(metadata, self._compression)
 
 
+class _PollClock(object):
+  """The poll-time clock reading one ``MatchContinuously`` round shares.
+
+  The start gate (a poll before ``start_timestamp`` emits nothing) and the
+  poll budget (only polls at or after ``start_timestamp`` consume it) must
+  agree on a single reading. With independent clock reads, a round straddling
+  the boundary could consume the budget without having matched anything. The
+  poll function writes the reading and the termination condition consumes it
+  within the same round; instances are not shared across bundle threads.
+  """
+  def __init__(self):
+    self.last_poll_micros: Optional[int] = None
+
+
+class _WatchWindowTermination(TerminationCondition):
+  """Stops after the polls that fall in the ``[start, stop)`` window.
+
+  ``max_polls`` is the ``PeriodicImpulse`` tick count
+  ``ceil((stop - start) / interval)``, so the number of polls is independent of
+  how fast the runner reschedules deferred work. Only polls at or after
+  ``start`` count toward the budget, judged by the poll's own clock reading in
+  ``clock``; earlier polls are deferred waits that must not consume it,
+  matching ``PeriodicImpulse``, which never advances a tick while waiting for
+  the start time.
+  """
+  def __init__(self, clock: _PollClock, start_micros: int, max_polls: int):
+    self._clock = clock
+    self._start_micros = start_micros
+    self._max_polls = max_polls
+
+  def for_new_input(self, now, element):
+    return 0
+
+  def on_poll_complete(self, state):
+    poll_micros = self._clock.last_poll_micros
+    if poll_micros is not None and poll_micros >= self._start_micros:
+      return state + 1
+    return state
+
+  def can_stop_polling(self, now, state):
+    return state >= self._max_polls
+
+  def state_coder(self):
+    return VarIntCoder()
+
+
+def _file_path_key(metadata: filesystem.FileMetadata) -> str:
+  return metadata.path
+
+
+def _file_path_and_mtime_key(
+    metadata: filesystem.FileMetadata) -> tuple[str, float]:
+  # Keying on the last-modified time makes a file whose timestamp changed look
+  # new again, mirroring the Java SDK's ExtractFilenameAndLastUpdateFn — which
+  # also rejects a missing (zero) timestamp, since updates could never be seen.
+  if not metadata.last_updated_in_seconds:
+    raise BeamIOError(
+        'MatchContinuously(match_updated_files=True) requires file '
+        'last-modified times, but %s reports none.' % metadata.path)
+  return metadata.path, metadata.last_updated_in_seconds
+
+
+class _MatchContinuouslyPollFn(PollFn):
+  """Polls a file pattern for ``MatchContinuously``, honoring empty-match rules.
+
+  Emits no outputs before ``start_timestamp`` so polling can start ahead of the
+  first intended match. Each match is stamped with the poll time as its event
+  time, and the watermark advances to the poll time so downstream event-time
+  windows keep progressing even when a poll finds no new files. Each round's
+  clock reading is recorded in ``clock`` so ``_WatchWindowTermination`` judges
+  the start boundary by the same reading as the gate below.
+  """
+  def __init__(self, empty_match_treatment, start_timestamp, clock=None):
+    self._empty_match_treatment = empty_match_treatment
+    self._start_micros = Timestamp.of(start_timestamp).micros
+    self._clock = clock if clock is not None else _PollClock()
+
+  def __call__(self, file_pattern: str) -> PollResult:
+    now = Timestamp.now()
+    self._clock.last_poll_micros = now.micros
+    if now.micros < self._start_micros:
+      return PollResult.incomplete(())
+    match_result = filesystems.FileSystems.match([file_pattern])[0]
+    if (not match_result.metadata_list and
+        not EmptyMatchTreatment.allow_empty_match(file_pattern,
+                                                  self._empty_match_treatment)):
+      raise BeamIOError(
+          'Empty match for pattern %s. Disallowed.' % file_pattern)
+    return PollResult.incomplete(
+        match_result.metadata_list, timestamp=now).with_watermark(now)
+
+
 class MatchContinuously(beam.PTransform):
   """Checks for new files for a given pattern every interval.
 
@@ -261,10 +362,11 @@ class MatchContinuously(beam.PTransform):
   guarantees.
 
   Matching continuously scales poorly, as it is stateful, and requires storing
-  file ids in memory. In addition, because it is memory-only, if a pipeline is
-  restarted, already processed files will be reprocessed. Consider an alternate
-  technique, such as Pub/Sub Notifications
-  (https://cloud.google.com/storage/docs/pubsub-notifications)
+  file ids for every file the pattern has matched. With ``has_deduplication``
+  enabled those ids are kept in the splittable DoFn restriction, so a runner
+  with checkpointing enabled restores them after a restart and does not
+  reprocess files. Consider an alternate technique, such as Pub/Sub
+  Notifications (https://cloud.google.com/storage/docs/pubsub-notifications)
   when using GCS if possible.
   """
   def __init__(
@@ -306,37 +408,81 @@ def __init__(
         'if possible')
 
   def expand(self, pbegin) -> beam.PCollection[filesystem.FileMetadata]:
-    # invoke periodic impulse
-    impulse = pbegin | PeriodicImpulse(
-        start_timestamp=self.start_ts,
-        stop_timestamp=self.stop_ts,
-        fire_interval=self.interval)
-
-    # match file pattern periodically
-    file_pattern = self.file_pattern
-    match_files = (
-        impulse
-        | 'GetFilePattern' >> beam.Map(lambda x: file_pattern)
-        | MatchAll(self.empty_match_treatment))
-
-    # apply deduplication strategy if required
+    if Duration.of(self.interval).micros <= 0:
+      raise ValueError('MatchContinuously interval must be positive.')
     if self.has_deduplication:
-      # Making a Key Value so each file has its own state.
-      match_files = match_files | 'ToKV' >> beam.Map(lambda x: (x.path, x))
-      if self.match_upd:
-        match_files = match_files | 'RemoveOldAlreadyRead' >> beam.ParDo(
-            _RemoveOldDuplicates())
-      else:
-        match_files = match_files | 'RemoveAlreadyRead' >> beam.ParDo(
-            _RemoveDuplicates())
-
-    # apply windowing if required. Apply at last because deduplication relies on
-    # the global window.
+      match_files = self._match_deduplicated(pbegin)
+    else:
+      match_files = self._match_all_each_poll(pbegin)
+
+    # Apply windowing last because dedup relies on the global window.
     if self.apply_windowing:
       match_files = match_files | beam.WindowInto(FixedWindows(self.interval))
 
     return match_files
 
+  def _match_deduplicated(self,
+                          pbegin) -> beam.PCollection[filesystem.FileMetadata]:
+    # The Watch transform polls the pattern and emits each file once per key:
+    # the path, joined by the last-modified time when matching updated files.
+    # stop_timestamp bounds the watch to the polls that fall in [start, stop).
+    clock = _PollClock()
+    if self.stop_ts == MAX_TIMESTAMP:
+      termination = never()
+    else:
+      start_ts = Timestamp.of(self.start_ts)
+      stop_ts = Timestamp.of(self.stop_ts)
+      if stop_ts < start_ts:
+        raise ValueError(
+            'MatchContinuously stop_timestamp %s precedes start_timestamp %s' %
+            (stop_ts, start_ts))
+      interval_micros = Duration.of(self.interval).micros
+      span_micros = (stop_ts - start_ts).micros
+      # Ceiling division reproduces PeriodicImpulse's tick count; the window
+      # upper bound is exclusive.
+      max_polls = -(-span_micros // interval_micros)
+      if max_polls == 0:
+        # An empty [start, stop) window never ticks in PeriodicImpulse. Fall
+        # back to the impulse path — with zero ticks dedup is moot — so the
+        # output stays empty and unbounded, rather than let Watch run its
+        # unconditional first poll.
+        return self._match_all_each_poll(pbegin)
+      termination = _WatchWindowTermination(clock, start_ts.micros, max_polls)
+    if self.match_upd:
+      output_key_fn = _file_path_and_mtime_key
+      output_key_coder = TupleCoder([StrUtf8Coder(), FloatCoder()])
+    else:
+      output_key_fn = _file_path_key
+      output_key_coder = StrUtf8Coder()
+    poll_fn = _MatchContinuouslyPollFn(
+        self.empty_match_treatment, self.start_ts, clock)
+    watch = Watch(
+        poll_fn,
+        poll_interval=self.interval,
+        termination=termination,
+        output_key_fn=output_key_fn,
+        output_key_coder=output_key_coder)
+    # Watch emits (pattern, file) pairs; keep the FileMetadata output type so
+    # downstream transforms stay typed instead of falling back to Any.
+    return (
+        pbegin
+        | 'Impulse' >> beam.Create([self.file_pattern])
+        | 'Watch' >> watch
+        | 'DropPattern' >> beam.Map(lambda kv: kv[1]).with_output_types(
+            filesystem.FileMetadata))
+
+  def _match_all_each_poll(self,
+                           pbegin) -> beam.PCollection[filesystem.FileMetadata]:
+    # No deduplication: re-emit every match on each poll.
+    return (
+        pbegin
+        | PeriodicImpulse(
+            start_timestamp=self.start_ts,
+            stop_timestamp=self.stop_ts,
+            fire_interval=self.interval)
+        | 'GetFilePattern' >> beam.Map(lambda x: self.file_pattern)
+        | MatchAll(self.empty_match_treatment))
+
 
 class ReadMatches(beam.PTransform):
   """Converts each result of MatchFiles() or MatchAll() to a ReadableFile.
@@ -892,50 +1038,3 @@ def finish_bundle(self):
               timestamp=key[1].start,
               windows=[key[1]]  # TODO(pabloem) HOW DO WE GET THE PANE
           ))
-
-
-class _RemoveDuplicates(beam.DoFn):
-  """Internal DoFn that filters out filenames already seen (even though the file
-  has updated)."""
-  COUNT_STATE = CombiningValueStateSpec('count', combine_fn=sum)
-
-  def process(
-      self,
-      element: tuple[str, filesystem.FileMetadata],
-      count_state=beam.DoFn.StateParam(COUNT_STATE)
-  ) -> Iterable[filesystem.FileMetadata]:
-
-    path = element[0]
-    file_metadata = element[1]
-    counter = count_state.read()
-
-    if counter == 0:
-      count_state.add(1)
-      _LOGGER.debug('Generated entry for file %s', path)
-      yield file_metadata
-    else:
-      _LOGGER.debug('File %s was already read, seen %d times', path, counter)
-
-
-class _RemoveOldDuplicates(beam.DoFn):
-  """Internal DoFn that filters out filenames already seen and timestamp
-  unchanged."""
-  TIME_STATE = CombiningValueStateSpec(
-      'count', combine_fn=partial(max, default=0.0))
-
-  def process(
-      self,
-      element: tuple[str, filesystem.FileMetadata],
-      time_state=beam.DoFn.StateParam(TIME_STATE)
-  ) -> Iterable[filesystem.FileMetadata]:
-    path = element[0]
-    file_metadata = element[1]
-    new_ts = file_metadata.last_updated_in_seconds
-    old_ts = time_state.read()
-
-    if old_ts < new_ts:
-      time_state.add(new_ts)
-      _LOGGER.debug('Generated entry for file %s', path)
-      yield file_metadata
-    else:
-      _LOGGER.debug('File %s was already read', path)
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index ce535265ef2f..42ce976c673e 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -28,13 +28,13 @@
 import uuid
 import warnings
 
-import pytest
-from hamcrest.library.text import stringmatches
-
 import apache_beam as beam
+import pytest
 from apache_beam.io import fileio
 from apache_beam.io.filebasedsink_test import _TestCaseWithTempDirCleanUp
+from apache_beam.io.filesystem import BeamIOError
 from apache_beam.io.filesystem import CompressionTypes
+from apache_beam.io.filesystem import FileMetadata
 from apache_beam.io.filesystems import FileSystems
 from apache_beam.options.pipeline_options import PipelineOptions
 from apache_beam.options.pipeline_options import StandardOptions
@@ -49,6 +49,7 @@
 from apache_beam.transforms.window import GlobalWindow
 from apache_beam.transforms.window import IntervalWindow
 from apache_beam.utils.timestamp import Timestamp
+from hamcrest.library.text import stringmatches
 
 warnings.filterwarnings(
     'ignore', category=FutureWarning, module='apache_beam.io.fileio_test')
@@ -420,6 +421,119 @@ def _create_extra_file(element):
 
       assert_that(match_continiously, equal_to(files))
 
+  def test_poll_fn_gates_on_start_timestamp(self):
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    self._create_temp_file(dir=tempdir)
+    pattern = FileSystems.join(tempdir, '*')
+
+    future_start = fileio._MatchContinuouslyPollFn(
+        fileio.EmptyMatchTreatment.ALLOW, Timestamp.now() + 3600)
+    self.assertEqual((), future_start(pattern).outputs)
+
+    past_start = fileio._MatchContinuouslyPollFn(
+        fileio.EmptyMatchTreatment.ALLOW, Timestamp.now() - 3600)
+    self.assertEqual(1, len(past_start(pattern).outputs))
+
+  def test_poll_fn_disallows_empty_match(self):
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    poll_fn = fileio._MatchContinuouslyPollFn(
+        fileio.EmptyMatchTreatment.DISALLOW, Timestamp.now() - 3600)
+    with self.assertRaises(BeamIOError):
+      poll_fn(FileSystems.join(tempdir, 'no-such-file'))
+
+  def test_poll_fn_stamps_outputs_with_poll_time(self):
+    # Matches always carry the poll time as event time, matching the Java
+    # SDK's MatchPollFn; matching updated files must not change that.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    self._create_temp_file(dir=tempdir)
+    poll_fn = fileio._MatchContinuouslyPollFn(
+        fileio.EmptyMatchTreatment.ALLOW, Timestamp.now() - 3600)
+    before = Timestamp.now()
+    result = poll_fn(FileSystems.join(tempdir, '*'))
+    after = Timestamp.now()
+    self.assertEqual(1, len(result.outputs))
+    output = result.outputs[0]
+    self.assertLessEqual(before, output.timestamp)
+    self.assertLessEqual(output.timestamp, after)
+    self.assertEqual(result.watermark, output.timestamp)
+
+  def test_match_updated_files_keys_on_path_and_mtime(self):
+    # An updated file dedups as new because its key changes, mirroring the
+    # Java SDK's ExtractFilenameAndLastUpdateFn.
+    metadata = FileMetadata('/tmp/a', 1, 1234.5)
+    self.assertEqual(('/tmp/a', 1234.5),
+                     fileio._file_path_and_mtime_key(metadata))
+
+  def test_match_updated_files_rejects_missing_mtime(self):
+    # Java's ExtractFilenameAndLastUpdateFn rejects a zero last-modified time:
+    # without mtimes, updates could never be detected.
+    with self.assertRaises(BeamIOError):
+      fileio._file_path_and_mtime_key(FileMetadata('/tmp/a', 1))
+
+  def test_start_equals_stop_matches_nothing(self):
+    # PeriodicImpulse's [start, stop) tick window is empty when start == stop;
+    # the deduplicated path must skip Watch's unconditional first poll by
+    # falling back to the impulse path, which also keeps the output unbounded.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    self._create_temp_file(dir=tempdir)
+    start = Timestamp.now()
+    with TestPipeline() as p:
+      match_continiously = (
+          p
+          | fileio.MatchContinuously(
+              file_pattern=FileSystems.join(tempdir, '*'),
+              interval=0.2,
+              start_timestamp=start,
+              stop_timestamp=start))
+      assert_that(match_continiously, equal_to([]))
+
+  def test_rejects_nonpositive_interval(self):
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    with self.assertRaisesRegex(ValueError, 'interval must be positive'):
+      with TestPipeline() as p:
+        _ = p | fileio.MatchContinuously(
+            file_pattern=FileSystems.join(tempdir, '*'), interval=0)
+
+  def test_watch_window_termination_ignores_pre_start_polls(self):
+    # Polls before start_timestamp are deferred waits and must not consume the
+    # budget, otherwise a future start_timestamp silently drops all output. The
+    # boundary is judged by the poll's own clock reading, so a round straddling
+    # the start cannot consume the budget without having matched.
+    start_micros = Timestamp.of(1000).micros
+    clock = fileio._PollClock()
+    term = fileio._WatchWindowTermination(clock, start_micros, max_polls=2)
+    now = Timestamp.of(999)
+    state = term.for_new_input(now, 'pattern')
+    clock.last_poll_micros = Timestamp.of(999).micros
+    state = term.on_poll_complete(state)
+    state = term.on_poll_complete(state)
+    self.assertFalse(term.can_stop_polling(now, state))
+    clock.last_poll_micros = Timestamp.of(1000).micros
+    state = term.on_poll_complete(state)
+    self.assertFalse(term.can_stop_polling(now, state))
+    state = term.on_poll_complete(state)
+    self.assertTrue(term.can_stop_polling(now, state))
+
+  def test_poll_fn_records_its_clock_reading_for_the_termination(self):
+    # The gate and the poll budget share one reading per round; see _PollClock.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    clock = fileio._PollClock()
+    poll_fn = fileio._MatchContinuouslyPollFn(
+        fileio.EmptyMatchTreatment.ALLOW, Timestamp.now() + 3600, clock)
+    self.assertIsNone(clock.last_poll_micros)
+    poll_fn(FileSystems.join(tempdir, '*'))
+    self.assertIsNotNone(clock.last_poll_micros)
+
+  def test_poll_fn_advances_watermark_on_empty_match(self):
+    # An empty (but allowed) match still carries a watermark so downstream
+    # event-time windows keep progressing when no new files appear.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    poll_fn = fileio._MatchContinuouslyPollFn(
+        fileio.EmptyMatchTreatment.ALLOW, Timestamp.now() - 3600)
+    result = poll_fn(FileSystems.join(tempdir, '*'))
+    self.assertEqual((), result.outputs)
+    self.assertIsNotNone(result.watermark)
+
 
 class WriteFilesTest(_TestCaseWithTempDirCleanUp):
 

From 4a804a48a5835dd5c874d51a9892e7b2edf5a98e Mon Sep 17 00:00:00 2001
From: Eliaazzz <Elialiu760317@outlook.com>
Date: Wed, 29 Jul 2026 18:01:23 +1000
Subject: [PATCH 02/16] Address review: close the coder inference gap, drop
 explicit key coders

registry.get_coder receives typing and native generic annotations such
as tuple[str, float] unconverted and falls back to pickling. Watch now
converts hints with convert_to_beam_type before the registry lookup, so
MatchContinuously's annotated key functions infer the same StrUtf8Coder
and TupleCoder the explicit settings supplied. Also trims the docstrings
and comments this PR adds.
---
 sdks/python/apache_beam/io/fileio.py     | 57 +++++++-----------------
 sdks/python/apache_beam/io/watch.py      | 12 ++++-
 sdks/python/apache_beam/io/watch_test.py | 12 +++++
 3 files changed, 39 insertions(+), 42 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index 32f1b05edfd3..a0315d3d6ffe 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -101,9 +101,6 @@
 from typing import Union
 
 import apache_beam as beam
-from apache_beam.coders.coders import FloatCoder
-from apache_beam.coders.coders import StrUtf8Coder
-from apache_beam.coders.coders import TupleCoder
 from apache_beam.coders.coders import VarIntCoder
 from apache_beam.io import filesystem
 from apache_beam.io import filesystems
@@ -261,15 +258,8 @@ def process(
 
 
 class _PollClock(object):
-  """The poll-time clock reading one ``MatchContinuously`` round shares.
-
-  The start gate (a poll before ``start_timestamp`` emits nothing) and the
-  poll budget (only polls at or after ``start_timestamp`` consume it) must
-  agree on a single reading. With independent clock reads, a round straddling
-  the boundary could consume the budget without having matched anything. The
-  poll function writes the reading and the termination condition consumes it
-  within the same round; instances are not shared across bundle threads.
-  """
+  """Shares one clock reading per poll round, so the start gate and the poll
+  budget judge the ``start_timestamp`` boundary consistently."""
   def __init__(self):
     self.last_poll_micros: Optional[int] = None
 
@@ -278,12 +268,8 @@ class _WatchWindowTermination(TerminationCondition):
   """Stops after the polls that fall in the ``[start, stop)`` window.
 
   ``max_polls`` is the ``PeriodicImpulse`` tick count
-  ``ceil((stop - start) / interval)``, so the number of polls is independent of
-  how fast the runner reschedules deferred work. Only polls at or after
-  ``start`` count toward the budget, judged by the poll's own clock reading in
-  ``clock``; earlier polls are deferred waits that must not consume it,
-  matching ``PeriodicImpulse``, which never advances a tick while waiting for
-  the start time.
+  ``ceil((stop - start) / interval)``; polls before ``start`` are waiting
+  rounds and do not consume the budget.
   """
   def __init__(self, clock: _PollClock, start_micros: int, max_polls: int):
     self._clock = clock
@@ -312,9 +298,8 @@ def _file_path_key(metadata: filesystem.FileMetadata) -> str:
 
 def _file_path_and_mtime_key(
     metadata: filesystem.FileMetadata) -> tuple[str, float]:
-  # Keying on the last-modified time makes a file whose timestamp changed look
-  # new again, mirroring the Java SDK's ExtractFilenameAndLastUpdateFn — which
-  # also rejects a missing (zero) timestamp, since updates could never be seen.
+  # Keying on the last-modified time makes a changed file look new again. A
+  # missing (zero) timestamp is rejected because updates could never be seen.
   if not metadata.last_updated_in_seconds:
     raise BeamIOError(
         'MatchContinuously(match_updated_files=True) requires file '
@@ -323,14 +308,11 @@ def _file_path_and_mtime_key(
 
 
 class _MatchContinuouslyPollFn(PollFn):
-  """Polls a file pattern for ``MatchContinuously``, honoring empty-match rules.
-
-  Emits no outputs before ``start_timestamp`` so polling can start ahead of the
-  first intended match. Each match is stamped with the poll time as its event
-  time, and the watermark advances to the poll time so downstream event-time
-  windows keep progressing even when a poll finds no new files. Each round's
-  clock reading is recorded in ``clock`` so ``_WatchWindowTermination`` judges
-  the start boundary by the same reading as the gate below.
+  """Polls a file pattern, honoring empty-match rules.
+
+  A poll before ``start_timestamp`` emits nothing. Matches carry the poll time
+  as their event time, and the watermark advances to the poll time so
+  event-time windows progress even when nothing new matches.
   """
   def __init__(self, empty_match_treatment, start_timestamp, clock=None):
     self._empty_match_treatment = empty_match_treatment
@@ -423,9 +405,8 @@ def expand(self, pbegin) -> beam.PCollection[filesystem.FileMetadata]:
 
   def _match_deduplicated(self,
                           pbegin) -> beam.PCollection[filesystem.FileMetadata]:
-    # The Watch transform polls the pattern and emits each file once per key:
-    # the path, joined by the last-modified time when matching updated files.
-    # stop_timestamp bounds the watch to the polls that fall in [start, stop).
+    # Watch emits each file once per key: the path, joined by the mtime when
+    # matching updated files; stop_timestamp bounds the polls to [start, stop).
     clock = _PollClock()
     if self.stop_ts == MAX_TIMESTAMP:
       termination = never()
@@ -442,26 +423,22 @@ def _match_deduplicated(self,
       # upper bound is exclusive.
       max_polls = -(-span_micros // interval_micros)
       if max_polls == 0:
-        # An empty [start, stop) window never ticks in PeriodicImpulse. Fall
-        # back to the impulse path — with zero ticks dedup is moot — so the
-        # output stays empty and unbounded, rather than let Watch run its
-        # unconditional first poll.
+        # An empty [start, stop) window never ticks; the impulse path keeps
+        # the output empty without Watch's unconditional first poll.
         return self._match_all_each_poll(pbegin)
       termination = _WatchWindowTermination(clock, start_ts.micros, max_polls)
     if self.match_upd:
       output_key_fn = _file_path_and_mtime_key
-      output_key_coder = TupleCoder([StrUtf8Coder(), FloatCoder()])
     else:
       output_key_fn = _file_path_key
-      output_key_coder = StrUtf8Coder()
     poll_fn = _MatchContinuouslyPollFn(
         self.empty_match_treatment, self.start_ts, clock)
+    # The key coder is inferred from the key function's return annotation.
     watch = Watch(
         poll_fn,
         poll_interval=self.interval,
         termination=termination,
-        output_key_fn=output_key_fn,
-        output_key_coder=output_key_coder)
+        output_key_fn=output_key_fn)
     # Watch emits (pattern, file) pairs; keep the FileMetadata output type so
     # downstream transforms stay typed instead of falling back to Any.
     return (
diff --git a/sdks/python/apache_beam/io/watch.py b/sdks/python/apache_beam/io/watch.py
index 2fcee7a8080f..7d0977580f2a 100644
--- a/sdks/python/apache_beam/io/watch.py
+++ b/sdks/python/apache_beam/io/watch.py
@@ -78,6 +78,7 @@ def poll(prefix) -> PollResult[str]:
 from apache_beam.transforms import PTransform
 from apache_beam.transforms import core
 from apache_beam.transforms.window import TimestampedValue
+from apache_beam.typehints import native_type_compatibility
 from apache_beam.utils.timestamp import MAX_TIMESTAMP
 from apache_beam.utils.timestamp import Duration
 from apache_beam.utils.timestamp import Timestamp
@@ -643,6 +644,13 @@ def _poll_output_type(poll_fn) -> Any:
   return Any
 
 
+def _coder_for_hint(hint) -> Coder:
+  # typing and native generic hints such as tuple[str, float] must be
+  # converted to Beam typehints, or the registry falls back to pickling.
+  return coders.registry.get_coder(
+      native_type_compatibility.convert_to_beam_type(hint))
+
+
 class Watch(PTransform):
   """Watches a growing set of outputs per input via a periodic poll function.
 
@@ -690,14 +698,14 @@ def expand(self, pcoll):
     if output_coder is None and isinstance(self._poll_fn, PollFn):
       output_coder = self._poll_fn.default_output_coder()
     if output_coder is None:
-      output_coder = coders.registry.get_coder(_poll_output_type(self._poll_fn))
+      output_coder = _coder_for_hint(_poll_output_type(self._poll_fn))
     if self._output_key_fn is None:
       # The output is its own dedup key, so the key coder is the output coder.
       key_fn = _identity
       key_coder = self._output_key_coder or output_coder
     else:
       key_fn = self._output_key_fn
-      key_coder = self._output_key_coder or coders.registry.get_coder(
+      key_coder = self._output_key_coder or _coder_for_hint(
           _return_type(self._output_key_fn))
     # Dedup hashes the encoded key, so equal keys must encode equally; use the
     # coder's deterministic form and reject coders that have none.
diff --git a/sdks/python/apache_beam/io/watch_test.py b/sdks/python/apache_beam/io/watch_test.py
index 8c1f6571da66..8d3d315d3033 100644
--- a/sdks/python/apache_beam/io/watch_test.py
+++ b/sdks/python/apache_beam/io/watch_test.py
@@ -444,6 +444,18 @@ def test_infers_output_coder_from_return_annotation(self):
           | Watch(_complete_poll, poll_interval=Duration(1)))
       self.assertEqual(typehints.Tuple[str, str], output.element_type)
 
+  def test_infers_coder_from_native_generic_annotation(self):
+    # tuple[str, float] resolves to a tuple coder, not the pickling fallback.
+    def poll(element) -> PollResult[tuple[str, float]]:
+      return PollResult.complete([(element, 1.0)])
+
+    with self._in_memory_pipeline() as p:
+      output = (
+          p | beam.Create(['k:']) | Watch(poll, poll_interval=Duration(1)))
+      self.assertEqual(
+          typehints.Tuple[str, typehints.Tuple[str, float]],
+          output.element_type)
+
   def test_uses_poll_fn_default_output_coder(self):
     with self._in_memory_pipeline() as p:
       output = (

From 4587416c3b59da74fd1bbdd594fc6deed600a8c2 Mon Sep 17 00:00:00 2001
From: Eliaazzz <Elialiu760317@outlook.com>
Date: Wed, 29 Jul 2026 18:38:56 +1000
Subject: [PATCH 03/16] Annotate poll output type, cover typing.Tuple
 inference, sort test imports

Annotates _MatchContinuouslyPollFn with PollResult[FileMetadata], covers
typing.Tuple key inference alongside the native form, reorders the
third-party test imports, and drops the remaining Java references from
test comments.
---
 sdks/python/apache_beam/io/fileio.py      |  2 +-
 sdks/python/apache_beam/io/fileio_test.py | 16 +++++++--------
 sdks/python/apache_beam/io/watch_test.py  | 25 +++++++++++++++--------
 3 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index a0315d3d6ffe..154732543e3d 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -319,7 +319,7 @@ def __init__(self, empty_match_treatment, start_timestamp, clock=None):
     self._start_micros = Timestamp.of(start_timestamp).micros
     self._clock = clock if clock is not None else _PollClock()
 
-  def __call__(self, file_pattern: str) -> PollResult:
+  def __call__(self, file_pattern: str) -> PollResult[filesystem.FileMetadata]:
     now = Timestamp.now()
     self._clock.last_poll_micros = now.micros
     if now.micros < self._start_micros:
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index 42ce976c673e..f5562e0202a0 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -28,8 +28,10 @@
 import uuid
 import warnings
 
-import apache_beam as beam
 import pytest
+from hamcrest.library.text import stringmatches
+
+import apache_beam as beam
 from apache_beam.io import fileio
 from apache_beam.io.filebasedsink_test import _TestCaseWithTempDirCleanUp
 from apache_beam.io.filesystem import BeamIOError
@@ -49,7 +51,6 @@
 from apache_beam.transforms.window import GlobalWindow
 from apache_beam.transforms.window import IntervalWindow
 from apache_beam.utils.timestamp import Timestamp
-from hamcrest.library.text import stringmatches
 
 warnings.filterwarnings(
     'ignore', category=FutureWarning, module='apache_beam.io.fileio_test')
@@ -442,8 +443,8 @@ def test_poll_fn_disallows_empty_match(self):
       poll_fn(FileSystems.join(tempdir, 'no-such-file'))
 
   def test_poll_fn_stamps_outputs_with_poll_time(self):
-    # Matches always carry the poll time as event time, matching the Java
-    # SDK's MatchPollFn; matching updated files must not change that.
+    # Matches always carry the poll time as event time; matching updated
+    # files must not change that.
     tempdir = '%s%s' % (self._new_tempdir(), os.sep)
     self._create_temp_file(dir=tempdir)
     poll_fn = fileio._MatchContinuouslyPollFn(
@@ -458,15 +459,14 @@ def test_poll_fn_stamps_outputs_with_poll_time(self):
     self.assertEqual(result.watermark, output.timestamp)
 
   def test_match_updated_files_keys_on_path_and_mtime(self):
-    # An updated file dedups as new because its key changes, mirroring the
-    # Java SDK's ExtractFilenameAndLastUpdateFn.
+    # An updated file dedups as new because its key changes.
     metadata = FileMetadata('/tmp/a', 1, 1234.5)
     self.assertEqual(('/tmp/a', 1234.5),
                      fileio._file_path_and_mtime_key(metadata))
 
   def test_match_updated_files_rejects_missing_mtime(self):
-    # Java's ExtractFilenameAndLastUpdateFn rejects a zero last-modified time:
-    # without mtimes, updates could never be detected.
+    # A zero last-modified time is rejected: without mtimes, updates could
+    # never be detected.
     with self.assertRaises(BeamIOError):
       fileio._file_path_and_mtime_key(FileMetadata('/tmp/a', 1))
 
diff --git a/sdks/python/apache_beam/io/watch_test.py b/sdks/python/apache_beam/io/watch_test.py
index 8d3d315d3033..472177ceaee6 100644
--- a/sdks/python/apache_beam/io/watch_test.py
+++ b/sdks/python/apache_beam/io/watch_test.py
@@ -18,6 +18,7 @@
 """Tests for the Watch transform."""
 
 import collections
+import typing
 import unittest
 
 import apache_beam as beam
@@ -444,17 +445,23 @@ def test_infers_output_coder_from_return_annotation(self):
           | Watch(_complete_poll, poll_interval=Duration(1)))
       self.assertEqual(typehints.Tuple[str, str], output.element_type)
 
-  def test_infers_coder_from_native_generic_annotation(self):
-    # tuple[str, float] resolves to a tuple coder, not the pickling fallback.
-    def poll(element) -> PollResult[tuple[str, float]]:
+  def test_infers_coder_from_generic_annotations(self):
+    # tuple[str, float] and typing.Tuple[str, float] resolve to a tuple coder,
+    # not the pickling fallback.
+    def native_poll(element) -> PollResult[tuple[str, float]]:
       return PollResult.complete([(element, 1.0)])
 
-    with self._in_memory_pipeline() as p:
-      output = (
-          p | beam.Create(['k:']) | Watch(poll, poll_interval=Duration(1)))
-      self.assertEqual(
-          typehints.Tuple[str, typehints.Tuple[str, float]],
-          output.element_type)
+    def typing_poll(
+        element) -> PollResult[typing.Tuple[str, float]]:  # noqa: UP006
+      return PollResult.complete([(element, 1.0)])
+
+    for poll in (native_poll, typing_poll):
+      with self._in_memory_pipeline() as p:
+        output = (
+            p | beam.Create(['k:']) | Watch(poll, poll_interval=Duration(1)))
+        self.assertEqual(
+            typehints.Tuple[str, typehints.Tuple[str, float]],
+            output.element_type)
 
   def test_uses_poll_fn_default_output_coder(self):
     with self._in_memory_pipeline() as p:

From b8b96ae6d74ceb017fae4fb5813b0a0cc2236f18 Mon Sep 17 00:00:00 2001
From: Eliaazzz <Elialiu760317@outlook.com>
Date: Thu, 30 Jul 2026 00:48:53 +1000
Subject: [PATCH 04/16] Split the coder inference fix into #39547

watch.py and watch_test.py return to master; the inference fix lands
separately so it can make the release cut. The annotated key functions
meanwhile fall back to the deterministic FastPrimitivesCoder form, which
keeps dedup correct.
---
 sdks/python/apache_beam/io/watch.py      | 12 ++----------
 sdks/python/apache_beam/io/watch_test.py | 19 -------------------
 2 files changed, 2 insertions(+), 29 deletions(-)

diff --git a/sdks/python/apache_beam/io/watch.py b/sdks/python/apache_beam/io/watch.py
index 7d0977580f2a..2fcee7a8080f 100644
--- a/sdks/python/apache_beam/io/watch.py
+++ b/sdks/python/apache_beam/io/watch.py
@@ -78,7 +78,6 @@ def poll(prefix) -> PollResult[str]:
 from apache_beam.transforms import PTransform
 from apache_beam.transforms import core
 from apache_beam.transforms.window import TimestampedValue
-from apache_beam.typehints import native_type_compatibility
 from apache_beam.utils.timestamp import MAX_TIMESTAMP
 from apache_beam.utils.timestamp import Duration
 from apache_beam.utils.timestamp import Timestamp
@@ -644,13 +643,6 @@ def _poll_output_type(poll_fn) -> Any:
   return Any
 
 
-def _coder_for_hint(hint) -> Coder:
-  # typing and native generic hints such as tuple[str, float] must be
-  # converted to Beam typehints, or the registry falls back to pickling.
-  return coders.registry.get_coder(
-      native_type_compatibility.convert_to_beam_type(hint))
-
-
 class Watch(PTransform):
   """Watches a growing set of outputs per input via a periodic poll function.
 
@@ -698,14 +690,14 @@ def expand(self, pcoll):
     if output_coder is None and isinstance(self._poll_fn, PollFn):
       output_coder = self._poll_fn.default_output_coder()
     if output_coder is None:
-      output_coder = _coder_for_hint(_poll_output_type(self._poll_fn))
+      output_coder = coders.registry.get_coder(_poll_output_type(self._poll_fn))
     if self._output_key_fn is None:
       # The output is its own dedup key, so the key coder is the output coder.
       key_fn = _identity
       key_coder = self._output_key_coder or output_coder
     else:
       key_fn = self._output_key_fn
-      key_coder = self._output_key_coder or _coder_for_hint(
+      key_coder = self._output_key_coder or coders.registry.get_coder(
           _return_type(self._output_key_fn))
     # Dedup hashes the encoded key, so equal keys must encode equally; use the
     # coder's deterministic form and reject coders that have none.
diff --git a/sdks/python/apache_beam/io/watch_test.py b/sdks/python/apache_beam/io/watch_test.py
index 472177ceaee6..8c1f6571da66 100644
--- a/sdks/python/apache_beam/io/watch_test.py
+++ b/sdks/python/apache_beam/io/watch_test.py
@@ -18,7 +18,6 @@
 """Tests for the Watch transform."""
 
 import collections
-import typing
 import unittest
 
 import apache_beam as beam
@@ -445,24 +444,6 @@ def test_infers_output_coder_from_return_annotation(self):
           | Watch(_complete_poll, poll_interval=Duration(1)))
       self.assertEqual(typehints.Tuple[str, str], output.element_type)
 
-  def test_infers_coder_from_generic_annotations(self):
-    # tuple[str, float] and typing.Tuple[str, float] resolve to a tuple coder,
-    # not the pickling fallback.
-    def native_poll(element) -> PollResult[tuple[str, float]]:
-      return PollResult.complete([(element, 1.0)])
-
-    def typing_poll(
-        element) -> PollResult[typing.Tuple[str, float]]:  # noqa: UP006
-      return PollResult.complete([(element, 1.0)])
-
-    for poll in (native_poll, typing_poll):
-      with self._in_memory_pipeline() as p:
-        output = (
-            p | beam.Create(['k:']) | Watch(poll, poll_interval=Duration(1)))
-        self.assertEqual(
-            typehints.Tuple[str, typehints.Tuple[str, float]],
-            output.element_type)
-
   def test_uses_poll_fn_default_output_coder(self):
     with self._in_memory_pipeline() as p:
       output = (

From 8a132afb4495fc14570571322190bb5a16a23ea7 Mon Sep 17 00:00:00 2001
From: Eliaazzz <Elialiu760317@outlook.com>
Date: Tue, 11 Aug 2026 00:23:28 +1000
Subject: [PATCH 05/16] Add a timestamp_cursor option to MatchContinuously

Opt-in timestamp_cursor=True backs deduplication with the Watch
transform's cursor mode, so the restriction holds one timestamp rather
than an id per matched file. The poll stamps each match with its
last-modified time, which is what the cursor dedups on, and the
watermark stays at the poll time.

Those event times are floored to the millisecond, the resolution a
runner keeps for element timestamps. A cursor taken from finer mtimes is
persisted truncated and returns below the outputs it came from, matching
every file again on the next poll.

The class docstring no longer calls matching continuously stateful
without qualification, since the cursor bounds the state, and the
startup warning is skipped in that mode.
---
 sdks/python/apache_beam/io/fileio.py      | 132 ++++++++++++++++------
 sdks/python/apache_beam/io/fileio_test.py |  95 ++++++++++++++++
 2 files changed, 190 insertions(+), 37 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index 154732543e3d..fdb69486dd91 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -119,6 +119,7 @@
 from apache_beam.transforms.window import FixedWindows
 from apache_beam.transforms.window import GlobalWindow
 from apache_beam.transforms.window import IntervalWindow
+from apache_beam.transforms.window import TimestampedValue
 from apache_beam.utils.timestamp import MAX_TIMESTAMP
 from apache_beam.utils.timestamp import Duration
 from apache_beam.utils.timestamp import Timestamp
@@ -292,32 +293,51 @@ def state_coder(self):
     return VarIntCoder()
 
 
+def _mtime_of(metadata: filesystem.FileMetadata, option: str) -> float:
+  # A missing (zero) timestamp is rejected because every file would then carry
+  # the same one, and updates could never be told apart.
+  if not metadata.last_updated_in_seconds:
+    raise BeamIOError(
+        'MatchContinuously(%s=True) requires file last-modified times, but '
+        '%s reports none.' % (option, metadata.path))
+  return metadata.last_updated_in_seconds
+
+
+def _mtime_timestamp(metadata: filesystem.FileMetadata) -> Timestamp:
+  # Floored to the millisecond a runner keeps for element timestamps, so the
+  # cursor compares against the same resolution it is persisted at.
+  micros = Timestamp.of(_mtime_of(metadata, 'timestamp_cursor')).micros
+  return Timestamp(micros=micros - micros % 1000)
+
+
 def _file_path_key(metadata: filesystem.FileMetadata) -> str:
   return metadata.path
 
 
 def _file_path_and_mtime_key(
     metadata: filesystem.FileMetadata) -> tuple[str, float]:
-  # Keying on the last-modified time makes a changed file look new again. A
-  # missing (zero) timestamp is rejected because updates could never be seen.
-  if not metadata.last_updated_in_seconds:
-    raise BeamIOError(
-        'MatchContinuously(match_updated_files=True) requires file '
-        'last-modified times, but %s reports none.' % metadata.path)
-  return metadata.path, metadata.last_updated_in_seconds
+  # Keying on the last-modified time makes a changed file look new again.
+  return metadata.path, _mtime_of(metadata, 'match_updated_files')
 
 
 class _MatchContinuouslyPollFn(PollFn):
   """Polls a file pattern, honoring empty-match rules.
 
-  A poll before ``start_timestamp`` emits nothing. Matches carry the poll time
-  as their event time, and the watermark advances to the poll time so
-  event-time windows progress even when nothing new matches.
+  A poll before ``start_timestamp`` emits nothing. The watermark advances to
+  the poll time so event-time windows progress even when nothing new matches.
+  Matches carry the poll time as their event time, or their last-modified time
+  under ``mtime_timestamps``, which is what the timestamp cursor dedups on.
   """
-  def __init__(self, empty_match_treatment, start_timestamp, clock=None):
+  def __init__(
+      self,
+      empty_match_treatment,
+      start_timestamp,
+      clock=None,
+      mtime_timestamps=False):
     self._empty_match_treatment = empty_match_treatment
     self._start_micros = Timestamp.of(start_timestamp).micros
     self._clock = clock if clock is not None else _PollClock()
+    self._mtime_timestamps = mtime_timestamps
 
   def __call__(self, file_pattern: str) -> PollResult[filesystem.FileMetadata]:
     now = Timestamp.now()
@@ -330,6 +350,12 @@ def __call__(self, file_pattern: str) -> PollResult[filesystem.FileMetadata]:
                                                   self._empty_match_treatment)):
       raise BeamIOError(
           'Empty match for pattern %s. Disallowed.' % file_pattern)
+    if self._mtime_timestamps:
+      outputs = [
+          TimestampedValue(metadata, _mtime_timestamp(metadata))
+          for metadata in match_result.metadata_list
+      ]
+      return PollResult.incomplete(outputs).with_watermark(now)
     return PollResult.incomplete(
         match_result.metadata_list, timestamp=now).with_watermark(now)
 
@@ -343,13 +369,18 @@ class MatchContinuously(beam.PTransform):
   MatchContinuously is experimental.  No backwards-compatibility
   guarantees.
 
-  Matching continuously scales poorly, as it is stateful, and requires storing
-  file ids for every file the pattern has matched. With ``has_deduplication``
-  enabled those ids are kept in the splittable DoFn restriction, so a runner
-  with checkpointing enabled restores them after a restart and does not
-  reprocess files. Consider an alternate technique, such as Pub/Sub
-  Notifications (https://cloud.google.com/storage/docs/pubsub-notifications)
-  when using GCS if possible.
+  Deduplication state lives in the splittable DoFn restriction, so a runner
+  with checkpointing enabled restores it after a restart and does not
+  reprocess files. That state holds one id per matched file and grows with the
+  directory, unless ``timestamp_cursor`` bounds it to a single timestamp. For
+  a growing directory on GCS, consider an alternate technique such as Pub/Sub
+  Notifications
+  (https://cloud.google.com/storage/docs/pubsub-notifications).
+
+  A match carries the poll time as its event time, or its last-modified time
+  under ``timestamp_cursor``. The watermark is the poll time either way, so a
+  file whose last-modified time lags its appearance in the listing is late for
+  event-time windows downstream.
   """
   def __init__(
       self,
@@ -360,7 +391,8 @@ def __init__(
       stop_timestamp=MAX_TIMESTAMP,
       match_updated_files=False,
       apply_windowing=False,
-      empty_match_treatment=EmptyMatchTreatment.ALLOW):
+      empty_match_treatment=EmptyMatchTreatment.ALLOW,
+      timestamp_cursor=False):
     """Initializes a MatchContinuously transform.
 
     Args:
@@ -373,6 +405,17 @@ def __init__(
         file with timestamp changes.
       apply_windowing: Whether each element should be assigned to
         individual window. If false, all elements will reside in global window.
+      timestamp_cursor: (When has_deduplication is set to True) dedup by
+        last-modified time instead of by file id, which bounds the state to a
+        single timestamp. Each poll emits only the files modified past the
+        newest one already emitted, compared at the millisecond resolution a
+        runner keeps for element timestamps. A file appearing with a
+        last-modified time at or below that mark is skipped, as happens with
+        copies that preserve the source time, backfills of older files, and
+        files written within the same millisecond as an earlier poll's newest
+        match. A file whose last-modified time advances is emitted again,
+        which makes match_updated_files redundant. Requires the filesystem to
+        report last-modified times.
     """
 
     self.file_pattern = file_pattern
@@ -383,11 +426,17 @@ def __init__(
     self.match_upd = match_updated_files
     self.apply_windowing = apply_windowing
     self.empty_match_treatment = empty_match_treatment
-    _LOGGER.warning(
-        'Matching Continuously is stateful, and can scale poorly. '
-        'Consider using Pub/Sub Notifications '
-        '(https://cloud.google.com/storage/docs/pubsub-notifications) '
-        'if possible')
+    self.timestamp_cursor = timestamp_cursor
+    if timestamp_cursor and not has_deduplication:
+      raise ValueError(
+          'MatchContinuously(timestamp_cursor=True) deduplicates, so it '
+          'requires has_deduplication=True.')
+    if not timestamp_cursor:
+      _LOGGER.warning(
+          'Matching Continuously is stateful, and can scale poorly. '
+          'Consider using Pub/Sub Notifications '
+          '(https://cloud.google.com/storage/docs/pubsub-notifications) '
+          'if possible')
 
   def expand(self, pbegin) -> beam.PCollection[filesystem.FileMetadata]:
     if Duration.of(self.interval).micros <= 0:
@@ -405,8 +454,9 @@ def expand(self, pbegin) -> beam.PCollection[filesystem.FileMetadata]:
 
   def _match_deduplicated(self,
                           pbegin) -> beam.PCollection[filesystem.FileMetadata]:
-    # Watch emits each file once per key: the path, joined by the mtime when
-    # matching updated files; stop_timestamp bounds the polls to [start, stop).
+    # Watch emits each file once per dedup key: the path, joined by the mtime
+    # when matching updated files, or the mtime alone under timestamp_cursor.
+    # stop_timestamp bounds the polls to [start, stop).
     clock = _PollClock()
     if self.stop_ts == MAX_TIMESTAMP:
       termination = never()
@@ -427,18 +477,26 @@ def _match_deduplicated(self,
         # the output empty without Watch's unconditional first poll.
         return self._match_all_each_poll(pbegin)
       termination = _WatchWindowTermination(clock, start_ts.micros, max_polls)
-    if self.match_upd:
-      output_key_fn = _file_path_and_mtime_key
-    else:
-      output_key_fn = _file_path_key
     poll_fn = _MatchContinuouslyPollFn(
-        self.empty_match_treatment, self.start_ts, clock)
-    # The key coder is inferred from the key function's return annotation.
-    watch = Watch(
-        poll_fn,
-        poll_interval=self.interval,
-        termination=termination,
-        output_key_fn=output_key_fn)
+        self.empty_match_treatment,
+        self.start_ts,
+        clock,
+        mtime_timestamps=self.timestamp_cursor)
+    if self.timestamp_cursor:
+      # The cursor dedups on the mtime the poll stamps, so there is no key.
+      watch = Watch(
+          poll_fn,
+          poll_interval=self.interval,
+          termination=termination,
+          timestamp_cursor=True)
+    else:
+      # The key coder is inferred from the key function's return annotation.
+      watch = Watch(
+          poll_fn,
+          poll_interval=self.interval,
+          termination=termination,
+          output_key_fn=(
+              _file_path_and_mtime_key if self.match_upd else _file_path_key))
     # Watch emits (pattern, file) pairs; keep the FileMetadata output type so
     # downstream transforms stay typed instead of falling back to Any.
     return (
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index f5562e0202a0..99c236680282 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -534,6 +534,101 @@ def test_poll_fn_advances_watermark_on_empty_match(self):
     self.assertEqual((), result.outputs)
     self.assertIsNotNone(result.watermark)
 
+  def test_poll_fn_stamps_outputs_with_mtime_for_the_cursor(self):
+    # The cursor dedups on the event time, so a match carries its own mtime
+    # while the watermark stays at the poll time.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    path = self._create_temp_file(dir=tempdir)
+    os.utime(path, (1234.5, 1234.5))
+    poll_fn = fileio._MatchContinuouslyPollFn(
+        fileio.EmptyMatchTreatment.ALLOW,
+        Timestamp.now() - 3600,
+        mtime_timestamps=True)
+    result = poll_fn(FileSystems.join(tempdir, '*'))
+    self.assertEqual(1, len(result.outputs))
+    self.assertEqual(Timestamp.of(1234.5), result.outputs[0].timestamp)
+    self.assertLess(result.outputs[0].timestamp, result.watermark)
+
+  def test_mtime_timestamp_floors_to_the_millisecond(self):
+    # A runner keeps element timestamps to the millisecond, so a cursor
+    # carrying finer mtimes would come back below the outputs it was taken
+    # from and match them all over again.
+    metadata = FileMetadata('/tmp/a', 1, 1786371369.335245)
+    self.assertEqual(
+        Timestamp.of(1786371369.335), fileio._mtime_timestamp(metadata))
+
+  def test_timestamp_cursor_rejects_missing_mtime(self):
+    # Without mtimes every match would carry the same event time, so the
+    # cursor would drop everything after the first poll.
+    with self.assertRaises(BeamIOError):
+      fileio._mtime_of(FileMetadata('/tmp/a', 1), 'timestamp_cursor')
+
+  def test_timestamp_cursor_requires_deduplication(self):
+    with self.assertRaisesRegex(ValueError, 'has_deduplication=True'):
+      fileio.MatchContinuously(
+          file_pattern='/tmp/*', has_deduplication=False, timestamp_cursor=True)
+
+  def test_timestamp_cursor_emits_files_modified_past_the_cursor(self):
+    files = []
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+
+    # Create a file to be matched before pipeline
+    files.append(self._create_temp_file(dir=tempdir))
+    # Add file name that will be created mid-pipeline
+    files.append(FileSystems.join(tempdir, 'extra'))
+
+    interval = 0.2
+    start = Timestamp.now()
+    stop = start + interval + 0.1
+
+    def _create_extra_file(element):
+      writer = FileSystems.create(FileSystems.join(tempdir, 'extra'))
+      writer.close()
+      return element.path
+
+    with TestPipeline() as p:
+      match_continiously = (
+          p
+          | fileio.MatchContinuously(
+              file_pattern=FileSystems.join(tempdir, '*'),
+              interval=interval,
+              start_timestamp=start,
+              stop_timestamp=stop,
+              timestamp_cursor=True)
+          | beam.Map(_create_extra_file))
+
+      assert_that(match_continiously, equal_to(files))
+
+  def test_timestamp_cursor_skips_files_modified_before_the_cursor(self):
+    # A file that lands with an mtime older than one already emitted sits
+    # behind the cursor, so it never appears.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    first = self._create_temp_file(dir=tempdir)
+
+    interval = 0.2
+    start = Timestamp.now()
+    stop = start + interval + 0.1
+
+    def _create_backdated_file(element):
+      path = FileSystems.join(tempdir, 'backdated')
+      writer = FileSystems.create(path)
+      writer.close()
+      os.utime(path, (1234.5, 1234.5))
+      return element.path
+
+    with TestPipeline() as p:
+      match_continiously = (
+          p
+          | fileio.MatchContinuously(
+              file_pattern=FileSystems.join(tempdir, '*'),
+              interval=interval,
+              start_timestamp=start,
+              stop_timestamp=stop,
+              timestamp_cursor=True)
+          | beam.Map(_create_backdated_file))
+
+      assert_that(match_continiously, equal_to([first]))
+
 
 class WriteFilesTest(_TestCaseWithTempDirCleanUp):
 

From 462d2f564b6e57ce8dcf60f4b24f21961db41420 Mon Sep 17 00:00:00 2001
From: Eliaazzz <Elialiu760317@outlook.com>
Date: Wed, 12 Aug 2026 23:00:53 +1000
Subject: [PATCH 06/16] Keep the Watch restriction's microsecond precision

TimestampCoder encodes milliseconds, so a persisted cursor returned up to a
millisecond below the outputs it was taken from and the next poll handed them
out again. A replayed checkpoint primary lost the same precision and repeated
its emission at a different event time.

Both now encode microseconds at a fixed width, and a payload of any other
width is rejected rather than read as a smaller timestamp. The two envelope
tags whose payload changed are retired rather than reused, since a millisecond
payload is the width of a microsecond one and would otherwise decode to a
wrong timestamp.
---
 sdks/python/apache_beam/io/watch.py      | 43 +++++++++++++++++++++---
 sdks/python/apache_beam/io/watch_test.py | 42 +++++++++++++++++++++++
 2 files changed, 80 insertions(+), 5 deletions(-)

diff --git a/sdks/python/apache_beam/io/watch.py b/sdks/python/apache_beam/io/watch.py
index 40b7e451ce6e..2d9a4d7ba3a6 100644
--- a/sdks/python/apache_beam/io/watch.py
+++ b/sdks/python/apache_beam/io/watch.py
@@ -302,6 +302,33 @@ class _NonPollingGrowthState(_GrowthState):
 # ------------------------------------------------------------------------------
 
 
+class _MicrosTimestampCoder(Coder):
+  """Coder for a :class:`Timestamp` that keeps microseconds.
+
+  :class:`TimestampCoder` keeps only milliseconds. A cursor rounded down that
+  way comes back below the outputs it was taken from and the next poll hands
+  them out again, and a replayed output can land in a different window than
+  the emission it repeats. The width is fixed so a cursor state stays one size.
+  """
+  _WIDTH = 8
+
+  def encode(self, value: Timestamp) -> bytes:
+    return value.micros.to_bytes(
+        _MicrosTimestampCoder._WIDTH, 'big', signed=True)
+
+  def decode(self, encoded: bytes) -> Timestamp:
+    if len(encoded) != _MicrosTimestampCoder._WIDTH:
+      # A short payload would decode to a smaller timestamp, which as a cursor
+      # hands out outputs already emitted.
+      raise ValueError(
+          'Watch timestamp payload is %d bytes, expected %d.' %
+          (len(encoded), _MicrosTimestampCoder._WIDTH))
+    return Timestamp(micros=int.from_bytes(encoded, 'big', signed=True))
+
+  def is_deterministic(self) -> bool:
+    return True
+
+
 class _TimestampedValueCoder(Coder):
   """Coder for :class:`TimestampedValue`.
 
@@ -311,7 +338,7 @@ class _TimestampedValueCoder(Coder):
   :class:`TupleCoder` and rebuilds the ``TimestampedValue`` on decode.
   """
   def __init__(self, value_coder: Coder):
-    self._tuple_coder = TupleCoder([value_coder, TimestampCoder()])
+    self._tuple_coder = TupleCoder([value_coder, _MicrosTimestampCoder()])
 
   def encode(self, value: TimestampedValue) -> bytes:
     return self._tuple_coder.encode((value.value, value.timestamp))
@@ -325,10 +352,16 @@ def is_deterministic(self) -> bool:
 
 
 class _StateTag(enum.IntEnum):
-  """Envelope tag selecting the encoded restriction variant."""
+  """Envelope tag selecting the encoded restriction variant.
+
+  A tag is retired rather than reused when its payload format changes. Tags 1
+  and 2 held the millisecond timestamps this coder no longer writes, and their
+  payloads are the width of the microsecond ones, so a reused tag would decode
+  them to a wrong timestamp instead of failing.
+  """
   POLLING = 0
-  NON_POLLING = 1
-  CURSOR_POLLING = 2
+  NON_POLLING = 3
+  CURSOR_POLLING = 4
 
 
 class _GrowthStateCoder(Coder):
@@ -353,7 +386,7 @@ def __init__(self, output_coder: Coder, termination: TerminationCondition):
     ])
     self._cursor_polling_coder = TupleCoder([
         termination.state_coder(),
-        TimestampCoder(),
+        _MicrosTimestampCoder(),
     ])
     self._non_polling_coder = TupleCoder([
         nullable_ts,
diff --git a/sdks/python/apache_beam/io/watch_test.py b/sdks/python/apache_beam/io/watch_test.py
index a07f98bfa8d7..7cc9157abbfc 100644
--- a/sdks/python/apache_beam/io/watch_test.py
+++ b/sdks/python/apache_beam/io/watch_test.py
@@ -35,6 +35,7 @@
 from apache_beam.io.watch import Watch
 from apache_beam.io.watch import _GrowthRestrictionTracker
 from apache_beam.io.watch import _GrowthStateCoder
+from apache_beam.io.watch import _MicrosTimestampCoder
 from apache_beam.io.watch import _never_seen_before
 from apache_beam.io.watch import _NonPollingGrowthState
 from apache_beam.io.watch import _past_cursor
@@ -56,6 +57,7 @@
 from apache_beam.transforms.window import TimestampedValue
 from apache_beam.typehints import typehints
 from apache_beam.utils.timestamp import MAX_TIMESTAMP
+from apache_beam.utils.timestamp import MIN_TIMESTAMP
 from apache_beam.utils.timestamp import Duration
 from apache_beam.utils.timestamp import Timestamp
 
@@ -145,6 +147,19 @@ def test_polling_round_trip_preserves_cursor(self):
     self.assertEqual(0, len(decoded.completed))
     self.assertIsNone(decoded.poll_watermark)  # not part of the payload
 
+  def test_polling_round_trip_preserves_subsecond_cursors(self):
+    # A cursor rounded down to the millisecond, as TimestampCoder does, comes
+    # back below the outputs it was taken from and hands them out again.
+    coder = _GrowthStateCoder(StrUtf8Coder(), never())
+    termination_state = never().for_new_input(Timestamp(0), 'input')
+    for cursor in (Timestamp(micros=1786371369335245),
+                   MIN_TIMESTAMP,
+                   MAX_TIMESTAMP):
+      with self.subTest(cursor=cursor):
+        state = _PollingGrowthState(
+            collections.OrderedDict(), None, termination_state, cursor)
+        self.assertEqual(cursor, coder.decode(coder.encode(state)).cursor)
+
   def test_cursorless_state_keeps_the_pre_cursor_byte_format(self):
     # A polling state without a cursor must encode exactly as before the
     # cursor existed, so in-flight hash-mode restrictions decode across an
@@ -177,6 +192,33 @@ def test_non_polling_round_trip_preserves_pending_outputs(self):
     self.assertEqual([('a', Timestamp(1)), ('b', Timestamp(2))],
                      [(o.value, o.timestamp) for o in decoded.pending.outputs])
 
+  def test_truncated_timestamp_payload_fails_to_decode(self):
+    coder = _MicrosTimestampCoder()
+    encoded = coder.encode(Timestamp(micros=1786371369335245))
+    with self.assertRaisesRegex(ValueError, 'expected 8'):
+      coder.decode(encoded[:-1])
+
+  def test_retired_state_tag_fails_to_decode(self):
+    # The millisecond payloads the retired tags carried are the width of the
+    # microsecond ones, so a reused tag would decode them to a wrong timestamp.
+    coder = _GrowthStateCoder(StrUtf8Coder(), never())
+    for retired in (1, 2):
+      with self.subTest(tag=retired):
+        encoded = TupleCoder([VarIntCoder(), BytesCoder()]).encode(
+            (retired, TimestampCoder().encode(Timestamp(5))))
+        with self.assertRaisesRegex(ValueError, 'unknown Watch growth state'):
+          coder.decode(encoded)
+
+  def test_non_polling_round_trip_preserves_subsecond_pending_outputs(self):
+    # A replay repeats the emission it stands in for, so a rounded timestamp
+    # can land the replay in a different window than the original.
+    coder = _GrowthStateCoder(StrUtf8Coder(), never())
+    timestamp = Timestamp(micros=1000750)
+    state = _NonPollingGrowthState(
+        PollResult((TimestampedValue('a', timestamp), ), None))
+    decoded = coder.decode(coder.encode(state))
+    self.assertEqual(timestamp, decoded.pending.outputs[0].timestamp)
+
 
 class NeverSeenBeforeTest(unittest.TestCase):
   def test_dedups_and_sorts_by_timestamp(self):

From afa436c69ba17dbc982288a307bcfceac59bb12c Mon Sep 17 00:00:00 2001
From: Eliaazzz <Elialiu760317@outlook.com>
Date: Wed, 12 Aug 2026 23:00:54 +1000
Subject: [PATCH 07/16] Address review on the MatchContinuously timestamp
 cursor

The poll stamps a match with Timestamp.of(mtime) now that the restriction keeps
microseconds, so the millisecond flooring is gone.

The watermark under timestamp_cursor tracks the filesystem clock: the newest
last-modified time matched, capped at the poll time, and left where it is by a
poll that matches nothing. A clock behind the local one cannot make a later
file late, and one ahead cannot carry the watermark with it.

_mtime_of is renamed _ensure_mtime and no longer takes the option name. The
option's doc states the resolution it compares at and that it needs a fresh
pipeline, since an in-place update would seed the cursor with poll times the
old state recorded.
---
 sdks/python/apache_beam/io/fileio.py      | 78 +++++++++++++----------
 sdks/python/apache_beam/io/fileio_test.py | 59 +++++++++++++----
 2 files changed, 89 insertions(+), 48 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index fdb69486dd91..a3201109d89e 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -293,23 +293,16 @@ def state_coder(self):
     return VarIntCoder()
 
 
-def _mtime_of(metadata: filesystem.FileMetadata, option: str) -> float:
+def _ensure_mtime(metadata: filesystem.FileMetadata) -> float:
   # A missing (zero) timestamp is rejected because every file would then carry
   # the same one, and updates could never be told apart.
   if not metadata.last_updated_in_seconds:
     raise BeamIOError(
-        'MatchContinuously(%s=True) requires file last-modified times, but '
-        '%s reports none.' % (option, metadata.path))
+        'MatchContinuously deduplicates by last-modified time, but %s reports '
+        'none.' % metadata.path)
   return metadata.last_updated_in_seconds
 
 
-def _mtime_timestamp(metadata: filesystem.FileMetadata) -> Timestamp:
-  # Floored to the millisecond a runner keeps for element timestamps, so the
-  # cursor compares against the same resolution it is persisted at.
-  micros = Timestamp.of(_mtime_of(metadata, 'timestamp_cursor')).micros
-  return Timestamp(micros=micros - micros % 1000)
-
-
 def _file_path_key(metadata: filesystem.FileMetadata) -> str:
   return metadata.path
 
@@ -317,16 +310,19 @@ def _file_path_key(metadata: filesystem.FileMetadata) -> str:
 def _file_path_and_mtime_key(
     metadata: filesystem.FileMetadata) -> tuple[str, float]:
   # Keying on the last-modified time makes a changed file look new again.
-  return metadata.path, _mtime_of(metadata, 'match_updated_files')
+  return metadata.path, _ensure_mtime(metadata)
 
 
 class _MatchContinuouslyPollFn(PollFn):
   """Polls a file pattern, honoring empty-match rules.
 
-  A poll before ``start_timestamp`` emits nothing. The watermark advances to
-  the poll time so event-time windows progress even when nothing new matches.
-  Matches carry the poll time as their event time, or their last-modified time
-  under ``mtime_timestamps``, which is what the timestamp cursor dedups on.
+  A poll before ``start_timestamp`` emits nothing. Matches carry the poll time
+  as their event time, and the watermark advances to the poll time so
+  event-time windows progress even when nothing new matches. Under
+  ``mtime_timestamps`` a match carries its last-modified time instead, which
+  is what the timestamp cursor dedups on, and the watermark is the newest
+  last-modified time matched, capped at the poll time. A poll that matches
+  nothing leaves the watermark where it is.
   """
   def __init__(
       self,
@@ -350,14 +346,23 @@ def __call__(self, file_pattern: str) -> PollResult[filesystem.FileMetadata]:
                                                   self._empty_match_treatment)):
       raise BeamIOError(
           'Empty match for pattern %s. Disallowed.' % file_pattern)
-    if self._mtime_timestamps:
-      outputs = [
-          TimestampedValue(metadata, _mtime_timestamp(metadata))
-          for metadata in match_result.metadata_list
-      ]
-      return PollResult.incomplete(outputs).with_watermark(now)
-    return PollResult.incomplete(
-        match_result.metadata_list, timestamp=now).with_watermark(now)
+    if not self._mtime_timestamps:
+      return PollResult.incomplete(
+          match_result.metadata_list, timestamp=now).with_watermark(now)
+    outputs = [
+        TimestampedValue(metadata, Timestamp.of(_ensure_mtime(metadata)))
+        for metadata in match_result.metadata_list
+    ]
+    if not outputs:
+      # Matching nothing is no reading of the filesystem clock. Holding the
+      # watermark keeps the files that clock has yet to reach from arriving
+      # behind it.
+      return PollResult.incomplete(())
+    # The filesystem clock can run behind the local one, so the watermark stops
+    # at the newest last-modified time matched, and never runs past the poll
+    # time when that clock is ahead.
+    newest = max(output.timestamp for output in outputs)
+    return PollResult.incomplete(outputs).with_watermark(min(newest, now))
 
 
 class MatchContinuously(beam.PTransform):
@@ -377,10 +382,12 @@ class MatchContinuously(beam.PTransform):
   Notifications
   (https://cloud.google.com/storage/docs/pubsub-notifications).
 
-  A match carries the poll time as its event time, or its last-modified time
-  under ``timestamp_cursor``. The watermark is the poll time either way, so a
-  file whose last-modified time lags its appearance in the listing is late for
-  event-time windows downstream.
+  A match carries the poll time as its event time, and the watermark follows
+  the poll time. Under ``timestamp_cursor`` a match carries its last-modified
+  time and the watermark tracks the filesystem clock instead: the newest
+  last-modified time matched, capped at the poll time, and left where it is by
+  a poll that matches nothing. A clock behind the local one then cannot make a
+  later file late, and one ahead cannot carry the watermark with it.
   """
   def __init__(
       self,
@@ -408,14 +415,15 @@ def __init__(
       timestamp_cursor: (When has_deduplication is set to True) dedup by
         last-modified time instead of by file id, which bounds the state to a
         single timestamp. Each poll emits only the files modified past the
-        newest one already emitted, compared at the millisecond resolution a
-        runner keeps for element timestamps. A file appearing with a
-        last-modified time at or below that mark is skipped, as happens with
-        copies that preserve the source time, backfills of older files, and
-        files written within the same millisecond as an earlier poll's newest
-        match. A file whose last-modified time advances is emitted again,
-        which makes match_updated_files redundant. Requires the filesystem to
-        report last-modified times.
+        newest last-modified time already emitted, compared to the microsecond
+        a Timestamp holds. A file at or below that mark is skipped, as happens
+        with copies that preserve the source time and with backfills of older
+        files. A modified file is emitted again once its reported time passes
+        the mark, and match_updated_files has no effect in this mode. Requires the filesystem to report
+        last-modified times, and a pipeline started fresh: turning it on while
+        updating a running pipeline seeds the cursor with the poll times the
+        old state recorded, which are not last-modified times, so files are
+        skipped or repeated.
     """
 
     self.file_pattern = file_pattern
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index 99c236680282..1d5108181c5f 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -535,33 +535,66 @@ def test_poll_fn_advances_watermark_on_empty_match(self):
     self.assertIsNotNone(result.watermark)
 
   def test_poll_fn_stamps_outputs_with_mtime_for_the_cursor(self):
-    # The cursor dedups on the event time, so a match carries its own mtime
-    # while the watermark stays at the poll time.
+    # The cursor dedups on the event time, so a match carries its own mtime.
+    # Sub-millisecond digits are kept, or a cursor taken from them would come
+    # back below the outputs it was taken from and match them all over again.
     tempdir = '%s%s' % (self._new_tempdir(), os.sep)
     path = self._create_temp_file(dir=tempdir)
-    os.utime(path, (1234.5, 1234.5))
+    os.utime(path, (1234.567891, 1234.567891))
     poll_fn = fileio._MatchContinuouslyPollFn(
         fileio.EmptyMatchTreatment.ALLOW,
         Timestamp.now() - 3600,
         mtime_timestamps=True)
     result = poll_fn(FileSystems.join(tempdir, '*'))
     self.assertEqual(1, len(result.outputs))
-    self.assertEqual(Timestamp.of(1234.5), result.outputs[0].timestamp)
-    self.assertLess(result.outputs[0].timestamp, result.watermark)
-
-  def test_mtime_timestamp_floors_to_the_millisecond(self):
-    # A runner keeps element timestamps to the millisecond, so a cursor
-    # carrying finer mtimes would come back below the outputs it was taken
-    # from and match them all over again.
-    metadata = FileMetadata('/tmp/a', 1, 1786371369.335245)
     self.assertEqual(
-        Timestamp.of(1786371369.335), fileio._mtime_timestamp(metadata))
+        Timestamp.of(os.path.getmtime(path)), result.outputs[0].timestamp)
+
+  def test_poll_fn_holds_the_mtime_watermark_at_the_newest_match(self):
+    # The filesystem clock can run behind the local one, so a watermark at the
+    # poll time would make the files it has yet to hand out late.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    os.utime(self._create_temp_file(dir=tempdir), (1234.5, 1234.5))
+    os.utime(self._create_temp_file(dir=tempdir), (2345.5, 2345.5))
+    poll_fn = fileio._MatchContinuouslyPollFn(
+        fileio.EmptyMatchTreatment.ALLOW,
+        Timestamp.now() - 3600,
+        mtime_timestamps=True)
+    result = poll_fn(FileSystems.join(tempdir, '*'))
+    self.assertEqual(Timestamp.of(2345.5), result.watermark)
+
+  def test_poll_fn_caps_the_mtime_watermark_at_the_poll_time(self):
+    # A filesystem clock running ahead must not carry the watermark with it,
+    # which pins the watermark to the poll time.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    path = self._create_temp_file(dir=tempdir)
+    ahead = Timestamp.now() + 3600
+    os.utime(path, (float(ahead), float(ahead)))
+    poll_fn = fileio._MatchContinuouslyPollFn(
+        fileio.EmptyMatchTreatment.ALLOW,
+        Timestamp.now() - 3600,
+        mtime_timestamps=True)
+    before = Timestamp.now()
+    result = poll_fn(FileSystems.join(tempdir, '*'))
+    self.assertTrue(before <= result.watermark <= Timestamp.now())
+
+  def test_poll_fn_holds_the_mtime_watermark_on_empty_match(self):
+    # No match is no reading of the filesystem clock. A watermark at the poll
+    # time would leave the files that clock has yet to reach behind it.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    poll_fn = fileio._MatchContinuouslyPollFn(
+        fileio.EmptyMatchTreatment.ALLOW,
+        Timestamp.now() - 3600,
+        mtime_timestamps=True)
+    result = poll_fn(FileSystems.join(tempdir, '*'))
+    self.assertEqual((), result.outputs)
+    self.assertIsNone(result.watermark)
 
   def test_timestamp_cursor_rejects_missing_mtime(self):
     # Without mtimes every match would carry the same event time, so the
     # cursor would drop everything after the first poll.
     with self.assertRaises(BeamIOError):
-      fileio._mtime_of(FileMetadata('/tmp/a', 1), 'timestamp_cursor')
+      fileio._ensure_mtime(FileMetadata('/tmp/a', 1))
 
   def test_timestamp_cursor_requires_deduplication(self):
     with self.assertRaisesRegex(ValueError, 'has_deduplication=True'):

From f68f51ea83e923bd762ee049f3b4286919dcd785 Mon Sep 17 00:00:00 2001
From: Eliaazzz <Elialiu760317@outlook.com>
Date: Thu, 13 Aug 2026 00:04:43 +1000
Subject: [PATCH 08/16] Rewrap the timestamp_cursor doc paragraph

---
 sdks/python/apache_beam/io/fileio.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index a3201109d89e..35614c7a7a28 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -419,11 +419,11 @@ def __init__(
         a Timestamp holds. A file at or below that mark is skipped, as happens
         with copies that preserve the source time and with backfills of older
         files. A modified file is emitted again once its reported time passes
-        the mark, and match_updated_files has no effect in this mode. Requires the filesystem to report
-        last-modified times, and a pipeline started fresh: turning it on while
-        updating a running pipeline seeds the cursor with the poll times the
-        old state recorded, which are not last-modified times, so files are
-        skipped or repeated.
+        the mark, and match_updated_files has no effect in this mode. Requires
+        the filesystem to report last-modified times, and a pipeline started
+        fresh: turning it on while updating a running pipeline seeds the cursor
+        with the poll times the old state recorded, which are not last-modified
+        times, so files are skipped or repeated.
     """
 
     self.file_pattern = file_pattern

From 9df671359022e6905862e8f6b7caae34da0dd174 Mon Sep 17 00:00:00 2001
From: Eliaazzz <Elialiu760317@outlook.com>
Date: Thu, 13 Aug 2026 12:09:27 +1000
Subject: [PATCH 09/16] Retire keys with the Watch cursor instead of replacing
 them

A cursor alone cannot tell two outputs at one event time apart: comparing
strictly drops the second, comparing loosely repeats both. Rounding only moves
where that tie falls, so the microsecond coder goes with it.

Dedup goes back to hashing the output key, and the cursor now bounds that state
rather than standing in for it. A key is retired once the greatest emitted
event time has moved allowed_lateness past it, so the restriction holds a
trailing window instead of every key ever seen. allowed_lateness defaults to
zero and widens the window for a source whose outputs arrive out of order.

The cursor is orthogonal to the key, so it no longer conflicts with
output_key_fn, and a restriction resumed in cursor mode keeps the hashes its
hash rounds recorded rather than seeding a cursor from them.
---
 sdks/python/apache_beam/io/watch.py      | 309 +++++++++++------------
 sdks/python/apache_beam/io/watch_test.py | 236 +++++++++--------
 2 files changed, 261 insertions(+), 284 deletions(-)

diff --git a/sdks/python/apache_beam/io/watch.py b/sdks/python/apache_beam/io/watch.py
index 2d9a4d7ba3a6..827edde7efd8 100644
--- a/sdks/python/apache_beam/io/watch.py
+++ b/sdks/python/apache_beam/io/watch.py
@@ -35,11 +35,10 @@
 hash equally across workers and restarts.
 
 By default, the Watch transform internally stores the hash of all items
-seen. If the incremental items returned by the poll function guarantee
-monotonic timestamp growth (new items on the next poll have timestamps
-larger than the largest of the previous poll), consider setting
-``timestamp_cursor=True`` for better performance, as it replaces the hash
-dedup with an O(1) event-time cursor; see :class:`Watch`.
+seen. If the items returned by the poll function arrive in roughly
+non-decreasing event time, consider setting ``timestamp_cursor=True``, which
+retires a hash once the greatest emitted event time has moved past it and so
+holds a trailing window instead of every item ever seen; see :class:`Watch`.
 
 Example::
 
@@ -274,9 +273,9 @@ class _PollingGrowthState(_GrowthState):
   """Keep-polling state: dedup state, watermark, termination state.
 
   ``completed`` maps a 16-byte output-key hash to the event time it was first
-  seen; it is insertion-ordered and treated as immutable. In timestamp-cursor
-  mode ``completed`` is empty and ``cursor`` is the greatest emitted event
-  time.
+  seen; it is insertion-ordered and treated as immutable. ``cursor`` is the
+  greatest emitted event time, set only in timestamp-cursor mode, where it
+  retires the keys it has moved past and bounds ``completed``.
   """
   completed: 'collections.OrderedDict[bytes, Timestamp]'
   poll_watermark: Optional[Timestamp]
@@ -302,33 +301,6 @@ class _NonPollingGrowthState(_GrowthState):
 # ------------------------------------------------------------------------------
 
 
-class _MicrosTimestampCoder(Coder):
-  """Coder for a :class:`Timestamp` that keeps microseconds.
-
-  :class:`TimestampCoder` keeps only milliseconds. A cursor rounded down that
-  way comes back below the outputs it was taken from and the next poll hands
-  them out again, and a replayed output can land in a different window than
-  the emission it repeats. The width is fixed so a cursor state stays one size.
-  """
-  _WIDTH = 8
-
-  def encode(self, value: Timestamp) -> bytes:
-    return value.micros.to_bytes(
-        _MicrosTimestampCoder._WIDTH, 'big', signed=True)
-
-  def decode(self, encoded: bytes) -> Timestamp:
-    if len(encoded) != _MicrosTimestampCoder._WIDTH:
-      # A short payload would decode to a smaller timestamp, which as a cursor
-      # hands out outputs already emitted.
-      raise ValueError(
-          'Watch timestamp payload is %d bytes, expected %d.' %
-          (len(encoded), _MicrosTimestampCoder._WIDTH))
-    return Timestamp(micros=int.from_bytes(encoded, 'big', signed=True))
-
-  def is_deterministic(self) -> bool:
-    return True
-
-
 class _TimestampedValueCoder(Coder):
   """Coder for :class:`TimestampedValue`.
 
@@ -338,7 +310,7 @@ class _TimestampedValueCoder(Coder):
   :class:`TupleCoder` and rebuilds the ``TimestampedValue`` on decode.
   """
   def __init__(self, value_coder: Coder):
-    self._tuple_coder = TupleCoder([value_coder, _MicrosTimestampCoder()])
+    self._tuple_coder = TupleCoder([value_coder, TimestampCoder()])
 
   def encode(self, value: TimestampedValue) -> bytes:
     return self._tuple_coder.encode((value.value, value.timestamp))
@@ -352,16 +324,10 @@ def is_deterministic(self) -> bool:
 
 
 class _StateTag(enum.IntEnum):
-  """Envelope tag selecting the encoded restriction variant.
-
-  A tag is retired rather than reused when its payload format changes. Tags 1
-  and 2 held the millisecond timestamps this coder no longer writes, and their
-  payloads are the width of the microsecond ones, so a reused tag would decode
-  them to a wrong timestamp instead of failing.
-  """
+  """Envelope tag selecting the encoded restriction variant."""
   POLLING = 0
-  NON_POLLING = 3
-  CURSOR_POLLING = 4
+  NON_POLLING = 1
+  CURSOR_POLLING = 2
 
 
 class _GrowthStateCoder(Coder):
@@ -370,23 +336,26 @@ class _GrowthStateCoder(Coder):
   A ``(tag, payload)`` envelope selects the variant; the payload is a
   variant-specific :class:`TupleCoder`. ``completed`` is encoded as an ordered
   list of ``(hash, timestamp)`` pairs so insertion order survives a round
-  trip. A cursor state encodes only its termination state and cursor; the
-  watermark is restored from the estimator state the runner persists. Hash
-  states keep the pre-cursor byte format. This format is internal to the
-  Python SDK.
+  trip. A cursor state adds the cursor to that payload; the keys it carries
+  are only those the cursor has yet to retire. States without a cursor keep
+  the pre-cursor byte format. This format is internal to the Python SDK.
   """
   def __init__(self, output_coder: Coder, termination: TerminationCondition):
     nullable_ts = NullableCoder(TimestampCoder())
+    completed_coder = coders.ListCoder(
+        TupleCoder([coders.BytesCoder(), TimestampCoder()]))
     self._envelope_coder = TupleCoder(
         [coders.VarIntCoder(), coders.BytesCoder()])
     self._polling_coder = TupleCoder([
         termination.state_coder(),
         nullable_ts,
-        coders.ListCoder(TupleCoder([coders.BytesCoder(), TimestampCoder()])),
+        completed_coder,
     ])
     self._cursor_polling_coder = TupleCoder([
         termination.state_coder(),
-        _MicrosTimestampCoder(),
+        nullable_ts,
+        completed_coder,
+        TimestampCoder(),
     ])
     self._non_polling_coder = TupleCoder([
         nullable_ts,
@@ -401,8 +370,11 @@ def encode(self, state: _GrowthState) -> bytes:
             state.poll_watermark,
             list(state.completed.items())))
         return self._envelope_coder.encode((_StateTag.POLLING, payload))
-      payload = self._cursor_polling_coder.encode(
-          (state.termination_state, state.cursor))
+      payload = self._cursor_polling_coder.encode((
+          state.termination_state,
+          state.poll_watermark,
+          list(state.completed.items()),
+          state.cursor))
       return self._envelope_coder.encode((_StateTag.CURSOR_POLLING, payload))
     payload = self._non_polling_coder.encode(
         (state.pending.watermark, list(state.pending.outputs)))
@@ -419,9 +391,13 @@ def decode(self, encoded: bytes) -> _GrowthState:
       watermark, outputs = self._non_polling_coder.decode(payload)
       return _NonPollingGrowthState(PollResult(tuple(outputs), watermark))
     if tag == _StateTag.CURSOR_POLLING:
-      termination_state, cursor = self._cursor_polling_coder.decode(payload)
+      termination_state, poll_watermark, items, cursor = (
+          self._cursor_polling_coder.decode(payload))
       return _PollingGrowthState(
-          collections.OrderedDict(), None, termination_state, cursor)
+          collections.OrderedDict(items),
+          poll_watermark,
+          termination_state,
+          cursor)
     raise ValueError('unknown Watch growth state tag: %r' % (tag, ))
 
   def is_deterministic(self) -> bool:
@@ -451,20 +427,39 @@ def _max_watermark(left: Optional[Timestamp],
   return max(left, right)
 
 
+def _retention_floor(
+    restriction: _PollingGrowthState,
+    allowed_lateness: Duration) -> Optional[Timestamp]:
+  """The event time below which a key is retired from ``completed``.
+
+  ``None`` leaves every key retained, which is hash dedup with unbounded
+  state. Otherwise an output at or above the floor is still deduped by key,
+  and one below it is taken as already seen.
+  """
+  if restriction.cursor is None:
+    return None
+  return restriction.cursor - allowed_lateness
+
+
 def _never_seen_before(
     restriction: _PollingGrowthState,
     result: PollResult,
     key_fn: Callable[[Any], Any],
-    key_coder: Coder) -> PollResult:
+    key_coder: Coder,
+    floor: Optional[Timestamp] = None) -> PollResult:
   """Filters a poll result down to outputs whose key was never seen before.
 
   Dedup hashes ``key_fn(output.value)`` against the restriction's completed
-  set, also dropping in-round duplicates. Outputs are sorted by timestamp so
-  the earliest one can serve as the inferred watermark.
+  set, also dropping in-round duplicates. An output below ``floor`` is dropped
+  without consulting the set, since the key that would prove it seen has been
+  retired. Outputs are sorted by timestamp so the earliest one can serve as
+  the inferred watermark.
   """
   new_outputs = []
   seen_this_round = set()
   for output in result.outputs:
+    if floor is not None and output.timestamp < floor:
+      continue
     key_hash = _hash_output(key_coder, key_fn(output.value))
     if key_hash in restriction.completed or key_hash in seen_this_round:
       continue
@@ -474,28 +469,18 @@ def _never_seen_before(
   return dataclasses.replace(result, outputs=tuple(new_outputs))
 
 
-def _cursor_of(restriction: _PollingGrowthState) -> Optional[Timestamp]:
-  """The dedup cursor: the stored one, or for a restriction switched over
-  from hash dedup, the greatest event time its hash map recorded."""
-  if restriction.cursor is not None:
-    return restriction.cursor
-  if restriction.completed:
-    return max(restriction.completed.values())
-  return None
-
-
-def _past_cursor(
-    restriction: _PollingGrowthState, result: PollResult) -> PollResult:
-  """Filters a poll result down to outputs strictly past the cursor, sorted
-  by timestamp so the earliest infers the watermark and the latest advances
-  the cursor."""
-  cursor = _cursor_of(restriction)
-  new_outputs = [
-      output for output in result.outputs
-      if cursor is None or output.timestamp > cursor
-  ]
-  new_outputs.sort(key=lambda output: output.timestamp)
-  return dataclasses.replace(result, outputs=tuple(new_outputs))
+def _retained(
+    completed: 'collections.OrderedDict[bytes, Timestamp]',
+    floor: Optional[Timestamp]) -> 'collections.OrderedDict[bytes, Timestamp]':
+  """Drops the keys the floor has retired, which bounds the state."""
+  if floor is None:
+    return completed
+  retained = collections.OrderedDict(
+      (key_hash, timestamp) for key_hash, timestamp in completed.items()
+      if timestamp >= floor)
+  # Reuse the parent map when the floor retired nothing, so a round that adds
+  # no key leaves the state object untouched.
+  return completed if len(retained) == len(completed) else retained
 
 
 class _GrowthRestrictionTracker(iobase.RestrictionTracker):
@@ -512,11 +497,13 @@ def __init__(
       restriction: _GrowthState,
       key_fn: Callable[[Any], Any],
       key_coder: Coder,
-      timestamp_cursor: bool = False):
+      timestamp_cursor: bool = False,
+      allowed_lateness: Duration = Duration(0)):
     self._restriction = restriction
     self._key_fn = key_fn
     self._key_coder = key_coder
     self._timestamp_cursor = timestamp_cursor
+    self._allowed_lateness = allowed_lateness
     self._claimed_result = None  # type: Optional[PollResult]
     self._claimed_termination_state = None  # type: Any
     self._claimed_hashes = None  # type: Optional[collections.OrderedDict]
@@ -525,6 +512,12 @@ def __init__(
   def _hash(self, value: Any) -> bytes:
     return _hash_output(self._key_coder, self._key_fn(value))
 
+  def _floor(self) -> Optional[Timestamp]:
+    if not self._timestamp_cursor or not isinstance(self._restriction,
+                                                    _PollingGrowthState):
+      return None
+    return _retention_floor(self._restriction, self._allowed_lateness)
+
   def current_restriction(self) -> _GrowthState:
     return self._restriction
 
@@ -538,35 +531,23 @@ def try_claim(self, position: tuple[PollResult, Any]) -> bool:
     if self._should_stop:
       return False
     result, termination_state = position
-    claimed_hashes = None
-    if self._timestamp_cursor:
-      # Cursor mode validates by timestamps and never hashes.
-      if isinstance(self._restriction, _PollingGrowthState):
-        cursor = _cursor_of(self._restriction)
-        if cursor is not None and any(output.timestamp <= cursor
-                                      for output in result.outputs):
-          return False
-      else:
-        # Values may lack stable equality without a deterministic coder, so a
-        # replay is identified by its timestamps.
-        expected = sorted(
-            output.timestamp for output in self._restriction.pending.outputs)
-        if expected != sorted(output.timestamp for output in result.outputs):
-          return False
+    claimed_hashes = collections.OrderedDict()
+    for output in result.outputs:
+      claimed_hashes[self._hash(output.value)] = output.timestamp
+    if isinstance(self._restriction, _PollingGrowthState):
+      if any(key_hash in self._restriction.completed
+             for key_hash in claimed_hashes):
+        return False
+      floor = self._floor()
+      if floor is not None and any(output.timestamp < floor
+                                   for output in result.outputs):
+        return False
     else:
-      claimed_hashes = collections.OrderedDict()
-      for output in result.outputs:
-        claimed_hashes[self._hash(output.value)] = output.timestamp
-      if isinstance(self._restriction, _PollingGrowthState):
-        if any(key_hash in self._restriction.completed
-               for key_hash in claimed_hashes):
-          return False
-      else:
-        expected = set(
-            self._hash(output.value)
-            for output in self._restriction.pending.outputs)
-        if expected != set(claimed_hashes):
-          return False
+      expected = set(
+          self._hash(output.value)
+          for output in self._restriction.pending.outputs)
+      if expected != set(claimed_hashes):
+        return False
     self._should_stop = True
     self._claimed_result = result
     self._claimed_termination_state = termination_state
@@ -587,24 +568,21 @@ def try_split(self, fraction_of_remainder):
     else:
       # The primary becomes a replay of the claimed round; the residual
       # resumes polling with the claimed round folded into the dedup state.
-      # A state holds hashes or a cursor, never both, so each mode drops the
-      # other mode's leftovers after a switch.
-      if self._timestamp_cursor:
-        completed = self._restriction.completed
-        if completed:
-          completed = collections.OrderedDict()
-        if self._claimed_result.outputs:
-          cursor = self._claimed_result.outputs[-1].timestamp
-        else:
-          cursor = _cursor_of(self._restriction)
-      elif self._claimed_hashes:
+      if self._claimed_hashes:
         completed = collections.OrderedDict(self._restriction.completed)
         completed.update(self._claimed_hashes)
-        cursor = None
       else:
         # An idle round reuses the parent map so empty polls stay O(1).
         completed = self._restriction.completed
-        cursor = None
+      cursor = None
+      if self._timestamp_cursor:
+        # The cursor only ever advances, and retires the keys it moves past.
+        cursor = _max_watermark(
+            self._restriction.cursor,
+            max((output.timestamp for output in self._claimed_result.outputs),
+                default=None))
+        if cursor is not None:
+          completed = _retained(completed, cursor - self._allowed_lateness)
       residual = _PollingGrowthState(
           completed,
           _max_watermark(
@@ -656,6 +634,7 @@ def __init__(
       key_fn: Callable[[Any], Any],
       key_coder: Coder,
       timestamp_cursor: bool = False,
+      allowed_lateness: Duration = Duration(0),
       now_fn: Optional[Callable[[], float]] = None):
     self._poll_fn = poll_fn
     self._termination = termination
@@ -664,6 +643,7 @@ def __init__(
     self._key_fn = key_fn
     self._key_coder = key_coder
     self._timestamp_cursor = timestamp_cursor
+    self._allowed_lateness = allowed_lateness
     self._now = now_fn or time.time
     self._restriction_coder = _GrowthStateCoder(output_coder, termination)
     # Count of late emissions seen on this worker, for throttled warnings.
@@ -678,7 +658,11 @@ def initial_restriction(self, element) -> _PollingGrowthState:
 
   def create_tracker(self, restriction) -> _GrowthRestrictionTracker:
     return _GrowthRestrictionTracker(
-        restriction, self._key_fn, self._key_coder, self._timestamp_cursor)
+        restriction,
+        self._key_fn,
+        self._key_coder,
+        self._timestamp_cursor,
+        self._allowed_lateness)
 
   def restriction_coder(self) -> Coder:
     return self._restriction_coder
@@ -718,11 +702,11 @@ def process(
     result = self._poll_fn(element)
     # Read the clock after the poll so a slow poll counts against termination.
     now = Timestamp.of(self._now())
+    floor = None
     if self._timestamp_cursor:
-      new_results = _past_cursor(restriction, result)
-    else:
-      new_results = _never_seen_before(
-          restriction, result, self._key_fn, self._key_coder)
+      floor = _retention_floor(restriction, self._allowed_lateness)
+    new_results = _never_seen_before(
+        restriction, result, self._key_fn, self._key_coder, floor)
     termination_state = restriction.termination_state
     if new_results.outputs:
       termination_state = self._termination.on_seen_new_output(
@@ -749,9 +733,10 @@ def process(
     else:
       watermark = None
     if self._timestamp_cursor:
-      new_cursor = (
-          new_results.outputs[-1].timestamp
-          if new_results.outputs else restriction.cursor)
+      # Outputs are timestamp-sorted, so the last one is the greatest.
+      new_cursor = _max_watermark(
+          restriction.cursor,
+          new_results.outputs[-1].timestamp if new_results.outputs else None)
       if new_cursor is not None and new_cursor >= MAX_TIMESTAMP:
         # A cursor at MAX is terminal; polling on would only drop outputs.
         return
@@ -845,14 +830,17 @@ class Watch(PTransform):
       inferred like ``output_coder`` when omitted. It is converted with
       ``as_deterministic_coder`` so equal keys always hash equally; a coder
       with no deterministic form is rejected.
-    timestamp_cursor: dedup by event time instead of by key. Each round emits
-      only outputs strictly past the greatest event time already emitted, so
-      the per-input state is a single timestamp. Requires every new output to
-      carry an event time strictly greater than all previously emitted ones;
-      re-listed old outputs at or below the cursor are dropped as already
-      seen. For sources whose new outputs can arrive at or below the cursor,
-      keep the default hash dedup. Incompatible with ``output_key_fn`` and
-      ``output_key_coder``.
+    timestamp_cursor: bound the dedup state by event time. Dedup still goes by
+      key, but a key is retired once the greatest emitted event time has moved
+      more than ``allowed_lateness`` past it, so the state holds a trailing
+      window rather than every key ever seen. An output below that mark is
+      taken as already seen and dropped, so this suits sources whose outputs
+      arrive in roughly non-decreasing event time; keep the default for
+      sources that can hand out much older outputs at any time.
+    allowed_lateness: how far below the cursor a key is still retained, as a
+      :class:`Duration` or in seconds. Widen it for a source whose outputs
+      arrive out of order, at the cost of a larger state. Ignored unless
+      ``timestamp_cursor`` is set; defaults to zero.
     now_fn: clock used for termination decisions; tests can inject one.
   """
   def __init__(
@@ -864,15 +852,16 @@ def __init__(
       output_key_fn: Optional[Callable[[Any], Any]] = None,
       output_key_coder: Optional[Coder] = None,
       timestamp_cursor: bool = False,
+      allowed_lateness=0,
       now_fn: Optional[Callable[[], float]] = None):
     super().__init__()
     if poll_interval is None:
       raise ValueError('Watch requires a poll_interval')
-    if timestamp_cursor and (output_key_fn is not None or
-                             output_key_coder is not None):
+    allowed_lateness = _as_duration(allowed_lateness)
+    if allowed_lateness < Duration(0):
       raise ValueError(
-          'timestamp_cursor dedups by event time, not by key; do not pass '
-          'output_key_fn or output_key_coder with timestamp_cursor=True.')
+          'Watch allowed_lateness must not be negative, got %s' %
+          allowed_lateness)
     self._poll_fn = poll_fn
     self._poll_interval = _as_duration(poll_interval)
     self._termination = termination or never()
@@ -880,6 +869,7 @@ def __init__(
     self._output_key_fn = output_key_fn
     self._output_key_coder = output_key_coder
     self._timestamp_cursor = timestamp_cursor
+    self._allowed_lateness = allowed_lateness
     self._now = now_fn
 
   def expand(self, pcoll):
@@ -888,28 +878,22 @@ def expand(self, pcoll):
       output_coder = self._poll_fn.default_output_coder()
     if output_coder is None:
       output_coder = _coder_for_hint(_poll_output_type(self._poll_fn))
-    if self._timestamp_cursor:
-      # Cursor dedup never hashes, so no deterministic key coder is needed.
+    if self._output_key_fn is None:
+      # The output is its own dedup key, so the key coder is the output coder.
       key_fn = _identity
-      key_coder = output_coder
+      key_coder = self._output_key_coder or output_coder
     else:
-      if self._output_key_fn is None:
-        # The output is its own dedup key, so the key coder is the output
-        # coder.
-        key_fn = _identity
-        key_coder = self._output_key_coder or output_coder
-      else:
-        key_fn = self._output_key_fn
-        key_coder = self._output_key_coder or _coder_for_hint(
-            _return_type(self._output_key_fn))
-      # Dedup hashes the encoded key, so equal keys must encode equally; use
-      # the coder's deterministic form and reject coders that have none.
-      key_coder = key_coder.as_deterministic_coder(
-          self.label,
-          'Watch dedups by hashing the encoded output key, so the key coder '
-          'must be deterministic. %s has no deterministic form; pass a '
-          'deterministic output_key_coder (or output_coder).' %
-          type(key_coder).__name__)
+      key_fn = self._output_key_fn
+      key_coder = self._output_key_coder or _coder_for_hint(
+          _return_type(self._output_key_fn))
+    # Dedup hashes the encoded key, so equal keys must encode equally; use the
+    # coder's deterministic form and reject coders that have none.
+    key_coder = key_coder.as_deterministic_coder(
+        self.label,
+        'Watch dedups by hashing the encoded output key, so the key coder '
+        'must be deterministic. %s has no deterministic form; pass a '
+        'deterministic output_key_coder (or output_coder).' %
+        type(key_coder).__name__)
     # Type the (input, output) pairs from the input type and the resolved
     # coder's type, so downstream transforms are typed and coder inference does
     # not fall back to pickling.
@@ -927,6 +911,7 @@ def expand(self, pcoll):
             key_fn,
             key_coder,
             self._timestamp_cursor,
+            self._allowed_lateness,
             self._now)).with_output_types(tuple[input_type, value_type])
 
 
diff --git a/sdks/python/apache_beam/io/watch_test.py b/sdks/python/apache_beam/io/watch_test.py
index 7cc9157abbfc..7e1870cd3c15 100644
--- a/sdks/python/apache_beam/io/watch_test.py
+++ b/sdks/python/apache_beam/io/watch_test.py
@@ -35,11 +35,10 @@
 from apache_beam.io.watch import Watch
 from apache_beam.io.watch import _GrowthRestrictionTracker
 from apache_beam.io.watch import _GrowthStateCoder
-from apache_beam.io.watch import _MicrosTimestampCoder
 from apache_beam.io.watch import _never_seen_before
 from apache_beam.io.watch import _NonPollingGrowthState
-from apache_beam.io.watch import _past_cursor
 from apache_beam.io.watch import _PollingGrowthState
+from apache_beam.io.watch import _retention_floor
 from apache_beam.io.watch import _WatchGrowthDoFn
 from apache_beam.io.watch import after_total_of
 from apache_beam.io.watch import never
@@ -79,9 +78,22 @@ def _tracker(restriction):
   return _GrowthRestrictionTracker(restriction, _identity, StrUtf8Coder())
 
 
-def _cursor_tracker(restriction):
+def _cursor_tracker(restriction, allowed_lateness=Duration(0)):
   return _GrowthRestrictionTracker(
-      restriction, _identity, StrUtf8Coder(), timestamp_cursor=True)
+      restriction,
+      _identity,
+      StrUtf8Coder(),
+      timestamp_cursor=True,
+      allowed_lateness=allowed_lateness)
+
+
+def _cursor_results(restriction, result, allowed_lateness=Duration(0)):
+  return _never_seen_before(
+      restriction,
+      result,
+      _identity,
+      StrUtf8Coder(),
+      _retention_floor(restriction, allowed_lateness))
 
 
 def _initial_polling(termination=None, now=Timestamp(0)):
@@ -135,30 +147,18 @@ def test_polling_round_trip_preserves_resume_state(self):
     self.assertEqual(termination_state, decoded.termination_state)
     self.assertIsNone(decoded.cursor)
 
-  def test_polling_round_trip_preserves_cursor(self):
+  def test_polling_round_trip_preserves_cursor_and_retained_keys(self):
     coder = _GrowthStateCoder(StrUtf8Coder(), never())
+    completed = collections.OrderedDict([(b'a' * 16, Timestamp(42))])
     state = _PollingGrowthState(
-        collections.OrderedDict(),
+        completed,
         Timestamp(5),
         never().for_new_input(Timestamp(0), 'input'),
         Timestamp(42))
     decoded = coder.decode(coder.encode(state))
     self.assertEqual(Timestamp(42), decoded.cursor)
-    self.assertEqual(0, len(decoded.completed))
-    self.assertIsNone(decoded.poll_watermark)  # not part of the payload
-
-  def test_polling_round_trip_preserves_subsecond_cursors(self):
-    # A cursor rounded down to the millisecond, as TimestampCoder does, comes
-    # back below the outputs it was taken from and hands them out again.
-    coder = _GrowthStateCoder(StrUtf8Coder(), never())
-    termination_state = never().for_new_input(Timestamp(0), 'input')
-    for cursor in (Timestamp(micros=1786371369335245),
-                   MIN_TIMESTAMP,
-                   MAX_TIMESTAMP):
-      with self.subTest(cursor=cursor):
-        state = _PollingGrowthState(
-            collections.OrderedDict(), None, termination_state, cursor)
-        self.assertEqual(cursor, coder.decode(coder.encode(state)).cursor)
+    self.assertEqual(list(completed.items()), list(decoded.completed.items()))
+    self.assertEqual(Timestamp(5), decoded.poll_watermark)
 
   def test_cursorless_state_keeps_the_pre_cursor_byte_format(self):
     # A polling state without a cursor must encode exactly as before the
@@ -192,33 +192,6 @@ def test_non_polling_round_trip_preserves_pending_outputs(self):
     self.assertEqual([('a', Timestamp(1)), ('b', Timestamp(2))],
                      [(o.value, o.timestamp) for o in decoded.pending.outputs])
 
-  def test_truncated_timestamp_payload_fails_to_decode(self):
-    coder = _MicrosTimestampCoder()
-    encoded = coder.encode(Timestamp(micros=1786371369335245))
-    with self.assertRaisesRegex(ValueError, 'expected 8'):
-      coder.decode(encoded[:-1])
-
-  def test_retired_state_tag_fails_to_decode(self):
-    # The millisecond payloads the retired tags carried are the width of the
-    # microsecond ones, so a reused tag would decode them to a wrong timestamp.
-    coder = _GrowthStateCoder(StrUtf8Coder(), never())
-    for retired in (1, 2):
-      with self.subTest(tag=retired):
-        encoded = TupleCoder([VarIntCoder(), BytesCoder()]).encode(
-            (retired, TimestampCoder().encode(Timestamp(5))))
-        with self.assertRaisesRegex(ValueError, 'unknown Watch growth state'):
-          coder.decode(encoded)
-
-  def test_non_polling_round_trip_preserves_subsecond_pending_outputs(self):
-    # A replay repeats the emission it stands in for, so a rounded timestamp
-    # can land the replay in a different window than the original.
-    coder = _GrowthStateCoder(StrUtf8Coder(), never())
-    timestamp = Timestamp(micros=1000750)
-    state = _NonPollingGrowthState(
-        PollResult((TimestampedValue('a', timestamp), ), None))
-    decoded = coder.decode(coder.encode(state))
-    self.assertEqual(timestamp, decoded.pending.outputs[0].timestamp)
-
 
 class NeverSeenBeforeTest(unittest.TestCase):
   def test_dedups_and_sorts_by_timestamp(self):
@@ -349,64 +322,95 @@ def test_idle_round_reuses_completed_map_object(self):
 
 
 class TimestampCursorTest(unittest.TestCase):
-  """Cursor-mode dedup: high-water-mark timestamp instead of a hash set."""
-  def test_keeps_state_o1_and_tracks_high_water_mark(self):
+  """Cursor-mode dedup: hash dedup whose keys the cursor retires."""
+  def test_bounds_the_key_set_to_the_newest_event_time(self):
     state = _initial_polling()
     result = PollResult.incomplete([_ts('a', 1), _ts('b', 2), _ts('c', 3)])
-    new_results = _past_cursor(state, result)
+    new_results = _cursor_results(state, result)
     self.assertEqual(['a', 'b', 'c'], [o.value for o in new_results.outputs])
     tracker = _cursor_tracker(state)
     self.assertTrue(tracker.try_claim((new_results, 0)))
     _, residual = tracker.try_split(0)
     self.assertIsInstance(residual, _PollingGrowthState)
-    self.assertEqual(0, len(residual.completed))  # no hash set
-    self.assertEqual(Timestamp(3), residual.cursor)  # high-water mark
+    self.assertEqual(Timestamp(3), residual.cursor)
+    # The two older keys are retired; only the one at the cursor is kept.
+    self.assertEqual(1, len(residual.completed))
+
+  def test_outputs_sharing_an_event_time_are_each_emitted_once(self):
+    # A bare cursor cannot tell two outputs at one event time apart, so it
+    # either drops the second or repeats both on the next re-list. The keys
+    # the cursor still retains are what distinguishes them.
+    state = _initial_polling()
+    first = _cursor_results(
+        state, PollResult.incomplete([_ts('a', 10), _ts('b', 10)]))
+    self.assertEqual(['a', 'b'], sorted(o.value for o in first.outputs))
+    tracker = _cursor_tracker(state)
+    self.assertTrue(tracker.try_claim((first, 0)))
+    _, residual = tracker.try_split(0)
+    self.assertEqual(Timestamp(10), residual.cursor)
+    relist = _cursor_results(
+        residual,
+        PollResult.incomplete([_ts('a', 10), _ts('b', 10), _ts('c', 10)]))
+    self.assertEqual(['c'], [o.value for o in relist.outputs])
 
-  def test_emits_only_outputs_after_the_cursor(self):
-    # A later round emits only outputs strictly past the cursor; a re-listed
-    # output (== cursor) and an earlier output (< cursor) are both dropped.
+  def test_drops_outputs_the_cursor_retired(self):
     state = _initial_polling()
     tracker = _cursor_tracker(state)
-    first = _past_cursor(state, PollResult.incomplete([_ts('a', 10)]))
+    first = _cursor_results(state, PollResult.incomplete([_ts('a', 10)]))
     self.assertTrue(tracker.try_claim((first, 0)))
     _, residual = tracker.try_split(0)
     self.assertEqual(Timestamp(10), residual.cursor)
-    second = _past_cursor(
+    second = _cursor_results(
         residual,
         PollResult.incomplete([_ts('early', 5), _ts('a', 10), _ts('c', 20)]))
-    self.assertEqual(['c'], [o.value for o in second.outputs])  # only 20 > 10
+    # 'early' is below the floor, 'a' is a retained key, only 'c' is new.
+    self.assertEqual(['c'], [o.value for o in second.outputs])
     resumed = _cursor_tracker(residual)
     self.assertTrue(resumed.try_claim((second, 0)))
     _, residual = resumed.try_split(0)
     self.assertEqual(Timestamp(20), residual.cursor)
 
+  def test_allowed_lateness_retains_keys_below_the_cursor(self):
+    # A wider window keeps deduping outputs that arrive behind the cursor
+    # instead of taking them as already seen.
+    lateness = Duration(10)
+    state = _initial_polling()
+    tracker = _cursor_tracker(state, lateness)
+    first = _cursor_results(
+        state, PollResult.incomplete([_ts('a', 20)]), lateness)
+    self.assertTrue(tracker.try_claim((first, 0)))
+    _, residual = tracker.try_split(0)
+    late = _cursor_results(
+        residual,
+        PollResult.incomplete([_ts('a', 20), _ts('late', 12), _ts('old', 5)]),
+        lateness)
+    self.assertEqual(['late'], [o.value for o in late.outputs])
+
   def test_relist_emits_each_output_exactly_once(self):
     # A full re-list of a growing collection at strictly increasing event
-    # times emits each output once; the state never accumulates a hash set.
+    # times emits each output once; the key set stays bounded throughout.
     state = _initial_polling()
     emitted = collections.Counter()
     for round_index in range(10):
       result = PollResult.incomplete(
           [_ts('f%d' % i, i + 1) for i in range(round_index + 1)])
-      new_results = _past_cursor(state, result)
+      new_results = _cursor_results(state, result)
       tracker = _cursor_tracker(state)
       self.assertTrue(tracker.try_claim((new_results, 0)))
       for output in new_results.outputs:
         emitted[output.value] += 1
       _, state = tracker.try_split(0)
-      self.assertEqual(0, len(state.completed))  # O(1) throughout
+      self.assertEqual(1, len(state.completed))
     self.assertEqual([1] * 10, [emitted['f%d' % i] for i in range(10)])
     self.assertEqual(Timestamp(10), state.cursor)
 
-  def test_round_below_high_water_mark_keeps_cursor_and_reuses_state(self):
-    # A round whose outputs are all at or below the cursor emits nothing and
-    # leaves the cursor unchanged; the (empty) completed map is reused as-is.
+  def test_round_below_the_cursor_leaves_it_unchanged(self):
     state = _initial_polling()
     tracker = _cursor_tracker(state)
-    first = _past_cursor(state, PollResult.incomplete([_ts('a', 10)]))
+    first = _cursor_results(state, PollResult.incomplete([_ts('a', 10)]))
     self.assertTrue(tracker.try_claim((first, 0)))
     _, residual1 = tracker.try_split(0)
-    stale = _past_cursor(
+    stale = _cursor_results(
         residual1, PollResult.incomplete([_ts('a', 10), _ts('old', 4)]))
     self.assertEqual((), stale.outputs)
     resumed = _cursor_tracker(residual1)
@@ -415,60 +419,42 @@ def test_round_below_high_water_mark_keeps_cursor_and_reuses_state(self):
     self.assertEqual(Timestamp(10), residual2.cursor)  # unchanged
     self.assertIs(residual1.completed, residual2.completed)
 
-  def test_claim_rejects_outputs_at_or_below_the_cursor(self):
-    # The tracker re-validates a claim, so a round that was not filtered
-    # against the cursor is rejected instead of emitting already-seen outputs.
+  def test_claim_rejects_retained_keys_and_retired_outputs(self):
+    # The tracker re-validates a claim, so a round that was not filtered is
+    # rejected instead of emitting already-seen outputs.
     state = _initial_polling()
     tracker = _cursor_tracker(state)
-    first = _past_cursor(state, PollResult.incomplete([_ts('a', 10)]))
+    first = _cursor_results(state, PollResult.incomplete([_ts('a', 10)]))
     self.assertTrue(tracker.try_claim((first, 0)))
     _, residual = tracker.try_split(0)
-    stale = PollResult.incomplete([_ts('a', 10)])
-    self.assertFalse(_cursor_tracker(residual).try_claim((stale, 0)))
-
-  def test_replay_validates_by_timestamps(self):
-    # Cursor mode never hashes, so a replay is validated by its timestamps.
-    pending = PollResult((_ts('a', 1), _ts('b', 2)), MAX_TIMESTAMP)
-    tracker = _cursor_tracker(_NonPollingGrowthState(pending))
-    partial = PollResult((_ts('a', 1), ), None)
-    self.assertFalse(tracker.try_claim((partial, None)))
-    self.assertTrue(tracker.try_claim((pending, None)))
-
-  def test_switching_hash_state_to_cursor_drops_the_hash_map(self):
-    # A restriction carried over from hash dedup still holds completed hashes;
-    # cursor mode ignores them, so the first cursor round must drop them and
-    # make the state O(1) rather than carry dead hashes forever.
-    legacy = _PollingGrowthState(
-        collections.OrderedDict([(b'a' * 16, Timestamp(1))]),
-        None,
-        never().for_new_input(Timestamp(0), 'input'))
-    result = _past_cursor(legacy, PollResult.incomplete([_ts('a', 100)]))
-    tracker = _cursor_tracker(legacy)
-    self.assertTrue(tracker.try_claim((result, 0)))
-    _, residual = tracker.try_split(0)
-    self.assertEqual(0, len(residual.completed))
-    self.assertEqual(Timestamp(100), residual.cursor)
-
-  def test_switching_hash_state_to_cursor_seeds_the_cursor(self):
-    # Outputs at or below the hash map's greatest recorded event time are
-    # already seen and must not re-emit after the switch.
-    legacy = _PollingGrowthState(
-        collections.OrderedDict([(b'a' * 16, Timestamp(5)),
-                                 (b'b' * 16, Timestamp(10))]),
-        None,
-        never().for_new_input(Timestamp(0), 'input'))
-    relist = PollResult.incomplete([_ts('a', 5), _ts('b', 10), _ts('c', 20)])
-    new_results = _past_cursor(legacy, relist)
-    self.assertEqual(['c'], [o.value for o in new_results.outputs])
+    self.assertFalse(
+        _cursor_tracker(residual).try_claim(
+            (PollResult.incomplete([_ts('a', 10)]), 0)))
+    self.assertFalse(
+        _cursor_tracker(residual).try_claim(
+            (PollResult.incomplete([_ts('old', 4)]), 0)))
+
+  def test_switching_hash_state_to_cursor_keeps_the_keys(self):
+    # A restriction resumed in cursor mode still holds the hashes from its
+    # hash rounds, so nothing re-emits; the cursor retires them from there on.
+    state = _initial_polling()
+    hash_tracker = _tracker(state)
+    first = _new_results(state, PollResult.incomplete([_ts('a', 5)]))
+    self.assertTrue(hash_tracker.try_claim((first, 0)))
+    _, legacy = hash_tracker.try_split(0)
+    self.assertIsNone(legacy.cursor)
+    relist = _cursor_results(
+        legacy, PollResult.incomplete([_ts('a', 5), _ts('c', 20)]))
+    self.assertEqual(['c'], [o.value for o in relist.outputs])
     tracker = _cursor_tracker(legacy)
-    self.assertTrue(tracker.try_claim((new_results, 0)))
+    self.assertTrue(tracker.try_claim((relist, 0)))
     _, residual = tracker.try_split(0)
-    self.assertEqual(0, len(residual.completed))
     self.assertEqual(Timestamp(20), residual.cursor)
+    self.assertEqual(1, len(residual.completed))
 
   def test_hash_round_drops_a_stale_cursor(self):
-    # The reverse switch: a hash round drops the cursor, so a state never
-    # holds hashes and a cursor at the same time.
+    # The reverse switch: a hash round retains every key, so the cursor that
+    # would retire them is dropped.
     state = _PollingGrowthState(
         collections.OrderedDict(), None, 0, cursor=Timestamp(10))
     tracker = _tracker(state)
@@ -486,7 +472,7 @@ def encoded_residual_after_claiming(count):
       result = PollResult.incomplete(
           [_ts('output%d' % i, i + 1) for i in range(count)])
       tracker = _cursor_tracker(state)
-      self.assertTrue(tracker.try_claim((_past_cursor(state, result), 0)))
+      self.assertTrue(tracker.try_claim((_cursor_results(state, result), 0)))
       _, residual = tracker.try_split(0)
       return coder.encode(residual)
 
@@ -827,20 +813,26 @@ def test_timestamp_cursor_dedups_growing_source(self):
               _growing_poll,
               poll_interval=Duration(0.05),
               timestamp_cursor=True))
-      # Each output is emitted exactly once via the high-water-mark cursor,
-      # with no hash set kept, across poll rounds and checkpoints.
+      # Each output is emitted exactly once, with the cursor retiring keys as
+      # it advances, across poll rounds and checkpoints.
       assert_that(
           output,
           equal_to([('x:', 'x:0'), ('x:', 'x:1'), ('x:', 'x:2'), ('y:', 'y:0'),
                     ('y:', 'y:1'), ('y:', 'y:2')]))
 
-  def test_timestamp_cursor_rejects_key_spec(self):
-    with self.assertRaises(ValueError):
-      Watch(
-          _complete_poll,
-          poll_interval=Duration(1),
-          output_key_fn=_first_char,
-          timestamp_cursor=True)
+  def test_timestamp_cursor_composes_with_an_output_key(self):
+    # The cursor bounds the state; the key still decides what counts as seen.
+    _POLL_CALLS.clear()
+    with self._in_memory_pipeline() as p:
+      output = (
+          p | beam.Create(['x:'])
+          | Watch(
+              _growing_poll,
+              poll_interval=Duration(0.05),
+              output_key_fn=_first_char,
+              timestamp_cursor=True))
+      # Every output shares a key, so only the first one is ever emitted.
+      assert_that(output, equal_to([('x:', 'x:0')]))
 
   def test_output_key_dedups_across_pipeline(self):
     with self._in_memory_pipeline() as p:

From 9ce7c97a916372a1817dbe4e783d8ae3d056c6ce Mon Sep 17 00:00:00 2001
From: Eliaazzz <Elialiu760317@outlook.com>
Date: Thu, 13 Aug 2026 12:09:27 +1000
Subject: [PATCH 10/16] Let the MatchContinuously cursor compose with the match
 key

The cursor bounds the deduplication state rather than replacing the key, so
the transform passes the same key function in both modes and
match_updated_files decides whether a changed file counts as new again.

The watermark under timestamp_cursor is the newest last-modified time matched,
capped at the poll time, and the poll time when nothing matched, so a quiet
directory no longer holds it back.
---
 sdks/python/apache_beam/io/fileio.py      | 72 +++++++++--------------
 sdks/python/apache_beam/io/fileio_test.py | 69 ++++++++++++++++++++--
 2 files changed, 94 insertions(+), 47 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index 35614c7a7a28..127a4ccd2852 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -320,9 +320,9 @@ class _MatchContinuouslyPollFn(PollFn):
   as their event time, and the watermark advances to the poll time so
   event-time windows progress even when nothing new matches. Under
   ``mtime_timestamps`` a match carries its last-modified time instead, which
-  is what the timestamp cursor dedups on, and the watermark is the newest
+  is what bounds the timestamp cursor, and the watermark is the newest
   last-modified time matched, capped at the poll time. A poll that matches
-  nothing leaves the watermark where it is.
+  nothing takes the poll time.
   """
   def __init__(
       self,
@@ -353,15 +353,11 @@ def __call__(self, file_pattern: str) -> PollResult[filesystem.FileMetadata]:
         TimestampedValue(metadata, Timestamp.of(_ensure_mtime(metadata)))
         for metadata in match_result.metadata_list
     ]
-    if not outputs:
-      # Matching nothing is no reading of the filesystem clock. Holding the
-      # watermark keeps the files that clock has yet to reach from arriving
-      # behind it.
-      return PollResult.incomplete(())
     # The filesystem clock can run behind the local one, so the watermark stops
     # at the newest last-modified time matched, and never runs past the poll
-    # time when that clock is ahead.
-    newest = max(output.timestamp for output in outputs)
+    # time when that clock is ahead. Matching nothing gives no reading of that
+    # clock, and the poll time keeps event-time windows progressing.
+    newest = max((output.timestamp for output in outputs), default=now)
     return PollResult.incomplete(outputs).with_watermark(min(newest, now))
 
 
@@ -377,17 +373,16 @@ class MatchContinuously(beam.PTransform):
   Deduplication state lives in the splittable DoFn restriction, so a runner
   with checkpointing enabled restores it after a restart and does not
   reprocess files. That state holds one id per matched file and grows with the
-  directory, unless ``timestamp_cursor`` bounds it to a single timestamp. For
-  a growing directory on GCS, consider an alternate technique such as Pub/Sub
-  Notifications
+  directory, unless ``timestamp_cursor`` bounds it to the newest matched
+  last-modified time. For a growing directory on GCS, consider an alternate
+  technique such as Pub/Sub Notifications
   (https://cloud.google.com/storage/docs/pubsub-notifications).
 
   A match carries the poll time as its event time, and the watermark follows
   the poll time. Under ``timestamp_cursor`` a match carries its last-modified
-  time and the watermark tracks the filesystem clock instead: the newest
-  last-modified time matched, capped at the poll time, and left where it is by
-  a poll that matches nothing. A clock behind the local one then cannot make a
-  later file late, and one ahead cannot carry the watermark with it.
+  time and the watermark is the newest one matched, capped at the poll time,
+  so a filesystem clock ahead of the local one cannot carry the watermark with
+  it.
   """
   def __init__(
       self,
@@ -412,18 +407,16 @@ def __init__(
         file with timestamp changes.
       apply_windowing: Whether each element should be assigned to
         individual window. If false, all elements will reside in global window.
-      timestamp_cursor: (When has_deduplication is set to True) dedup by
-        last-modified time instead of by file id, which bounds the state to a
-        single timestamp. Each poll emits only the files modified past the
-        newest last-modified time already emitted, compared to the microsecond
-        a Timestamp holds. A file at or below that mark is skipped, as happens
-        with copies that preserve the source time and with backfills of older
-        files. A modified file is emitted again once its reported time passes
-        the mark, and match_updated_files has no effect in this mode. Requires
-        the filesystem to report last-modified times, and a pipeline started
-        fresh: turning it on while updating a running pipeline seeds the cursor
-        with the poll times the old state recorded, which are not last-modified
-        times, so files are skipped or repeated.
+      timestamp_cursor: (When has_deduplication is set to True) bound the
+        deduplication state by last-modified time. Files are still deduplicated
+        by id, but an id is retired once a newer file has been matched, so the
+        state holds the newest last-modified time and the ids sharing it rather
+        than one id per file the pattern has ever matched. A file whose
+        last-modified time is older than that mark is taken as already seen and
+        skipped, as happens with copies that preserve the source time and with
+        backfills of older files. Requires the filesystem to report
+        last-modified times, and matches then carry their last-modified time as
+        their event time instead of the poll time.
     """
 
     self.file_pattern = file_pattern
@@ -490,21 +483,14 @@ def _match_deduplicated(self,
         self.start_ts,
         clock,
         mtime_timestamps=self.timestamp_cursor)
-    if self.timestamp_cursor:
-      # The cursor dedups on the mtime the poll stamps, so there is no key.
-      watch = Watch(
-          poll_fn,
-          poll_interval=self.interval,
-          termination=termination,
-          timestamp_cursor=True)
-    else:
-      # The key coder is inferred from the key function's return annotation.
-      watch = Watch(
-          poll_fn,
-          poll_interval=self.interval,
-          termination=termination,
-          output_key_fn=(
-              _file_path_and_mtime_key if self.match_upd else _file_path_key))
+    # The key coder is inferred from the key function's return annotation.
+    watch = Watch(
+        poll_fn,
+        poll_interval=self.interval,
+        termination=termination,
+        output_key_fn=(
+            _file_path_and_mtime_key if self.match_upd else _file_path_key),
+        timestamp_cursor=self.timestamp_cursor)
     # Watch emits (pattern, file) pairs; keep the FileMetadata output type so
     # downstream transforms stay typed instead of falling back to Any.
     return (
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index 1d5108181c5f..da3fca1ac03d 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -578,17 +578,18 @@ def test_poll_fn_caps_the_mtime_watermark_at_the_poll_time(self):
     result = poll_fn(FileSystems.join(tempdir, '*'))
     self.assertTrue(before <= result.watermark <= Timestamp.now())
 
-  def test_poll_fn_holds_the_mtime_watermark_on_empty_match(self):
-    # No match is no reading of the filesystem clock. A watermark at the poll
-    # time would leave the files that clock has yet to reach behind it.
+  def test_poll_fn_advances_the_mtime_watermark_on_empty_match(self):
+    # No match is no reading of the filesystem clock, so the watermark takes
+    # the poll time and event-time windows keep progressing.
     tempdir = '%s%s' % (self._new_tempdir(), os.sep)
     poll_fn = fileio._MatchContinuouslyPollFn(
         fileio.EmptyMatchTreatment.ALLOW,
         Timestamp.now() - 3600,
         mtime_timestamps=True)
+    before = Timestamp.now()
     result = poll_fn(FileSystems.join(tempdir, '*'))
     self.assertEqual((), result.outputs)
-    self.assertIsNone(result.watermark)
+    self.assertTrue(before <= result.watermark <= Timestamp.now())
 
   def test_timestamp_cursor_rejects_missing_mtime(self):
     # Without mtimes every match would carry the same event time, so the
@@ -662,6 +663,66 @@ def _create_backdated_file(element):
 
       assert_that(match_continiously, equal_to([first]))
 
+  def test_timestamp_cursor_emits_a_file_sharing_the_newest_mtime(self):
+    # A file landing with the same last-modified time as the newest one
+    # already emitted is still new. Filesystems that report to the
+    # millisecond, GCS among them, hand out such ties routinely.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    first = self._create_temp_file(dir=tempdir)
+    os.utime(first, (1234.5, 1234.5))
+    twin = FileSystems.join(tempdir, 'twin')
+
+    interval = 0.2
+    start = Timestamp.now()
+    stop = start + interval + 0.1
+
+    def _create_twin(element):
+      writer = FileSystems.create(twin)
+      writer.close()
+      os.utime(twin, (1234.5, 1234.5))
+      return element.path
+
+    with TestPipeline() as p:
+      match_continiously = (
+          p
+          | fileio.MatchContinuously(
+              file_pattern=FileSystems.join(tempdir, '*'),
+              interval=interval,
+              start_timestamp=start,
+              stop_timestamp=stop,
+              timestamp_cursor=True)
+          | beam.Map(_create_twin))
+
+      assert_that(match_continiously, equal_to([first, twin]))
+
+  def test_timestamp_cursor_leaves_updated_files_out_by_default(self):
+    # match_updated_files still decides whether a changed file counts as new,
+    # so an update is skipped unless it is asked for.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    path = self._create_temp_file(dir=tempdir)
+    os.utime(path, (1234.5, 1234.5))
+
+    interval = 0.2
+    start = Timestamp.now()
+    stop = start + interval + 0.1
+
+    def _touch(element):
+      os.utime(path, (2345.5, 2345.5))
+      return element.path
+
+    with TestPipeline() as p:
+      match_continiously = (
+          p
+          | fileio.MatchContinuously(
+              file_pattern=FileSystems.join(tempdir, '*'),
+              interval=interval,
+              start_timestamp=start,
+              stop_timestamp=stop,
+              timestamp_cursor=True)
+          | beam.Map(_touch))
+
+      assert_that(match_continiously, equal_to([path]))
+
 
 class WriteFilesTest(_TestCaseWithTempDirCleanUp):
 

From 0526f7d9b515bc70b15053929174bcd7e6eb879b Mon Sep 17 00:00:00 2001
From: Elia LIU <Elialiu760317@outlook.com>
Date: Thu, 13 Aug 2026 14:51:33 +1000
Subject: [PATCH 11/16] Release the mtime watermark when a poll finds nothing
 newer

Holding the watermark at the newest last-modified time matched stalls a
directory that is quiet rather than empty: every poll re-lists the same files,
so the newest one never moves and the watermark sits at its last-modified time
while the poll time runs away from it. Only an empty match released it, which
is not the case that stalls.

The hold now follows the evidence. A poll that turns up a last-modified time
newer than any before it has just read the filesystem clock, so the watermark
stops there and files still in flight behind a clock that lags the local one
are not late. A poll that finds nothing newer has no fresh reading to go on, so
the watermark takes the poll time and event-time windows keep closing. A
continuously fed directory therefore trails the filesystem clock throughout,
and a quiet one catches up.

Also records what bounding the deduplication state costs: a file modified after
its id was retired reads as new and is matched a second time, whatever
match_updated_files says. The MatchContinuously test that covered the update
case never advanced the cursor, so it did not reach the retirement; the Watch
test added here pins it.
---
 sdks/python/apache_beam/io/fileio.py      | 48 ++++++++++++++++-------
 sdks/python/apache_beam/io/fileio_test.py | 41 ++++++++++++++++++-
 sdks/python/apache_beam/io/watch_test.py  | 26 ++++++++++++
 3 files changed, 98 insertions(+), 17 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index 127a4ccd2852..5f99feeda27f 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -320,9 +320,9 @@ class _MatchContinuouslyPollFn(PollFn):
   as their event time, and the watermark advances to the poll time so
   event-time windows progress even when nothing new matches. Under
   ``mtime_timestamps`` a match carries its last-modified time instead, which
-  is what bounds the timestamp cursor, and the watermark is the newest
-  last-modified time matched, capped at the poll time. A poll that matches
-  nothing takes the poll time.
+  is what bounds the timestamp cursor, and the watermark trails the newest
+  last-modified time while that time keeps advancing, otherwise it takes the
+  poll time.
   """
   def __init__(
       self,
@@ -334,6 +334,9 @@ def __init__(
     self._start_micros = Timestamp.of(start_timestamp).micros
     self._clock = clock if clock is not None else _PollClock()
     self._mtime_timestamps = mtime_timestamps
+    # Greatest last-modified time handed out so far, to tell a poll that found
+    # something newer from one that only re-listed what was already there.
+    self._newest_mtime = None  # type: Optional[Timestamp]
 
   def __call__(self, file_pattern: str) -> PollResult[filesystem.FileMetadata]:
     now = Timestamp.now()
@@ -353,12 +356,21 @@ def __call__(self, file_pattern: str) -> PollResult[filesystem.FileMetadata]:
         TimestampedValue(metadata, Timestamp.of(_ensure_mtime(metadata)))
         for metadata in match_result.metadata_list
     ]
-    # The filesystem clock can run behind the local one, so the watermark stops
-    # at the newest last-modified time matched, and never runs past the poll
-    # time when that clock is ahead. Matching nothing gives no reading of that
-    # clock, and the poll time keeps event-time windows progressing.
-    newest = max((output.timestamp for output in outputs), default=now)
-    return PollResult.incomplete(outputs).with_watermark(min(newest, now))
+    # A poll that turned up a newer last-modified time than any before it just
+    # read the filesystem clock, so the watermark stops there rather than at
+    # the poll time, and files still in flight behind a filesystem clock that
+    # lags the local one are not late. It is also capped at the poll time, so a
+    # clock running ahead cannot carry the watermark with it. A poll that found
+    # nothing newer has no fresh reading to go on, so the watermark takes the
+    # poll time and a quiet directory does not stall event-time windows.
+    newest = max((output.timestamp for output in outputs), default=None)
+    if newest is not None and (self._newest_mtime is None or
+                               newest > self._newest_mtime):
+      self._newest_mtime = newest
+      watermark = min(newest, now)
+    else:
+      watermark = now
+    return PollResult.incomplete(outputs).with_watermark(watermark)
 
 
 class MatchContinuously(beam.PTransform):
@@ -380,9 +392,12 @@ class MatchContinuously(beam.PTransform):
 
   A match carries the poll time as its event time, and the watermark follows
   the poll time. Under ``timestamp_cursor`` a match carries its last-modified
-  time and the watermark is the newest one matched, capped at the poll time,
-  so a filesystem clock ahead of the local one cannot carry the watermark with
-  it.
+  time, and the watermark trails the newest last-modified time for as long as
+  polls keep turning up newer files, so files still in flight behind a
+  filesystem clock that lags the local one are not late. It is capped at the
+  poll time, so a filesystem clock ahead of the local one cannot carry the
+  watermark with it, and a poll that turns up nothing newer releases it to the
+  poll time, so a quiet directory does not stall event-time windows.
   """
   def __init__(
       self,
@@ -414,9 +429,12 @@ def __init__(
         than one id per file the pattern has ever matched. A file whose
         last-modified time is older than that mark is taken as already seen and
         skipped, as happens with copies that preserve the source time and with
-        backfills of older files. Requires the filesystem to report
-        last-modified times, and matches then carry their last-modified time as
-        their event time instead of the poll time.
+        backfills of older files. Bounding the state this way costs the
+        guarantee that a file is matched once for all time: a file modified
+        after its id was retired looks new again and is matched a second time,
+        whatever ``match_updated_files`` says. Requires the filesystem to
+        report last-modified times, and matches then carry their last-modified
+        time as their event time instead of the poll time.
     """
 
     self.file_pattern = file_pattern
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index da3fca1ac03d..8d3642fc74ec 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -563,6 +563,39 @@ def test_poll_fn_holds_the_mtime_watermark_at_the_newest_match(self):
     result = poll_fn(FileSystems.join(tempdir, '*'))
     self.assertEqual(Timestamp.of(2345.5), result.watermark)
 
+  def test_poll_fn_releases_the_mtime_watermark_once_nothing_is_newer(self):
+    # Holding at the newest match forever would stall a directory that is
+    # merely quiet rather than empty: every poll re-lists the same old files,
+    # and the watermark would sit at their last-modified time while the poll
+    # time ran away from it, so event-time windows would never close.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    os.utime(self._create_temp_file(dir=tempdir), (1234.5, 1234.5))
+    poll_fn = fileio._MatchContinuouslyPollFn(
+        fileio.EmptyMatchTreatment.ALLOW,
+        Timestamp.now() - 3600,
+        mtime_timestamps=True)
+    pattern = FileSystems.join(tempdir, '*')
+    self.assertEqual(Timestamp.of(1234.5), poll_fn(pattern).watermark)
+    before = Timestamp.now()
+    quiet = poll_fn(pattern)
+    self.assertEqual(1, len(quiet.outputs))
+    self.assertTrue(before <= quiet.watermark <= Timestamp.now())
+
+  def test_poll_fn_holds_the_mtime_watermark_again_for_a_newer_match(self):
+    # A poll that does turn up a newer file has read the filesystem clock
+    # again, so the hold comes back rather than being spent once.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    os.utime(self._create_temp_file(dir=tempdir), (1234.5, 1234.5))
+    poll_fn = fileio._MatchContinuouslyPollFn(
+        fileio.EmptyMatchTreatment.ALLOW,
+        Timestamp.now() - 3600,
+        mtime_timestamps=True)
+    pattern = FileSystems.join(tempdir, '*')
+    poll_fn(pattern)
+    poll_fn(pattern)
+    os.utime(self._create_temp_file(dir=tempdir), (2345.5, 2345.5))
+    self.assertEqual(Timestamp.of(2345.5), poll_fn(pattern).watermark)
+
   def test_poll_fn_caps_the_mtime_watermark_at_the_poll_time(self):
     # A filesystem clock running ahead must not carry the watermark with it,
     # which pins the watermark to the poll time.
@@ -695,9 +728,13 @@ def _create_twin(element):
 
       assert_that(match_continiously, equal_to([first, twin]))
 
-  def test_timestamp_cursor_leaves_updated_files_out_by_default(self):
+  def test_timestamp_cursor_leaves_updated_files_out_while_their_id_is_held(
+      self):
     # match_updated_files still decides whether a changed file counts as new,
-    # so an update is skipped unless it is asked for.
+    # so an update is skipped unless it is asked for. That only holds while
+    # the file's id is still retained: once the cursor has moved past it and
+    # retired it, a later update looks new again. See the retirement test in
+    # watch_test.py.
     tempdir = '%s%s' % (self._new_tempdir(), os.sep)
     path = self._create_temp_file(dir=tempdir)
     os.utime(path, (1234.5, 1234.5))
diff --git a/sdks/python/apache_beam/io/watch_test.py b/sdks/python/apache_beam/io/watch_test.py
index 7e1870cd3c15..a44428af5923 100644
--- a/sdks/python/apache_beam/io/watch_test.py
+++ b/sdks/python/apache_beam/io/watch_test.py
@@ -386,6 +386,32 @@ def test_allowed_lateness_retains_keys_below_the_cursor(self):
         lateness)
     self.assertEqual(['late'], [o.value for o in late.outputs])
 
+  def test_a_retired_key_returning_later_is_emitted_again(self):
+    # What bounding the state costs. A key is retired by the event time it was
+    # recorded with, so a key that comes back at a later event time, after the
+    # cursor has moved past the one it was recorded with, has nothing left to
+    # prove it was seen. This is the case for a file modified after the cursor
+    # passed it: it is emitted a second time, whatever the key function says
+    # about updates. Keep the default hash dedup where that matters.
+    state = _initial_polling()
+    tracker = _cursor_tracker(state)
+    first = _cursor_results(state, PollResult.incomplete([_ts('a', 10)]))
+    self.assertTrue(tracker.try_claim((first, 0)))
+    _, residual = tracker.try_split(0)
+    # 'b' moves the cursor past the event time 'a' was recorded with, which
+    # retires 'a'.
+    second = _cursor_results(
+        residual, PollResult.incomplete([_ts('a', 10), _ts('b', 20)]))
+    self.assertEqual(['b'], [o.value for o in second.outputs])
+    resumed = _cursor_tracker(residual)
+    self.assertTrue(resumed.try_claim((second, 0)))
+    _, residual = resumed.try_split(0)
+    self.assertEqual([Timestamp(20)], list(residual.completed.values()))
+    # 'a' now returns above the floor, so it reads as new.
+    third = _cursor_results(
+        residual, PollResult.incomplete([_ts('a', 30), _ts('b', 20)]))
+    self.assertEqual(['a'], [o.value for o in third.outputs])
+
   def test_relist_emits_each_output_exactly_once(self):
     # A full re-list of a growing collection at strictly increasing event
     # times emits each output once; the key set stays bounded throughout.

From f184d2f3651bf8c27b2c967b4273289bdf571504 Mon Sep 17 00:00:00 2001
From: Elia LIU <Elialiu760317@outlook.com>
Date: Thu, 13 Aug 2026 15:28:22 +1000
Subject: [PATCH 12/16] Revert "Release the mtime watermark when a poll finds
 nothing newer"

This reverts commit 0526f7d9b515bc70b15053929174bcd7e6eb879b.
---
 sdks/python/apache_beam/io/fileio.py      | 48 +++++++----------------
 sdks/python/apache_beam/io/fileio_test.py | 41 +------------------
 sdks/python/apache_beam/io/watch_test.py  | 26 ------------
 3 files changed, 17 insertions(+), 98 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index 5f99feeda27f..127a4ccd2852 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -320,9 +320,9 @@ class _MatchContinuouslyPollFn(PollFn):
   as their event time, and the watermark advances to the poll time so
   event-time windows progress even when nothing new matches. Under
   ``mtime_timestamps`` a match carries its last-modified time instead, which
-  is what bounds the timestamp cursor, and the watermark trails the newest
-  last-modified time while that time keeps advancing, otherwise it takes the
-  poll time.
+  is what bounds the timestamp cursor, and the watermark is the newest
+  last-modified time matched, capped at the poll time. A poll that matches
+  nothing takes the poll time.
   """
   def __init__(
       self,
@@ -334,9 +334,6 @@ def __init__(
     self._start_micros = Timestamp.of(start_timestamp).micros
     self._clock = clock if clock is not None else _PollClock()
     self._mtime_timestamps = mtime_timestamps
-    # Greatest last-modified time handed out so far, to tell a poll that found
-    # something newer from one that only re-listed what was already there.
-    self._newest_mtime = None  # type: Optional[Timestamp]
 
   def __call__(self, file_pattern: str) -> PollResult[filesystem.FileMetadata]:
     now = Timestamp.now()
@@ -356,21 +353,12 @@ def __call__(self, file_pattern: str) -> PollResult[filesystem.FileMetadata]:
         TimestampedValue(metadata, Timestamp.of(_ensure_mtime(metadata)))
         for metadata in match_result.metadata_list
     ]
-    # A poll that turned up a newer last-modified time than any before it just
-    # read the filesystem clock, so the watermark stops there rather than at
-    # the poll time, and files still in flight behind a filesystem clock that
-    # lags the local one are not late. It is also capped at the poll time, so a
-    # clock running ahead cannot carry the watermark with it. A poll that found
-    # nothing newer has no fresh reading to go on, so the watermark takes the
-    # poll time and a quiet directory does not stall event-time windows.
-    newest = max((output.timestamp for output in outputs), default=None)
-    if newest is not None and (self._newest_mtime is None or
-                               newest > self._newest_mtime):
-      self._newest_mtime = newest
-      watermark = min(newest, now)
-    else:
-      watermark = now
-    return PollResult.incomplete(outputs).with_watermark(watermark)
+    # The filesystem clock can run behind the local one, so the watermark stops
+    # at the newest last-modified time matched, and never runs past the poll
+    # time when that clock is ahead. Matching nothing gives no reading of that
+    # clock, and the poll time keeps event-time windows progressing.
+    newest = max((output.timestamp for output in outputs), default=now)
+    return PollResult.incomplete(outputs).with_watermark(min(newest, now))
 
 
 class MatchContinuously(beam.PTransform):
@@ -392,12 +380,9 @@ class MatchContinuously(beam.PTransform):
 
   A match carries the poll time as its event time, and the watermark follows
   the poll time. Under ``timestamp_cursor`` a match carries its last-modified
-  time, and the watermark trails the newest last-modified time for as long as
-  polls keep turning up newer files, so files still in flight behind a
-  filesystem clock that lags the local one are not late. It is capped at the
-  poll time, so a filesystem clock ahead of the local one cannot carry the
-  watermark with it, and a poll that turns up nothing newer releases it to the
-  poll time, so a quiet directory does not stall event-time windows.
+  time and the watermark is the newest one matched, capped at the poll time,
+  so a filesystem clock ahead of the local one cannot carry the watermark with
+  it.
   """
   def __init__(
       self,
@@ -429,12 +414,9 @@ def __init__(
         than one id per file the pattern has ever matched. A file whose
         last-modified time is older than that mark is taken as already seen and
         skipped, as happens with copies that preserve the source time and with
-        backfills of older files. Bounding the state this way costs the
-        guarantee that a file is matched once for all time: a file modified
-        after its id was retired looks new again and is matched a second time,
-        whatever ``match_updated_files`` says. Requires the filesystem to
-        report last-modified times, and matches then carry their last-modified
-        time as their event time instead of the poll time.
+        backfills of older files. Requires the filesystem to report
+        last-modified times, and matches then carry their last-modified time as
+        their event time instead of the poll time.
     """
 
     self.file_pattern = file_pattern
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index 8d3642fc74ec..da3fca1ac03d 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -563,39 +563,6 @@ def test_poll_fn_holds_the_mtime_watermark_at_the_newest_match(self):
     result = poll_fn(FileSystems.join(tempdir, '*'))
     self.assertEqual(Timestamp.of(2345.5), result.watermark)
 
-  def test_poll_fn_releases_the_mtime_watermark_once_nothing_is_newer(self):
-    # Holding at the newest match forever would stall a directory that is
-    # merely quiet rather than empty: every poll re-lists the same old files,
-    # and the watermark would sit at their last-modified time while the poll
-    # time ran away from it, so event-time windows would never close.
-    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
-    os.utime(self._create_temp_file(dir=tempdir), (1234.5, 1234.5))
-    poll_fn = fileio._MatchContinuouslyPollFn(
-        fileio.EmptyMatchTreatment.ALLOW,
-        Timestamp.now() - 3600,
-        mtime_timestamps=True)
-    pattern = FileSystems.join(tempdir, '*')
-    self.assertEqual(Timestamp.of(1234.5), poll_fn(pattern).watermark)
-    before = Timestamp.now()
-    quiet = poll_fn(pattern)
-    self.assertEqual(1, len(quiet.outputs))
-    self.assertTrue(before <= quiet.watermark <= Timestamp.now())
-
-  def test_poll_fn_holds_the_mtime_watermark_again_for_a_newer_match(self):
-    # A poll that does turn up a newer file has read the filesystem clock
-    # again, so the hold comes back rather than being spent once.
-    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
-    os.utime(self._create_temp_file(dir=tempdir), (1234.5, 1234.5))
-    poll_fn = fileio._MatchContinuouslyPollFn(
-        fileio.EmptyMatchTreatment.ALLOW,
-        Timestamp.now() - 3600,
-        mtime_timestamps=True)
-    pattern = FileSystems.join(tempdir, '*')
-    poll_fn(pattern)
-    poll_fn(pattern)
-    os.utime(self._create_temp_file(dir=tempdir), (2345.5, 2345.5))
-    self.assertEqual(Timestamp.of(2345.5), poll_fn(pattern).watermark)
-
   def test_poll_fn_caps_the_mtime_watermark_at_the_poll_time(self):
     # A filesystem clock running ahead must not carry the watermark with it,
     # which pins the watermark to the poll time.
@@ -728,13 +695,9 @@ def _create_twin(element):
 
       assert_that(match_continiously, equal_to([first, twin]))
 
-  def test_timestamp_cursor_leaves_updated_files_out_while_their_id_is_held(
-      self):
+  def test_timestamp_cursor_leaves_updated_files_out_by_default(self):
     # match_updated_files still decides whether a changed file counts as new,
-    # so an update is skipped unless it is asked for. That only holds while
-    # the file's id is still retained: once the cursor has moved past it and
-    # retired it, a later update looks new again. See the retirement test in
-    # watch_test.py.
+    # so an update is skipped unless it is asked for.
     tempdir = '%s%s' % (self._new_tempdir(), os.sep)
     path = self._create_temp_file(dir=tempdir)
     os.utime(path, (1234.5, 1234.5))
diff --git a/sdks/python/apache_beam/io/watch_test.py b/sdks/python/apache_beam/io/watch_test.py
index a44428af5923..7e1870cd3c15 100644
--- a/sdks/python/apache_beam/io/watch_test.py
+++ b/sdks/python/apache_beam/io/watch_test.py
@@ -386,32 +386,6 @@ def test_allowed_lateness_retains_keys_below_the_cursor(self):
         lateness)
     self.assertEqual(['late'], [o.value for o in late.outputs])
 
-  def test_a_retired_key_returning_later_is_emitted_again(self):
-    # What bounding the state costs. A key is retired by the event time it was
-    # recorded with, so a key that comes back at a later event time, after the
-    # cursor has moved past the one it was recorded with, has nothing left to
-    # prove it was seen. This is the case for a file modified after the cursor
-    # passed it: it is emitted a second time, whatever the key function says
-    # about updates. Keep the default hash dedup where that matters.
-    state = _initial_polling()
-    tracker = _cursor_tracker(state)
-    first = _cursor_results(state, PollResult.incomplete([_ts('a', 10)]))
-    self.assertTrue(tracker.try_claim((first, 0)))
-    _, residual = tracker.try_split(0)
-    # 'b' moves the cursor past the event time 'a' was recorded with, which
-    # retires 'a'.
-    second = _cursor_results(
-        residual, PollResult.incomplete([_ts('a', 10), _ts('b', 20)]))
-    self.assertEqual(['b'], [o.value for o in second.outputs])
-    resumed = _cursor_tracker(residual)
-    self.assertTrue(resumed.try_claim((second, 0)))
-    _, residual = resumed.try_split(0)
-    self.assertEqual([Timestamp(20)], list(residual.completed.values()))
-    # 'a' now returns above the floor, so it reads as new.
-    third = _cursor_results(
-        residual, PollResult.incomplete([_ts('a', 30), _ts('b', 20)]))
-    self.assertEqual(['a'], [o.value for o in third.outputs])
-
   def test_relist_emits_each_output_exactly_once(self):
     # A full re-list of a growing collection at strictly increasing event
     # times emits each output once; the key set stays bounded throughout.

From 626ee74c97ee9c225edd6463cfc56c5e5760bfc8 Mon Sep 17 00:00:00 2001
From: Elia LIU <Elialiu760317@outlook.com>
Date: Thu, 13 Aug 2026 14:51:33 +1000
Subject: [PATCH 13/16] Release the mtime watermark when a poll finds nothing
 newer

Holding the watermark at the newest last-modified time matched stalls a
directory that is quiet rather than empty: every poll re-lists the same files,
so the newest one never moves and the watermark sits at its last-modified time
while the poll time runs away from it. Only an empty match released it, which
is not the case that stalls.

The hold now follows the evidence. A poll that turns up a last-modified time
newer than any before it has just read the filesystem clock, so the watermark
stops there and files still in flight behind a clock that lags the local one
are not late. A poll that finds nothing newer has no fresh reading to go on, so
the watermark takes the poll time and event-time windows keep closing. A
continuously fed directory therefore trails the filesystem clock throughout,
and a quiet one catches up.

Also records what bounding the deduplication state costs: a file modified after
its id was retired reads as new and is matched a second time, whatever
match_updated_files says. The MatchContinuously test that covered the update
case never advanced the cursor, so it did not reach the retirement; the Watch
test added here pins it.
---
 sdks/python/apache_beam/io/fileio.py      | 48 ++++++++++++++++-------
 sdks/python/apache_beam/io/fileio_test.py | 41 ++++++++++++++++++-
 sdks/python/apache_beam/io/watch_test.py  | 26 ++++++++++++
 3 files changed, 98 insertions(+), 17 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index 127a4ccd2852..5f99feeda27f 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -320,9 +320,9 @@ class _MatchContinuouslyPollFn(PollFn):
   as their event time, and the watermark advances to the poll time so
   event-time windows progress even when nothing new matches. Under
   ``mtime_timestamps`` a match carries its last-modified time instead, which
-  is what bounds the timestamp cursor, and the watermark is the newest
-  last-modified time matched, capped at the poll time. A poll that matches
-  nothing takes the poll time.
+  is what bounds the timestamp cursor, and the watermark trails the newest
+  last-modified time while that time keeps advancing, otherwise it takes the
+  poll time.
   """
   def __init__(
       self,
@@ -334,6 +334,9 @@ def __init__(
     self._start_micros = Timestamp.of(start_timestamp).micros
     self._clock = clock if clock is not None else _PollClock()
     self._mtime_timestamps = mtime_timestamps
+    # Greatest last-modified time handed out so far, to tell a poll that found
+    # something newer from one that only re-listed what was already there.
+    self._newest_mtime = None  # type: Optional[Timestamp]
 
   def __call__(self, file_pattern: str) -> PollResult[filesystem.FileMetadata]:
     now = Timestamp.now()
@@ -353,12 +356,21 @@ def __call__(self, file_pattern: str) -> PollResult[filesystem.FileMetadata]:
         TimestampedValue(metadata, Timestamp.of(_ensure_mtime(metadata)))
         for metadata in match_result.metadata_list
     ]
-    # The filesystem clock can run behind the local one, so the watermark stops
-    # at the newest last-modified time matched, and never runs past the poll
-    # time when that clock is ahead. Matching nothing gives no reading of that
-    # clock, and the poll time keeps event-time windows progressing.
-    newest = max((output.timestamp for output in outputs), default=now)
-    return PollResult.incomplete(outputs).with_watermark(min(newest, now))
+    # A poll that turned up a newer last-modified time than any before it just
+    # read the filesystem clock, so the watermark stops there rather than at
+    # the poll time, and files still in flight behind a filesystem clock that
+    # lags the local one are not late. It is also capped at the poll time, so a
+    # clock running ahead cannot carry the watermark with it. A poll that found
+    # nothing newer has no fresh reading to go on, so the watermark takes the
+    # poll time and a quiet directory does not stall event-time windows.
+    newest = max((output.timestamp for output in outputs), default=None)
+    if newest is not None and (self._newest_mtime is None or
+                               newest > self._newest_mtime):
+      self._newest_mtime = newest
+      watermark = min(newest, now)
+    else:
+      watermark = now
+    return PollResult.incomplete(outputs).with_watermark(watermark)
 
 
 class MatchContinuously(beam.PTransform):
@@ -380,9 +392,12 @@ class MatchContinuously(beam.PTransform):
 
   A match carries the poll time as its event time, and the watermark follows
   the poll time. Under ``timestamp_cursor`` a match carries its last-modified
-  time and the watermark is the newest one matched, capped at the poll time,
-  so a filesystem clock ahead of the local one cannot carry the watermark with
-  it.
+  time, and the watermark trails the newest last-modified time for as long as
+  polls keep turning up newer files, so files still in flight behind a
+  filesystem clock that lags the local one are not late. It is capped at the
+  poll time, so a filesystem clock ahead of the local one cannot carry the
+  watermark with it, and a poll that turns up nothing newer releases it to the
+  poll time, so a quiet directory does not stall event-time windows.
   """
   def __init__(
       self,
@@ -414,9 +429,12 @@ def __init__(
         than one id per file the pattern has ever matched. A file whose
         last-modified time is older than that mark is taken as already seen and
         skipped, as happens with copies that preserve the source time and with
-        backfills of older files. Requires the filesystem to report
-        last-modified times, and matches then carry their last-modified time as
-        their event time instead of the poll time.
+        backfills of older files. Bounding the state this way costs the
+        guarantee that a file is matched once for all time: a file modified
+        after its id was retired looks new again and is matched a second time,
+        whatever ``match_updated_files`` says. Requires the filesystem to
+        report last-modified times, and matches then carry their last-modified
+        time as their event time instead of the poll time.
     """
 
     self.file_pattern = file_pattern
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index da3fca1ac03d..8d3642fc74ec 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -563,6 +563,39 @@ def test_poll_fn_holds_the_mtime_watermark_at_the_newest_match(self):
     result = poll_fn(FileSystems.join(tempdir, '*'))
     self.assertEqual(Timestamp.of(2345.5), result.watermark)
 
+  def test_poll_fn_releases_the_mtime_watermark_once_nothing_is_newer(self):
+    # Holding at the newest match forever would stall a directory that is
+    # merely quiet rather than empty: every poll re-lists the same old files,
+    # and the watermark would sit at their last-modified time while the poll
+    # time ran away from it, so event-time windows would never close.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    os.utime(self._create_temp_file(dir=tempdir), (1234.5, 1234.5))
+    poll_fn = fileio._MatchContinuouslyPollFn(
+        fileio.EmptyMatchTreatment.ALLOW,
+        Timestamp.now() - 3600,
+        mtime_timestamps=True)
+    pattern = FileSystems.join(tempdir, '*')
+    self.assertEqual(Timestamp.of(1234.5), poll_fn(pattern).watermark)
+    before = Timestamp.now()
+    quiet = poll_fn(pattern)
+    self.assertEqual(1, len(quiet.outputs))
+    self.assertTrue(before <= quiet.watermark <= Timestamp.now())
+
+  def test_poll_fn_holds_the_mtime_watermark_again_for_a_newer_match(self):
+    # A poll that does turn up a newer file has read the filesystem clock
+    # again, so the hold comes back rather than being spent once.
+    tempdir = '%s%s' % (self._new_tempdir(), os.sep)
+    os.utime(self._create_temp_file(dir=tempdir), (1234.5, 1234.5))
+    poll_fn = fileio._MatchContinuouslyPollFn(
+        fileio.EmptyMatchTreatment.ALLOW,
+        Timestamp.now() - 3600,
+        mtime_timestamps=True)
+    pattern = FileSystems.join(tempdir, '*')
+    poll_fn(pattern)
+    poll_fn(pattern)
+    os.utime(self._create_temp_file(dir=tempdir), (2345.5, 2345.5))
+    self.assertEqual(Timestamp.of(2345.5), poll_fn(pattern).watermark)
+
   def test_poll_fn_caps_the_mtime_watermark_at_the_poll_time(self):
     # A filesystem clock running ahead must not carry the watermark with it,
     # which pins the watermark to the poll time.
@@ -695,9 +728,13 @@ def _create_twin(element):
 
       assert_that(match_continiously, equal_to([first, twin]))
 
-  def test_timestamp_cursor_leaves_updated_files_out_by_default(self):
+  def test_timestamp_cursor_leaves_updated_files_out_while_their_id_is_held(
+      self):
     # match_updated_files still decides whether a changed file counts as new,
-    # so an update is skipped unless it is asked for.
+    # so an update is skipped unless it is asked for. That only holds while
+    # the file's id is still retained: once the cursor has moved past it and
+    # retired it, a later update looks new again. See the retirement test in
+    # watch_test.py.
     tempdir = '%s%s' % (self._new_tempdir(), os.sep)
     path = self._create_temp_file(dir=tempdir)
     os.utime(path, (1234.5, 1234.5))
diff --git a/sdks/python/apache_beam/io/watch_test.py b/sdks/python/apache_beam/io/watch_test.py
index 7e1870cd3c15..a44428af5923 100644
--- a/sdks/python/apache_beam/io/watch_test.py
+++ b/sdks/python/apache_beam/io/watch_test.py
@@ -386,6 +386,32 @@ def test_allowed_lateness_retains_keys_below_the_cursor(self):
         lateness)
     self.assertEqual(['late'], [o.value for o in late.outputs])
 
+  def test_a_retired_key_returning_later_is_emitted_again(self):
+    # What bounding the state costs. A key is retired by the event time it was
+    # recorded with, so a key that comes back at a later event time, after the
+    # cursor has moved past the one it was recorded with, has nothing left to
+    # prove it was seen. This is the case for a file modified after the cursor
+    # passed it: it is emitted a second time, whatever the key function says
+    # about updates. Keep the default hash dedup where that matters.
+    state = _initial_polling()
+    tracker = _cursor_tracker(state)
+    first = _cursor_results(state, PollResult.incomplete([_ts('a', 10)]))
+    self.assertTrue(tracker.try_claim((first, 0)))
+    _, residual = tracker.try_split(0)
+    # 'b' moves the cursor past the event time 'a' was recorded with, which
+    # retires 'a'.
+    second = _cursor_results(
+        residual, PollResult.incomplete([_ts('a', 10), _ts('b', 20)]))
+    self.assertEqual(['b'], [o.value for o in second.outputs])
+    resumed = _cursor_tracker(residual)
+    self.assertTrue(resumed.try_claim((second, 0)))
+    _, residual = resumed.try_split(0)
+    self.assertEqual([Timestamp(20)], list(residual.completed.values()))
+    # 'a' now returns above the floor, so it reads as new.
+    third = _cursor_results(
+        residual, PollResult.incomplete([_ts('a', 30), _ts('b', 20)]))
+    self.assertEqual(['a'], [o.value for o in third.outputs])
+
   def test_relist_emits_each_output_exactly_once(self):
     # A full re-list of a growing collection at strictly increasing event
     # times emits each output once; the key set stays bounded throughout.

From 7669d4abbe952f3c9042cf6970e431edac24d57d Mon Sep 17 00:00:00 2001
From: Eliaazzz <Elialiu760317@outlook.com>
Date: Sat, 15 Aug 2026 09:58:05 +1000
Subject: [PATCH 14/16] [Python] Key the MatchContinuously cursor on the path
 and mtime

Keying on the path alone left an update out while its key was retained and
let one through once the cursor had retired the key, so whether an updated
file matched depended on cursor timing.

The cursor now keys on the path and the last-modified time. An update is a
new key and is always matched. Retiring a key can no longer duplicate a
match either, since the only file that would recreate a retired key carries
the same last-modified time and is skipped by the same mark.
---
 sdks/python/apache_beam/io/fileio.py      | 26 ++++++++++++-----------
 sdks/python/apache_beam/io/fileio_test.py | 14 ++++++------
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index 5f99feeda27f..3e9cd4af7981 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -423,16 +423,16 @@ def __init__(
       apply_windowing: Whether each element should be assigned to
         individual window. If false, all elements will reside in global window.
       timestamp_cursor: (When has_deduplication is set to True) bound the
-        deduplication state by last-modified time. Files are still deduplicated
-        by id, but an id is retired once a newer file has been matched, so the
-        state holds the newest last-modified time and the ids sharing it rather
-        than one id per file the pattern has ever matched. A file whose
-        last-modified time is older than that mark is taken as already seen and
-        skipped, as happens with copies that preserve the source time and with
-        backfills of older files. Bounding the state this way costs the
-        guarantee that a file is matched once for all time: a file modified
-        after its id was retired looks new again and is matched a second time,
-        whatever ``match_updated_files`` says. Requires the filesystem to
+        deduplication state by last-modified time. Files are deduplicated by
+        path and last-modified time, and a key is retired once the newest match
+        has moved past it, so the state holds a trailing window rather than one
+        key per file the pattern has ever matched. Retiring a key cannot
+        duplicate a match, because the only file that would recreate it carries
+        the same last-modified time and is skipped by the same mark. A file
+        whose last-modified time is older than that mark is taken as already
+        seen and skipped, as happens with copies that preserve the source time
+        and with backfills of older files. Implies the ``match_updated_files``
+        key, so an updated file is matched again. Requires the filesystem to
         report last-modified times, and matches then carry their last-modified
         time as their event time instead of the poll time.
     """
@@ -474,7 +474,8 @@ def expand(self, pbegin) -> beam.PCollection[filesystem.FileMetadata]:
   def _match_deduplicated(self,
                           pbegin) -> beam.PCollection[filesystem.FileMetadata]:
     # Watch emits each file once per dedup key: the path, joined by the mtime
-    # when matching updated files, or the mtime alone under timestamp_cursor.
+    # when matching updated files or under timestamp_cursor, which needs the
+    # mtime in the key so that retiring a key cannot duplicate a match.
     # stop_timestamp bounds the polls to [start, stop).
     clock = _PollClock()
     if self.stop_ts == MAX_TIMESTAMP:
@@ -507,7 +508,8 @@ def _match_deduplicated(self,
         poll_interval=self.interval,
         termination=termination,
         output_key_fn=(
-            _file_path_and_mtime_key if self.match_upd else _file_path_key),
+            _file_path_and_mtime_key
+            if self.match_upd or self.timestamp_cursor else _file_path_key),
         timestamp_cursor=self.timestamp_cursor)
     # Watch emits (pattern, file) pairs; keep the FileMetadata output type so
     # downstream transforms stay typed instead of falling back to Any.
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index 8d3642fc74ec..ba56d3ec0b53 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -728,13 +728,11 @@ def _create_twin(element):
 
       assert_that(match_continiously, equal_to([first, twin]))
 
-  def test_timestamp_cursor_leaves_updated_files_out_while_their_id_is_held(
-      self):
-    # match_updated_files still decides whether a changed file counts as new,
-    # so an update is skipped unless it is asked for. That only holds while
-    # the file's id is still retained: once the cursor has moved past it and
-    # retired it, a later update looks new again. See the retirement test in
-    # watch_test.py.
+  def test_timestamp_cursor_emits_an_updated_file(self):
+    # The cursor keys on the path and the last-modified time, so an update is
+    # a new key and is matched again. Keying on the path alone would leave the
+    # update out while the key is retained and let it through once the cursor
+    # retired the key, which made the outcome depend on cursor timing.
     tempdir = '%s%s' % (self._new_tempdir(), os.sep)
     path = self._create_temp_file(dir=tempdir)
     os.utime(path, (1234.5, 1234.5))
@@ -758,7 +756,7 @@ def _touch(element):
               timestamp_cursor=True)
           | beam.Map(_touch))
 
-      assert_that(match_continiously, equal_to([path]))
+      assert_that(match_continiously, equal_to([path, path]))
 
 
 class WriteFilesTest(_TestCaseWithTempDirCleanUp):

From 1a56c7d9121a1a3030b0d15303520f941e26c10e Mon Sep 17 00:00:00 2001
From: Eliaazzz <Elialiu760317@outlook.com>
Date: Mon, 17 Aug 2026 23:36:51 +1000
Subject: [PATCH 15/16] [Python] Imply match_updated_files under the timestamp
 cursor

MatchContinuously(timestamp_cursor=True) now warns and sets
match_updated_files=True instead of leaving the two options to disagree,
so the key function consults match_updated_files alone.
---
 sdks/python/apache_beam/io/fileio.py      | 24 +++++++++++++----------
 sdks/python/apache_beam/io/fileio_test.py |  7 +++++++
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index 3e9cd4af7981..0a6a2daec62a 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -446,11 +446,17 @@ def __init__(
     self.apply_windowing = apply_windowing
     self.empty_match_treatment = empty_match_treatment
     self.timestamp_cursor = timestamp_cursor
-    if timestamp_cursor and not has_deduplication:
-      raise ValueError(
-          'MatchContinuously(timestamp_cursor=True) deduplicates, so it '
-          'requires has_deduplication=True.')
-    if not timestamp_cursor:
+    if timestamp_cursor:
+      if not has_deduplication:
+        raise ValueError(
+            'MatchContinuously(timestamp_cursor=True) deduplicates, so it '
+            'requires has_deduplication=True.')
+      if not match_updated_files:
+        _LOGGER.warning(
+            'MatchContinuously(timestamp_cursor=True) implies '
+            'match_updated_files=True.')
+        self.match_upd = True
+    else:
       _LOGGER.warning(
           'Matching Continuously is stateful, and can scale poorly. '
           'Consider using Pub/Sub Notifications '
@@ -474,9 +480,8 @@ def expand(self, pbegin) -> beam.PCollection[filesystem.FileMetadata]:
   def _match_deduplicated(self,
                           pbegin) -> beam.PCollection[filesystem.FileMetadata]:
     # Watch emits each file once per dedup key: the path, joined by the mtime
-    # when matching updated files or under timestamp_cursor, which needs the
-    # mtime in the key so that retiring a key cannot duplicate a match.
-    # stop_timestamp bounds the polls to [start, stop).
+    # when matching updated files. stop_timestamp bounds the polls to
+    # [start, stop).
     clock = _PollClock()
     if self.stop_ts == MAX_TIMESTAMP:
       termination = never()
@@ -508,8 +513,7 @@ def _match_deduplicated(self,
         poll_interval=self.interval,
         termination=termination,
         output_key_fn=(
-            _file_path_and_mtime_key
-            if self.match_upd or self.timestamp_cursor else _file_path_key),
+            _file_path_and_mtime_key if self.match_upd else _file_path_key),
         timestamp_cursor=self.timestamp_cursor)
     # Watch emits (pattern, file) pairs; keep the FileMetadata output type so
     # downstream transforms stay typed instead of falling back to Any.
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index ba56d3ec0b53..1c650653804c 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -635,6 +635,13 @@ def test_timestamp_cursor_requires_deduplication(self):
       fileio.MatchContinuously(
           file_pattern='/tmp/*', has_deduplication=False, timestamp_cursor=True)
 
+  def test_timestamp_cursor_implies_matching_updated_files(self):
+    # timestamp_cursor forces match_updated_files=True, so an update is a new
+    # key whatever the caller passed.
+    match = fileio.MatchContinuously(
+        file_pattern='/tmp/*', timestamp_cursor=True)
+    self.assertTrue(match.match_upd)
+
   def test_timestamp_cursor_emits_files_modified_past_the_cursor(self):
     files = []
     tempdir = '%s%s' % (self._new_tempdir(), os.sep)

From acbf862ab64bf82d69ccf9fa126da7afff7faa92 Mon Sep 17 00:00:00 2001
From: Eliaazzz <Elialiu760317@outlook.com>
Date: Mon, 17 Aug 2026 23:37:14 +1000
Subject: [PATCH 16/16] [Python] Cut the timestamp cursor docs down to the
 contract

The user-facing docs described the retention mechanism. State what a
caller observes instead: the option bounds the deduplication state for
better performance, and a file arriving with an older last-modified time
is skipped.
---
 sdks/python/apache_beam/io/fileio.py | 59 ++++++++++------------------
 sdks/python/apache_beam/io/watch.py  | 27 ++++++-------
 2 files changed, 33 insertions(+), 53 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index 0a6a2daec62a..fcce83fa59ec 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -317,12 +317,9 @@ class _MatchContinuouslyPollFn(PollFn):
   """Polls a file pattern, honoring empty-match rules.
 
   A poll before ``start_timestamp`` emits nothing. Matches carry the poll time
-  as their event time, and the watermark advances to the poll time so
-  event-time windows progress even when nothing new matches. Under
-  ``mtime_timestamps`` a match carries its last-modified time instead, which
-  is what bounds the timestamp cursor, and the watermark trails the newest
-  last-modified time while that time keeps advancing, otherwise it takes the
-  poll time.
+  as their event time, or their last-modified time under ``mtime_timestamps``,
+  where the watermark trails the newest last-modified time for as long as
+  polls keep turning up newer ones.
   """
   def __init__(
       self,
@@ -356,13 +353,10 @@ def __call__(self, file_pattern: str) -> PollResult[filesystem.FileMetadata]:
         TimestampedValue(metadata, Timestamp.of(_ensure_mtime(metadata)))
         for metadata in match_result.metadata_list
     ]
-    # A poll that turned up a newer last-modified time than any before it just
-    # read the filesystem clock, so the watermark stops there rather than at
-    # the poll time, and files still in flight behind a filesystem clock that
-    # lags the local one are not late. It is also capped at the poll time, so a
-    # clock running ahead cannot carry the watermark with it. A poll that found
-    # nothing newer has no fresh reading to go on, so the watermark takes the
-    # poll time and a quiet directory does not stall event-time windows.
+    # A poll that turned up a newer last-modified time has just read the
+    # filesystem clock, so the watermark stops there, capped at the poll time.
+    # A poll that found nothing newer takes the poll time, so a quiet
+    # directory does not stall event-time windows.
     newest = max((output.timestamp for output in outputs), default=None)
     if newest is not None and (self._newest_mtime is None or
                                newest > self._newest_mtime):
@@ -382,22 +376,16 @@ class MatchContinuously(beam.PTransform):
   MatchContinuously is experimental.  No backwards-compatibility
   guarantees.
 
-  Deduplication state lives in the splittable DoFn restriction, so a runner
-  with checkpointing enabled restores it after a restart and does not
-  reprocess files. That state holds one id per matched file and grows with the
-  directory, unless ``timestamp_cursor`` bounds it to the newest matched
-  last-modified time. For a growing directory on GCS, consider an alternate
-  technique such as Pub/Sub Notifications
-  (https://cloud.google.com/storage/docs/pubsub-notifications).
+  Deduplication state is checkpointed, so a runner with checkpointing enabled
+  restores it after a restart and does not reprocess files. That state grows
+  with the number of files matched, unless ``timestamp_cursor`` bounds it. For
+  a growing directory on GCS, consider an alternate technique such as Pub/Sub
+  Notifications (https://cloud.google.com/storage/docs/pubsub-notifications).
 
   A match carries the poll time as its event time, and the watermark follows
   the poll time. Under ``timestamp_cursor`` a match carries its last-modified
-  time, and the watermark trails the newest last-modified time for as long as
-  polls keep turning up newer files, so files still in flight behind a
-  filesystem clock that lags the local one are not late. It is capped at the
-  poll time, so a filesystem clock ahead of the local one cannot carry the
-  watermark with it, and a poll that turns up nothing newer releases it to the
-  poll time, so a quiet directory does not stall event-time windows.
+  time instead, and the watermark holds at the newest one matched, capped at
+  the poll time, until a poll turns up nothing newer and releases it.
   """
   def __init__(
       self,
@@ -422,19 +410,12 @@ def __init__(
         file with timestamp changes.
       apply_windowing: Whether each element should be assigned to
         individual window. If false, all elements will reside in global window.
-      timestamp_cursor: (When has_deduplication is set to True) bound the
-        deduplication state by last-modified time. Files are deduplicated by
-        path and last-modified time, and a key is retired once the newest match
-        has moved past it, so the state holds a trailing window rather than one
-        key per file the pattern has ever matched. Retiring a key cannot
-        duplicate a match, because the only file that would recreate it carries
-        the same last-modified time and is skipped by the same mark. A file
-        whose last-modified time is older than that mark is taken as already
-        seen and skipped, as happens with copies that preserve the source time
-        and with backfills of older files. Implies the ``match_updated_files``
-        key, so an updated file is matched again. Requires the filesystem to
-        report last-modified times, and matches then carry their last-modified
-        time as their event time instead of the poll time.
+      timestamp_cursor: (When match_updated_files and has_deduplication are set
+        to True) bound the deduplication state by last-modified time. By
+        default, all file modification history is tracked. If set to true, file
+        modification history prior to the max(mtime of last poll result) are
+        dropped, for better performance. A file that appears with an older
+        last-modified time is then taken as already seen and skipped.
     """
 
     self.file_pattern = file_pattern
diff --git a/sdks/python/apache_beam/io/watch.py b/sdks/python/apache_beam/io/watch.py
index 827edde7efd8..70adaa444178 100644
--- a/sdks/python/apache_beam/io/watch.py
+++ b/sdks/python/apache_beam/io/watch.py
@@ -36,9 +36,8 @@
 
 By default, the Watch transform internally stores the hash of all items
 seen. If the items returned by the poll function arrive in roughly
-non-decreasing event time, consider setting ``timestamp_cursor=True``, which
-retires a hash once the greatest emitted event time has moved past it and so
-holds a trailing window instead of every item ever seen; see :class:`Watch`.
+non-decreasing event time, consider setting ``timestamp_cursor=True`` for
+better performance; see :class:`Watch`.
 
 Example::
 
@@ -830,17 +829,17 @@ class Watch(PTransform):
       inferred like ``output_coder`` when omitted. It is converted with
       ``as_deterministic_coder`` so equal keys always hash equally; a coder
       with no deterministic form is rejected.
-    timestamp_cursor: bound the dedup state by event time. Dedup still goes by
-      key, but a key is retired once the greatest emitted event time has moved
-      more than ``allowed_lateness`` past it, so the state holds a trailing
-      window rather than every key ever seen. An output below that mark is
-      taken as already seen and dropped, so this suits sources whose outputs
-      arrive in roughly non-decreasing event time; keep the default for
-      sources that can hand out much older outputs at any time.
-    allowed_lateness: how far below the cursor a key is still retained, as a
-      :class:`Duration` or in seconds. Widen it for a source whose outputs
-      arrive out of order, at the cost of a larger state. Ignored unless
-      ``timestamp_cursor`` is set; defaults to zero.
+    timestamp_cursor: bound the dedup state by event time, for better
+      performance. An output more than ``allowed_lateness`` behind the greatest
+      event time emitted so far is taken as already seen and dropped, so this
+      suits sources whose outputs arrive in roughly non-decreasing event time;
+      keep the default for sources that can hand out much older outputs at any
+      time.
+    allowed_lateness: how far behind the greatest emitted event time an output
+      is still deduplicated by key, as a :class:`Duration` or in seconds.
+      Widen it for a source whose outputs arrive out of order, at the cost of a
+      larger state. Ignored unless ``timestamp_cursor`` is set; defaults to
+      zero.
     now_fn: clock used for termination decisions; tests can inject one.
   """
   def __init__(