diff --git a/dogsdogsdogs/examples/delta_query2.rs b/dogsdogsdogs/examples/delta_query2.rs index f29a95a1a..b9c53d0a8 100644 --- a/dogsdogsdogs/examples/delta_query2.rs +++ b/dogsdogsdogs/examples/delta_query2.rs @@ -43,7 +43,7 @@ fn main() { changes1, forward2, closure, - |t1,t2| t1.lt(t2), // This one ignores concurrent updates. + true, // This one ignores concurrent updates. |key, val1, val2| (key.clone(), (val1.clone(), val2.clone())), ); @@ -52,7 +52,7 @@ fn main() { changes2, forward1, closure, - |t1,t2| t1.le(t2), // This one can "see" concurrent updates. + false, // This one can "see" concurrent updates. |key, val1, val2| (key.clone(), (val2.clone(), val1.clone())), ); diff --git a/dogsdogsdogs/src/operators/count.rs b/dogsdogsdogs/src/operators/count.rs index 8aa0eebc7..3ab2bfbb5 100644 --- a/dogsdogsdogs/src/operators/count.rs +++ b/dogsdogsdogs/src/operators/count.rs @@ -63,16 +63,10 @@ where builder.push_into(((triple, payload.clone()), initial.clone(), diff1.clone())); }; - use crate::operators::half_join::half_join_internal_unsafe as half_join_unsafe; - // Branch once here, so that each comparison monomorphizes rather than testing `strict` at - // every timestamp. The cost is instantiating `half_join` twice. - if strict { - half_join_unsafe::<_, _, _, _, _, _, _, _, Output>( - requests, arrangement, frontier_func, |t1, t2| t1 < t2, |_timer, _count| false, output_func) - } - else { - half_join_unsafe::<_, _, _, _, _, _, _, _, Output>( - requests, arrangement, frontier_func, |t1, t2| t1 <= t2, |_timer, _count| false, output_func) - } + use crate::operators::half_join::cursors::half_join_internal_unsafe as half_join_unsafe; + // `strict` now reaches the join as a value rather than as a comparison closure, so there is + // nothing left to monomorphize by branching here; the test is made per arrangement time. + half_join_unsafe::<_, _, _, _, _, _, _, Output>( + requests, arrangement, frontier_func, strict, |_timer, _count| false, output_func) .as_collection() } diff --git a/dogsdogsdogs/src/operators/half_join.rs b/dogsdogsdogs/src/operators/half_join.rs index 102a0f108..f1ee2a07d 100644 --- a/dogsdogsdogs/src/operators/half_join.rs +++ b/dogsdogsdogs/src/operators/half_join.rs @@ -1,368 +1,551 @@ -/// Streaming asymmetric join between updates (K,V1) and an arrangement on (K,V2). +/// Streaming asymmetric join between an update stream and a maintained arrangement. /// /// The asymmetry is that the join only responds to streamed updates, not to changes in the arrangement. -/// Streamed updates join only with matching arranged updates at lesser times *in the total order*, and -/// subject to a predicate supplied by the user (roughly: strictly less, or not). +/// The operator is the basis of several multi-way join implementations, which use networks of stateless +/// operators rather than sequences of stateful operators. /// -/// This behavior can ensure that any pair of matching updates interact exactly once. +/// The standard recipe for a multi-way join among collections A, B, .. Z is to create a dataflow path for +/// each collection, which responds to changes in the collection. Each path uses a sequence of half join +/// operators to prompt the response to each streamed input change when joined with "prior" updates from +/// the other collections. A total order on update times, often just the `Ord` implementation, can then be +/// extended to all relations to impose a total order on updates to any of the collections, which can then +/// ensure that each tuple of updates is processed exactly once (usually at the time of the "last" update). /// -/// There are various forms of this operator with tangled closures about how to emit the outputs and -/// wrangle the logical compaction frontier in order to preserve the distinctions around times that are -/// strictly less (conventional compaction logic would collapse unequal times to the frontier, and lose -/// the distiction). -/// -/// The methods also carry an auxiliary time next to the value, which is used to advance the joined times. -/// This is .. a byproduct of wanting to allow advancing times a la `join_function`, without breaking the -/// coupling by total order on "initial time". -/// -/// The doccomments for individual methods are a bit of a mess. Sorry. +/// The operator design is foremost as a "tactic", which allows an implementor to supply the understanding +/// of the streamed update containers and the maintained trace batches, without complicating the operator. +/// The implementation requires an explanation of how to mark updates as "eligible", via the `Batcher` trait, +/// and then what to do with the eligible updates and existing batches, through the `HalfJoinTactic` trait. use std::collections::VecDeque; use std::ops::Mul; -use timely::ContainerBuilder; -use timely::container::CapacityContainerBuilder; +use timely::{Container, ContainerBuilder}; +use timely::container::{CapacityContainerBuilder, NoopBuilder}; use timely::dataflow::Stream; -use timely::dataflow::channels::pact::{Pipeline, Exchange}; +use timely::dataflow::channels::pact::{ParallelizationContract, Pipeline}; use timely::dataflow::operators::Operator; +use timely::dataflow::operators::generic::OutputBuilderSession; use timely::PartialOrder; -use timely::progress::{Antichain, ChangeBatch, Timestamp}; -use timely::progress::frontier::MutableAntichain; +use timely::progress::{Antichain, Timestamp}; +use timely::progress::frontier::{AntichainRef, MutableAntichain}; use differential_dataflow::{ExchangeData, VecCollection, AsCollection, Hashable}; -use differential_dataflow::difference::{Monoid, Semigroup}; +use differential_dataflow::difference::Semigroup; use differential_dataflow::lattice::Lattice; use differential_dataflow::operators::arrange::Arranged; -use differential_dataflow::trace::{BatchCursor, BatchDiff, BatchTimeGat, BatchVal, Cursor, Navigable, TraceReader}; +use differential_dataflow::trace::{BatchReader, BatchCursor, BatchDiff, BatchVal, Cursor, Navigable, TraceReader}; use differential_dataflow::trace::cursor::cursor_list; use differential_dataflow::consolidation::{consolidate, consolidate_updates}; use differential_dataflow::trace::implementations::BatchContainer; use timely::dataflow::operators::CapabilitySet; -/// A binary equijoin that responds to updates on only its first input. -/// -/// This operator responds to inputs of the form -/// -/// ```ignore -/// ((key, val1, time1), initial_time, diff1) -/// ``` -/// -/// where `initial_time` is less or equal to `time1`, and produces as output +/// An implementation suitable to define half-join behavior among its referenced types. /// -/// ```ignore -/// ((output_func(key, val1, val2), lub(time1, time2)), initial_time, diff1 * diff2) -/// ``` -/// -/// for each `((key, val2), time2, diff2)` present in `arrangement`, where -/// `time2` is less than `initial_time` *UNDER THE TOTAL ORDER ON TIMES*. -/// This last constraint is important to ensure that we correctly produce -/// all pairs of output updates across multiple `half_join` operators. -/// -/// Notice that the time is hoisted up into data. The expectation is that -/// once out of the "delta flow region", the updates will be `delay`d to the -/// times specified in the payloads. -pub fn half_join<'scope, K, V, R, Tr, FF, CF, DOut, S>( - stream: VecCollection<'scope, Tr::Time, (K, V, Tr::Time), R>, - arrangement: Arranged<'scope, Tr>, - frontier_func: FF, - comparison: CF, - mut output_func: S, -) -> VecCollection<'scope, Tr::Time, (DOut, Tr::Time), >>::Output> -where - K: Hashable + ExchangeData, - V: ExchangeData, - R: ExchangeData + Monoid, - Tr: TraceReader+Clone+'static, - BatchCursor: Cursor