From 84f739bde8b56516fd18330468d7447c27d07f5e Mon Sep 17 00:00:00 2001 From: Mancer1 Date: Sat, 13 Jun 2026 17:54:49 +0200 Subject: [PATCH 01/10] PiecewiseLinearUtils.computeBreakpointSuccessive is compressed in O(n) --- .../functional/PiecewiseLinearUtils.java | 612 +++++++++--------- 1 file changed, 316 insertions(+), 296 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java index 7b0b4bfa960..dcf8f6726ea 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java @@ -7,300 +7,320 @@ import java.util.List; public class PiecewiseLinearUtils { - /** - * Utility methods for piecewise linear compression of matric columns - * supports compression used the segmented least squares algorithm which is implemented with dynamic programming - * and a successive method, which puts all values in a segment till the target loss is exceeded - */ - - private PiecewiseLinearUtils() { - - } - - public static final class SegmentedRegression { - private final int[] breakpoints; - private final double[] slopes; - private final double[] intercepts; - - public SegmentedRegression(int[] breakpoints, double[] slopes, double[] intercepts) { - this.breakpoints = breakpoints; - this.slopes = slopes; - this.intercepts = intercepts; - } - - public int[] getBreakpoints() { - return breakpoints; - } - - public double[] getSlopes() { - return slopes; - } - - public double[] getIntercepts() { - return intercepts; - } - } - - public static double[] getColumn(MatrixBlock in, int colIndex) { - final int numRows = in.getNumRows(); - final double[] column = new double[numRows]; - - for(int row = 0; row < numRows; row++) { - column[row] = in.get(row, colIndex); - } - return column; - } - - public static SegmentedRegression compressSegmentedLeastSquares(double[] column, CompressionSettings cs) { - //compute Breakpoints for a Column with dynamic Programming - final List breakpointsList = computeBreakpoints(cs, column); - final int[] breakpoints = breakpointsList.stream().mapToInt(Integer::intValue).toArray(); - - //get values for Regression - final int numSeg = breakpoints.length - 1; - final double[] slopes = new double[numSeg]; - final double[] intercepts = new double[numSeg]; - - // Regress per Segment - for(int seg = 0; seg < numSeg; seg++) { - final int SegStart = breakpoints[seg]; - final int SegEnd = breakpoints[seg + 1]; - - final double[] line = regressSegment(column, SegStart, SegEnd); - slopes[seg] = line[0]; //slope regession line - intercepts[seg] = line[1]; //intercept regression line - } - - return new SegmentedRegression(breakpoints, slopes, intercepts); - } - - public static SegmentedRegression compressSuccessivePiecewiseLinear(double[] column, CompressionSettings cs) { - //compute Breakpoints for a Column with a sukzessive breakpoints algorithm - - final List breakpointsList = computeBreakpointSuccessive(column, cs); - final int[] breakpoints = breakpointsList.stream().mapToInt(Integer::intValue).toArray(); - - //get values for Regression - final int numSeg = breakpoints.length - 1; - final double[] slopes = new double[numSeg]; - final double[] intercepts = new double[numSeg]; - - // Regress per Segment - for(int seg = 0; seg < numSeg; seg++) { - final int segstart = breakpoints[seg]; - final int segEnd = breakpoints[seg + 1]; - final double[] line = regressSegment(column, segstart, segEnd); - slopes[seg] = line[0]; - intercepts[seg] = line[1]; - } - return new SegmentedRegression(breakpoints, slopes, intercepts); - } - - /** - * Computes breakpoints for a column using segmented least squares with dynamic programming - * Iteratively reduces lambda to increase the number of segments until the target MSE is met. - * - * @param cs compression settings containing the target loss - * @param column the column values to segment - * @return list of breakpoint indices, starting with 0 - */ - public static List computeBreakpoints(CompressionSettings cs, double[] column) { - final int numElements = column.length; - final double targetMSE = cs.getPiecewiseTargetLoss(); - final double sseMax = numElements * targetMSE; // max allowed total SSE - - //start with high lambda an reduce iteratively - double lambda = Math.max(10.0, sseMax * 2.0); - List bestBreaks = Arrays.asList(0, numElements); - double bestSSE = computeTotalSSE(column, bestBreaks); - - for (int iter = 0; iter < 50; iter++) { - List breaks = computeBreakpointsLambda(column, lambda); - double totalSSE = computeTotalSSE(column, breaks); - int numSegs = breaks.size() - 1; - - if (totalSSE < bestSSE) { - bestSSE = totalSSE; - bestBreaks = new ArrayList<>(breaks); - } - //target loss reached - if (bestSSE <= sseMax) { - return bestBreaks; - } - - // only one segment left, break condition - if (numSegs <= 1) { - break; - } - // reducing lambda to allow more segments in next iteration - lambda *= 0.8; - } - - return bestBreaks; - } - - /** - * Computes optimal breakpoints, each segment has a SEE plus a - - */ - - public static List computeBreakpointsLambda(double[] column, double lambda) { - final int n = column.length; - final double[] costs = new double[n + 1]; // min cost to reach i - final int[] prev = new int[n + 1]; - - Arrays.fill(costs, Double.POSITIVE_INFINITY); - costs[0] = 0.0; - // precompute all segment costs to avoid recomputation in dynamic programming - double[][] segCosts = new double[n+1][n+1]; - for(int i = 0; i < n; i++) { - for(int j = i+1; j <= n; j++) { - segCosts[i][j] = computeSegmentCost(column, i, j); - } - } - // for each point j, find the cheapest previous breakpoint i - for(int j = 1; j <= n; j++) { - for(int i = 0; i < j; i++) { - // cost equals the SSE of segment [i,j] plus penalty plus best costs - double cost = costs[i] + segCosts[i][j] + lambda; - if(cost < costs[j]) { - costs[j] = cost; - prev[j] = i; - } - } - } - - // Backtrack to previous points to recover the breakpoints - List breaks = new ArrayList<>(); - int j = n; - while(j > 0) { - breaks.add(j); - j = prev[j]; - } - breaks.add(0); - Collections.reverse(breaks); - return breaks; - } - - /** - * computes the segment cost - * @param column column values - * @param start start index - * @param end end index - * @return SSE of the regression line over the segment - */ - public static double computeSegmentCost(double[] column, int start, int end) { - final int segSize = end - start; - if(segSize <= 1) - return 0.0; - - final double[] ab = regressSegment(column, start, end); - final double slope = ab[0]; - final double intercept = ab[1]; - - double sse = 0.0; - for(int i = start; i < end; i++) { - double err = column[i] - (slope * i + intercept); - sse += err * err; - } - return sse; - } - - /** - * computes the total SSE over all segments defined by the given breakpoints - * @param column - * @param breaks - * @return sum of the total SSE - */ - public static double computeTotalSSE(double[] column, List breaks) { - double total = 0.0; - for(int s = 0; s < breaks.size() - 1; s++) { - final int start = breaks.get(s); - final int end = breaks.get(s + 1); - total += computeSegmentCost(column, start, end); - } - return total; - } - - public static double[] regressSegment(double[] column, int start, int end) { - final int numElements = end - start; - if(numElements <= 0) - return new double[] {0.0, 0.0}; - - double sumOfRowIndices = 0, sumOfColumnValues = 0, sumOfRowIndicesSquared = 0, productRowIndexTimesColumnValue = 0; - for(int i = start; i < end; i++) { - sumOfRowIndices += i; - sumOfColumnValues += column[i]; - sumOfRowIndicesSquared += i * i; - productRowIndexTimesColumnValue += i * column[i]; - } - - - final double denominatorForSlope = - numElements * sumOfRowIndicesSquared - sumOfRowIndices * sumOfRowIndices; - final double slope; - final double intercept; - if(denominatorForSlope == 0) { - slope = 0.0; - intercept = sumOfColumnValues / numElements; - } - else { - slope = (numElements * productRowIndexTimesColumnValue - sumOfRowIndices * sumOfColumnValues) / - denominatorForSlope; - intercept = (sumOfColumnValues - slope * sumOfRowIndices) / numElements; - } - return new double[] {slope, intercept}; - } - - /** - * computes breakpoints for a column using a successive algorithm - * extends each segment until the SEE reaches the target loss, then start a new segment - * @param column column values - * @param cs compression setting for setting the target loss - * @return list of breakpoint indices - */ - public static List computeBreakpointSuccessive(double[] column, CompressionSettings cs) { - final int numElements = column.length; - final double targetMSE = cs.getPiecewiseTargetLoss(); - if (Double.isNaN(targetMSE) || targetMSE <= 0) { - return Arrays.asList(0, numElements); // fallback single segment - } - - List breakpoints = new ArrayList<>(); - breakpoints.add(0); - int currentStart = 0; - - while (currentStart < numElements) { - int bestEnd = -1; // no end found - - for (int end = currentStart + 1; end <= numElements; end++) { - double sse = computeSegmentCost(column, currentStart, end); - if(sse > (end - currentStart) * targetMSE) { - // end-1 is last valid end; if end == segStart+1 force min segment of length 1 - bestEnd = (end == currentStart + 1) ? end : end - 1; - break; - } - } - - if (bestEnd == -1) { - bestEnd = numElements;// all remaining points fitting within budget - } - - // safety guard not allow zero segments - if (bestEnd <= currentStart) { - bestEnd = Math.min(currentStart + 1, numElements); - } - - breakpoints.add(bestEnd); - currentStart = bestEnd; - } - - // make sure, that the last breakpoint equals numElements - int last = breakpoints.get(breakpoints.size() - 1); - if (last != numElements) { - breakpoints.add(numElements); - } - - return breakpoints; - } + /** + * Utility methods for piecewise linear compression of matric columns + * supports compression used the segmented least squares algorithm which is implemented with dynamic programming + * and a successive method, which puts all values in a segment till the target loss is exceeded + */ + + private PiecewiseLinearUtils() { + + } + + public static final class SegmentedRegression { + private final int[] breakpoints; + private final double[] slopes; + private final double[] intercepts; + + public SegmentedRegression(int[] breakpoints, double[] slopes, double[] intercepts) { + this.breakpoints = breakpoints; + this.slopes = slopes; + this.intercepts = intercepts; + } + + public int[] getBreakpoints() { + return breakpoints; + } + + public double[] getSlopes() { + return slopes; + } + + public double[] getIntercepts() { + return intercepts; + } + } + + public static double[] getColumn(MatrixBlock in, int colIndex) { + final int numRows = in.getNumRows(); + final double[] column = new double[numRows]; + + for (int row = 0; row < numRows; row++) { + column[row] = in.get(row, colIndex); + } + return column; + } + + public static SegmentedRegression compressSegmentedLeastSquares(double[] column, CompressionSettings cs) { + //compute Breakpoints for a Column with dynamic Programming + final List breakpointsList = computeBreakpoints(cs, column); + final int[] breakpoints = breakpointsList.stream().mapToInt(Integer::intValue).toArray(); + + //get values for Regression + final int numSeg = breakpoints.length - 1; + final double[] slopes = new double[numSeg]; + final double[] intercepts = new double[numSeg]; + + // Regress per Segment + for (int seg = 0; seg < numSeg; seg++) { + final int SegStart = breakpoints[seg]; + final int SegEnd = breakpoints[seg + 1]; + + final double[] line = regressSegment(column, SegStart, SegEnd); + slopes[seg] = line[0]; //slope regession line + intercepts[seg] = line[1]; //intercept regression line + } + + return new SegmentedRegression(breakpoints, slopes, intercepts); + } + + public static SegmentedRegression compressSuccessivePiecewiseLinear(double[] column, CompressionSettings cs) { + //compute Breakpoints for a Column with a sukzessive breakpoints algorithm + + final List breakpointsList = computeBreakpointSuccessive(column, cs); + final int[] breakpoints = breakpointsList.stream().mapToInt(Integer::intValue).toArray(); + + //get values for Regression + final int numSeg = breakpoints.length - 1; + final double[] slopes = new double[numSeg]; + final double[] intercepts = new double[numSeg]; + + // Regress per Segment + for (int seg = 0; seg < numSeg; seg++) { + final int segstart = breakpoints[seg]; + final int segEnd = breakpoints[seg + 1]; + final double[] line = regressSegment(column, segstart, segEnd); + slopes[seg] = line[0]; + intercepts[seg] = line[1]; + } + return new SegmentedRegression(breakpoints, slopes, intercepts); + } + + /** + * Computes breakpoints for a column using segmented least squares with dynamic programming + * Iteratively reduces lambda to increase the number of segments until the target MSE is met. + * + * @param cs compression settings containing the target loss + * @param column the column values to segment + * @return list of breakpoint indices, starting with 0 + */ + public static List computeBreakpoints(CompressionSettings cs, double[] column) { + final int numElements = column.length; + final double targetMSE = cs.getPiecewiseTargetLoss(); + final double sseMax = numElements * targetMSE; // max allowed total SSE + + //start with high lambda an reduce iteratively + double lambda = Math.max(10.0, sseMax * 2.0); + List bestBreaks = Arrays.asList(0, numElements); + double bestSSE = computeTotalSSE(column, bestBreaks); + + for (int iter = 0; iter < 50; iter++) { + List breaks = computeBreakpointsLambda(column, lambda); + double totalSSE = computeTotalSSE(column, breaks); + int numSegs = breaks.size() - 1; + + if (totalSSE < bestSSE) { + bestSSE = totalSSE; + bestBreaks = new ArrayList<>(breaks); + } + //target loss reached + if (bestSSE <= sseMax) { + return bestBreaks; + } + + // only one segment left, break condition + if (numSegs <= 1) { + break; + } + // reducing lambda to allow more segments in next iteration + lambda *= 0.8; + } + + return bestBreaks; + } + + /** + * Computes optimal breakpoints, each segment has a SEE plus a + */ + + public static List computeBreakpointsLambda(double[] column, double lambda) { + final int n = column.length; + final double[] costs = new double[n + 1]; // min cost to reach i + final int[] prev = new int[n + 1]; + + Arrays.fill(costs, Double.POSITIVE_INFINITY); + costs[0] = 0.0; + // precompute all segment costs to avoid recomputation in dynamic programming + double[][] segCosts = new double[n + 1][n + 1]; + for (int i = 0; i < n; i++) { + for (int j = i + 1; j <= n; j++) { + segCosts[i][j] = computeSegmentCost(column, i, j); + } + } + // for each point j, find the cheapest previous breakpoint i + for (int j = 1; j <= n; j++) { + for (int i = 0; i < j; i++) { + // cost equals the SSE of segment [i,j] plus penalty plus best costs + double cost = costs[i] + segCosts[i][j] + lambda; + if (cost < costs[j]) { + costs[j] = cost; + prev[j] = i; + } + } + } + + // Backtrack to previous points to recover the breakpoints + List breaks = new ArrayList<>(); + int j = n; + while (j > 0) { + breaks.add(j); + j = prev[j]; + } + breaks.add(0); + Collections.reverse(breaks); + return breaks; + } + + /** + * computes the segment cost + * + * @param column column values + * @param start start index + * @param end end index + * @return SSE of the regression line over the segment + */ + public static double computeSegmentCost(double[] column, int start, int end) { + final int segSize = end - start; + if (segSize <= 1) + return 0.0; + + final double[] ab = regressSegment(column, start, end); + final double slope = ab[0]; + final double intercept = ab[1]; + + double sse = 0.0; + for (int i = start; i < end; i++) { + double err = column[i] - (slope * i + intercept); + sse += err * err; + } + return sse; + } + + /** + * computes the total SSE over all segments defined by the given breakpoints + * + * @param column + * @param breaks + * @return sum of the total SSE + */ + public static double computeTotalSSE(double[] column, List breaks) { + double total = 0.0; + for (int s = 0; s < breaks.size() - 1; s++) { + final int start = breaks.get(s); + final int end = breaks.get(s + 1); + total += computeSegmentCost(column, start, end); + } + return total; + } + + public static double[] regressSegment(double[] column, int start, int end) { + final int numElements = end - start; + if (numElements <= 0) + return new double[]{0.0, 0.0}; + + double sumOfRowIndices = 0, sumOfColumnValues = 0, sumOfRowIndicesSquared = 0, productRowIndexTimesColumnValue = 0; + for (int i = start; i < end; i++) { + sumOfRowIndices += i; + sumOfColumnValues += column[i]; + sumOfRowIndicesSquared += i * i; + productRowIndexTimesColumnValue += i * column[i]; + } + + + final double denominatorForSlope = + numElements * sumOfRowIndicesSquared - sumOfRowIndices * sumOfRowIndices; + final double slope; + final double intercept; + if (denominatorForSlope == 0) { + slope = 0.0; + intercept = sumOfColumnValues / numElements; + } else { + slope = (numElements * productRowIndexTimesColumnValue - sumOfRowIndices * sumOfColumnValues) / + denominatorForSlope; + intercept = (sumOfColumnValues - slope * sumOfRowIndices) / numElements; + } + return new double[]{slope, intercept}; + } + + /** + * computes breakpoints for a y using a successive algorithm + * extends each segment until the SEE reaches the target loss, then start a new segment + * + * @param y y values + * @param cs compression setting for setting the target loss + * @return list of breakpoint indices + */ + public static List computeBreakpointSuccessive(double[] y, CompressionSettings cs) { + final int numElements = y.length; + final double targetMSE = cs.getPiecewiseTargetLoss(); + if (Double.isNaN(targetMSE) || targetMSE <= 0) { + return Arrays.asList(0, numElements); // fallback single segment + } + + List breakpoints = new ArrayList<>(); + breakpoints.add(0); + double sumX = 0.0, sumY = 0.0, sumXY = 0.0, sumX2 = 0.0, sumY2 = 0.0; + int segmentLength = 0; + double beta, alpha; + + for (int n = 0; n < numElements; n++){ + double x = n; + double Y = y[n]; + sumX += x; + sumY += Y; + sumX2 += x * x; + sumY2 += Y * Y; + sumXY += x * Y; + segmentLength++; + + if(segmentLength > 1) { + + final double alphaBetaDenominator = segmentLength * sumX2 - sumX * sumX; + if (alphaBetaDenominator == 0.0){ + beta = 0.0; + alpha = sumY / segmentLength; + } + else{ + beta = (segmentLength * sumXY - sumX * sumY )/alphaBetaDenominator; + alpha = (sumY * sumX2 - sumX * sumXY)/alphaBetaDenominator; + } + + double sse = Math.max(0.0,sumY2 - alpha * sumY - beta * sumXY); //sum of least squares + if(sse > segmentLength * targetMSE){ + int bestEnd = (segmentLength > 2) ? n-1 : n; + breakpoints.add(bestEnd); + + if (bestEnd == n - 1) { + segmentLength = 1; + sumX = x; + sumY = Y; + sumX2 = x * x; + sumY2 = Y * Y; + sumXY = x * Y; + } else { + segmentLength = 0; + sumX = sumY = sumX2 = sumY2 = sumXY = 0.0; + } + } + } + } + + // make sure, that the last breakpoint equals numElements + int last = breakpoints.get(breakpoints.size() - 1); + if (last != numElements) { + breakpoints.add(numElements); + } + + return breakpoints; + } } From ed18dcdce759b9bb5720262f97ad66f253da7098 Mon Sep 17 00:00:00 2001 From: Mancer1 Date: Mon, 15 Jun 2026 19:45:01 +0200 Subject: [PATCH 02/10] Old DP algorithm and its usages are removed --- .../compress/colgroup/ColGroupFactory.java | 25 +----------- .../functional/PiecewiseLinearUtils.java | 23 ----------- ...ewiseLinearCompressionPerformanceTest.java | 39 +------------------ 3 files changed, 3 insertions(+), 84 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java index 89bb040d444..b2b929feb6a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java @@ -1110,7 +1110,7 @@ public static AColGroup compressPiecewiseLinearFunctional(IColIndex colIndexes, for(int col = 0; col < numCols; col++) { final int colIdx = colIndexes.get(col); double[] column = PiecewiseLinearUtils.getColumn(in, colIdx); - PiecewiseLinearUtils.SegmentedRegression fit = PiecewiseLinearUtils.compressSegmentedLeastSquares(column, + PiecewiseLinearUtils.SegmentedRegression fit = PiecewiseLinearUtils.compressSuccessivePiecewiseLinear(column, cs); breakpointsPerCol[col] = fit.getBreakpoints(); interceptsPerCol[col] = fit.getIntercepts(); @@ -1122,29 +1122,6 @@ public static AColGroup compressPiecewiseLinearFunctional(IColIndex colIndexes, } - public static AColGroup compressPiecewiseLinearFunctionalSuccessive(IColIndex colIndexes, MatrixBlock in, - CompressionSettings cs) { - final int numRows = in.getNumRows(); - final int numCols = colIndexes.size(); - int[][] breakpointsPerCol = new int[numCols][]; - double[][] slopesPerCol = new double[numCols][]; - double[][] interceptsPerCol = new double[numCols][]; - - for(int col = 0; col < numCols; col++) { - final int colIdx = colIndexes.get(col); - double[] column = PiecewiseLinearUtils.getColumn(in, colIdx); - PiecewiseLinearUtils.SegmentedRegression fit = PiecewiseLinearUtils.compressSuccessivePiecewiseLinear( - column, cs); - breakpointsPerCol[col] = fit.getBreakpoints(); - interceptsPerCol[col] = fit.getIntercepts(); - slopesPerCol[col] = fit.getSlopes(); - - } - return ColGroupPiecewiseLinearCompressed.create(colIndexes, breakpointsPerCol, slopesPerCol, interceptsPerCol, - numRows); - - } - private AColGroup compressSDCFromSparseTransposedBlock(IColIndex cols, int nrUniqueEstimate, double tupleSparsity) { if(cols.size() > 1) return compressMultiColSDCFromSparseTransposedBlock(cols, nrUniqueEstimate, tupleSparsity); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java index dcf8f6726ea..cc92113d0e9 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java @@ -51,29 +51,6 @@ public static double[] getColumn(MatrixBlock in, int colIndex) { return column; } - public static SegmentedRegression compressSegmentedLeastSquares(double[] column, CompressionSettings cs) { - //compute Breakpoints for a Column with dynamic Programming - final List breakpointsList = computeBreakpoints(cs, column); - final int[] breakpoints = breakpointsList.stream().mapToInt(Integer::intValue).toArray(); - - //get values for Regression - final int numSeg = breakpoints.length - 1; - final double[] slopes = new double[numSeg]; - final double[] intercepts = new double[numSeg]; - - // Regress per Segment - for (int seg = 0; seg < numSeg; seg++) { - final int SegStart = breakpoints[seg]; - final int SegEnd = breakpoints[seg + 1]; - - final double[] line = regressSegment(column, SegStart, SegEnd); - slopes[seg] = line[0]; //slope regession line - intercepts[seg] = line[1]; //intercept regression line - } - - return new SegmentedRegression(breakpoints, slopes, intercepts); - } - public static SegmentedRegression compressSuccessivePiecewiseLinear(double[] column, CompressionSettings cs) { //compute Breakpoints for a Column with a sukzessive breakpoints algorithm diff --git a/src/test/java/org/apache/sysds/performance/PiecewiseLinearCompressionPerformanceTest.java b/src/test/java/org/apache/sysds/performance/PiecewiseLinearCompressionPerformanceTest.java index 6046bdfb20b..25b9d5df7e0 100644 --- a/src/test/java/org/apache/sysds/performance/PiecewiseLinearCompressionPerformanceTest.java +++ b/src/test/java/org/apache/sysds/performance/PiecewiseLinearCompressionPerformanceTest.java @@ -94,13 +94,13 @@ private static void benchmarkSuccessive(MatrixBlock mb, double loss) { cs.setPiecewiseTargetLoss(loss); IColIndex colIndexes = ColIndexFactory.create(numCol); - ColGroupFactory.compressPiecewiseLinearFunctionalSuccessive(colIndexes, mb, cs); + ColGroupFactory.compressPiecewiseLinearFunctional(colIndexes, mb, cs); Timing t = new Timing(); AColGroup cg = null; t.start(); for(int i = 0; i < REPS; i++) - cg = ColGroupFactory.compressPiecewiseLinearFunctionalSuccessive(colIndexes, mb, cs); + cg = ColGroupFactory.compressPiecewiseLinearFunctional(colIndexes, mb, cs); double time = t.stop() / REPS; long size = cg.getExactSizeOnDisk(); @@ -112,34 +112,6 @@ private static void benchmarkSuccessive(MatrixBlock mb, double loss) { loss, avgSegments(cg), size / 1e6, saving, time, reconstructionMSE(mb, cg)); } - /** - * benchmarks dynamic programming compression for a given matrix and target loss - * no repetition, because DP is too slow due complexity - * reports segments, compressed data size, runtime and reconstruction - * @param mb original matrix to compress - * @param loss target loss param - */ - private static void benchmarkDP(MatrixBlock mb, double loss) { - long origSize = mb.getInMemorySize(); - int numColumns = mb.getNumColumns(); - CompressionSettings cs = new CompressionSettingsBuilder().create(); - cs.setPiecewiseTargetLoss(loss); - IColIndex colIndexes = ColIndexFactory.create(numColumns); - - Timing t = new Timing(); - t.start(); - AColGroup cg = ColGroupFactory.compressPiecewiseLinearFunctional(colIndexes, mb, cs); - double time = t.stop(); - - long size = cg.getExactSizeOnDisk(); - String saving = size < origSize - ? String.format("saved %3.0f%%", 100.0 - 100.0 * size / origSize) - : String.format("LARGER +%.0f%%", 100.0 * size / origSize - 100); - - System.out.printf(" DP loss=%.0e %5.1f segs %6.2f MB (%s) %6.1f ms MSE=%.2e%n", - loss, avgSegments(cg), size / 1e6, saving, time, reconstructionMSE(mb, cg)); - } - public static void main(String[] args) { System.out.println("=== Piecewise Linear Compression Benchmark ===\n"); @@ -157,12 +129,5 @@ public static void main(String[] args) { for(double loss : LOSSES) benchmarkSuccessive(mb, loss); } - - // DP quality reference on small matrix - System.out.println("\n=== DP: quality reference (nrows=1000, ncols=10) ==="); - MatrixBlock mbSmall = generateTestMatrix(1000, 10); - System.out.printf("original=%.2f MB%n", mbSmall.getInMemorySize() / 1e6); - for(double loss : LOSSES) - benchmarkDP(mbSmall, loss); } } From 9383091f793d10cea2e9238c36cc9039a83942eb Mon Sep 17 00:00:00 2001 From: Ugway Date: Mon, 15 Jun 2026 20:28:51 +0200 Subject: [PATCH 03/10] add DDCLZW enum value to ColGroupType --- .../org/apache/sysds/runtime/compress/colgroup/AColGroup.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java index 57bba0bef15..fc14f866a09 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java @@ -88,7 +88,7 @@ public boolean isSDC() { */ protected static enum ColGroupType { UNCOMPRESSED, RLE, OLE, DDC, CONST, EMPTY, SDC, SDCSingle, SDCSingleZeros, SDCZeros, SDCFOR, DDCFOR, DeltaDDC, - LinearFunctional, PiecewiseLinear; + LinearFunctional, PiecewiseLinear, DDCLZW; } /** The ColGroup indexes contained in the ColGroup */ From 5048313a74385b6cca509f7f22ee4cae3ee372d6 Mon Sep 17 00:00:00 2001 From: m-ollka Date: Tue, 23 Jun 2026 00:44:27 +0200 Subject: [PATCH 04/10] new ColGroupPiecewiseLinearCompressed methods; fix testComputeSUm --- .../compress/colgroup/ColGroupFactory.java | 5 + .../ColGroupPiecewiseLinearCompressed.java | 721 ++++++++++++++++-- .../functional/PiecewiseLinearUtils.java | 21 +- ...ecewiseLinearCompressedOperationsTest.java | 7 +- 4 files changed, 672 insertions(+), 82 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java index b2b929feb6a..59a0a456ad6 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java @@ -1122,6 +1122,11 @@ public static AColGroup compressPiecewiseLinearFunctional(IColIndex colIndexes, } + public static AColGroup compressPiecewiseLinearFunctionalSuccessive(IColIndex colIndexes, MatrixBlock in, + CompressionSettings cs) { + return compressPiecewiseLinearFunctional(colIndexes, in, cs); + } + private AColGroup compressSDCFromSparseTransposedBlock(IColIndex cols, int nrUniqueEstimate, double tupleSparsity) { if(cols.size() > 1) return compressMultiColSDCFromSparseTransposedBlock(cols, nrUniqueEstimate, tupleSparsity); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java index f05a5d46e79..b87c581264d 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java @@ -1,6 +1,8 @@ package org.apache.sysds.runtime.compress.colgroup; import org.apache.commons.lang3.NotImplementedException; +import org.apache.sysds.runtime.compress.DMLCompressionException; +import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme; import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator; @@ -17,7 +19,9 @@ import org.apache.sysds.runtime.matrix.operators.UnaryOperator; import org.apache.sysds.utils.MemoryEstimates; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; /** * This class represents a new ColGroup which is compresses column into segments (piecewise linear) to represent the @@ -232,10 +236,10 @@ public long getExactSizeOnDisk() { * @param c output array to accumulate column sums into * @param nRows number of rows, which is used because it is covered by the breakpoints */ + /** Accumulates the sum of all decompressed values across all columns into c[0]. */ @Override public void computeSum(double[] c, int nRows) { for(int col = 0; col < _colIndexes.size(); col++) { - double sum = 0.0; int[] breakpoints = breakpointsPerCol[col]; double[] intercepts = interceptsPerCol[col]; double[] slopes = slopesPerCol[col]; @@ -248,23 +252,31 @@ public void computeSum(double[] c, int nRows) { continue; double sumX = (double) len * (2.0 * start + (len - 1)) / 2.0; - sum += slopes[seg] * sumX + intercepts[seg] * len; + c[0] += slopes[seg] * sumX + intercepts[seg] * len; } - c[col] += sum; } } - /** - * Computes column sums by delegating to computeSum Methods are identical because every ColGroup just knows its own - * column - * - * @param c The array to add the column sum to. - * @param nRows The number of rows in the column group. - */ - + /** Accumulates the sum for each column into c[_colIndexes.get(col)]. */ @Override public void computeColSums(double[] c, int nRows) { - computeSum(c, nRows); + for(int col = 0; col < _colIndexes.size(); col++) { + int gcol = _colIndexes.get(col); + int[] breakpoints = breakpointsPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + double[] slopes = slopesPerCol[col]; + + for(int seg = 0; seg < slopes.length; seg++) { + int start = breakpoints[seg]; + int end = breakpoints[seg + 1]; + int len = end - start; + if(len <= 0) + continue; + + double sumX = (double) len * (2.0 * start + (len - 1)) / 2.0; + c[gcol] += slopes[seg] * sumX + intercepts[seg] * len; + } + } } @Override @@ -447,232 +459,809 @@ private boolean colContainsValue(int col, double pattern) { @Override public AColGroup unaryOperation(UnaryOperator op) { - throw new NotImplementedException(); + throw new NotImplementedException("unaryOperation not supported for PiecewiseLinear"); } @Override public AColGroup replace(double pattern, double replace) { - throw new NotImplementedException(); + throw new NotImplementedException("replace not supported for PiecewiseLinear"); } + /** + * Computes global min or max over all decompressed values. For each linear segment the extreme is at one endpoint. + */ @Override protected double computeMxx(double c, Builtin builtin) { - throw new NotImplementedException(); + for(int col = 0; col < _colIndexes.size(); col++) { + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + for(int seg = 0; seg + 1 < bp.length; seg++) { + int start = bp[seg]; + int end = bp[seg + 1] - 1; // last row index in this segment + if(start > end) + continue; + double valStart = slopes[seg] * start + intercepts[seg]; + double valEnd = slopes[seg] * end + intercepts[seg]; + c = builtin.execute(c, valStart); + c = builtin.execute(c, valEnd); + } + } + return c; } + /** Computes per-column min or max, storing in c[_colIndexes.get(col)]. */ @Override protected void computeColMxx(double[] c, Builtin builtin) { - throw new NotImplementedException(); + for(int col = 0; col < _colIndexes.size(); col++) { + int gcol = _colIndexes.get(col); + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + for(int seg = 0; seg + 1 < bp.length; seg++) { + int start = bp[seg]; + int end = bp[seg + 1] - 1; + if(start > end) + continue; + double valStart = slopes[seg] * start + intercepts[seg]; + double valEnd = slopes[seg] * end + intercepts[seg]; + c[gcol] = builtin.execute(c[gcol], valStart); + c[gcol] = builtin.execute(c[gcol], valEnd); + } + } } + /** + * Computes sum of squares of all decompressed values using the closed-form formula: + * sum_{i=start}^{end-1} (m*i + b)^2 = m^2*sumI2 + 2*m*b*sumI + b^2*len + */ @Override protected void computeSumSq(double[] c, int nRows) { - throw new NotImplementedException(); - + double total = 0.0; + for(int col = 0; col < _colIndexes.size(); col++) + total += segmentSumSq(col); + c[0] += total; } + /** Computes per-column sum of squares. */ @Override protected void computeColSumsSq(double[] c, int nRows) { - throw new NotImplementedException(); + for(int col = 0; col < _colIndexes.size(); col++) + c[_colIndexes.get(col)] += segmentSumSq(col); + } + + private double segmentSumSq(int col) { + double total = 0.0; + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + for(int seg = 0; seg + 1 < bp.length; seg++) { + int start = bp[seg]; + int end = bp[seg + 1]; + int len = end - start; + if(len <= 0) + continue; + double m = slopes[seg]; + double b = intercepts[seg]; + double sumI = (double) len * (2.0 * start + (len - 1)) / 2.0; + double sumI2 = sumOfSquares(start, end); + total += m * m * sumI2 + 2.0 * m * b * sumI + b * b * len; + } + return total; + } + /** sum_{i=start}^{end-1} i^2, using the closed form end*(end-1)*(2*end-1)/6 - start*(start-1)*(2*start-1)/6 */ + private static double sumOfSquares(int start, int end) { + double s = 0; + if(end > 0) + s += (double) end * (end - 1) * (2 * end - 1) / 6.0; + if(start > 0) + s -= (double) start * (start - 1) * (2 * start - 1) / 6.0; + return s; } + /** Adds preAgg[rix] to c[rix] for each row in [rl, ru). preAgg is the row-sum across all columns. */ @Override protected void computeRowSums(double[] c, int rl, int ru, double[] preAgg) { - throw new NotImplementedException(); - + for(int rix = rl; rix < ru; rix++) + c[rix] += preAgg[rix]; } + /** Applies builtin(c[rix], preAgg[rix]) for each row in [rl, ru). preAgg is the row min/max across columns. */ @Override protected void computeRowMxx(double[] c, Builtin builtin, int rl, int ru, double[] preAgg) { - throw new NotImplementedException(); - + for(int rix = rl; rix < ru; rix++) + c[rix] = builtin.execute(c[rix], preAgg[rix]); } + /** Computes the product of all decompressed values, accumulated into c[0]. */ @Override protected void computeProduct(double[] c, int nRows) { - throw new NotImplementedException(); - + for(int col = 0; col < _colIndexes.size(); col++) { + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + for(int seg = 0; seg + 1 < bp.length; seg++) { + double m = slopes[seg]; + double b = intercepts[seg]; + for(int r = bp[seg]; r < bp[seg + 1]; r++) { + double v = m * r + b; + if(v == 0) { + c[0] = 0; + return; + } + c[0] *= v; + } + } + } } + /** Multiplies c[rix] by the product of all column values at row rix (from preAgg). */ @Override protected void computeRowProduct(double[] c, int rl, int ru, double[] preAgg) { - throw new NotImplementedException(); - + for(int rix = rl; rix < ru; rix++) + c[rix] *= preAgg[rix]; } + /** Computes per-column product of all decompressed values. */ @Override protected void computeColProduct(double[] c, int nRows) { - throw new NotImplementedException(); - + for(int col = 0; col < _colIndexes.size(); col++) { + int gcol = _colIndexes.get(col); + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + for(int seg = 0; seg + 1 < bp.length; seg++) { + double m = slopes[seg]; + double b = intercepts[seg]; + for(int r = bp[seg]; r < bp[seg + 1]; r++) { + double v = m * r + b; + if(v == 0) { + c[gcol] = 0; + break; + } + c[gcol] *= v; + } + if(c[gcol] == 0) + break; + } + } } + /** Returns array[r] = sum of all column values at row r (used by computeRowSums). */ @Override protected double[] preAggSumRows() { - throw new NotImplementedException(); + double[] agg = new double[numRows]; + for(int col = 0; col < _colIndexes.size(); col++) { + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + for(int seg = 0; seg + 1 < bp.length; seg++) { + double m = slopes[seg]; + double b = intercepts[seg]; + for(int r = bp[seg]; r < bp[seg + 1]; r++) + agg[r] += m * r + b; + } + } + return agg; } + /** Returns array[r] = sum of squared column values at row r (used by computeRowSums for SumSq). */ @Override protected double[] preAggSumSqRows() { - throw new NotImplementedException(); + double[] agg = new double[numRows]; + for(int col = 0; col < _colIndexes.size(); col++) { + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + for(int seg = 0; seg + 1 < bp.length; seg++) { + double m = slopes[seg]; + double b = intercepts[seg]; + for(int r = bp[seg]; r < bp[seg + 1]; r++) { + double v = m * r + b; + agg[r] += v * v; + } + } + } + return agg; } + /** Returns array[r] = product of all column values at row r (used by computeRowProduct). */ @Override protected double[] preAggProductRows() { - throw new NotImplementedException(); + double[] agg = new double[numRows]; + Arrays.fill(agg, 1.0); + for(int col = 0; col < _colIndexes.size(); col++) { + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + for(int seg = 0; seg + 1 < bp.length; seg++) { + double m = slopes[seg]; + double b = intercepts[seg]; + for(int r = bp[seg]; r < bp[seg + 1]; r++) + agg[r] *= m * r + b; + } + } + return agg; } + /** Returns array[r] = builtin applied across all column values at row r (used by computeRowMxx). */ @Override protected double[] preAggBuiltinRows(Builtin builtin) { - throw new NotImplementedException(); + double init = builtin.getBuiltinCode() == Builtin.BuiltinCode.MAX ? Double.NEGATIVE_INFINITY + : Double.POSITIVE_INFINITY; + double[] agg = new double[numRows]; + Arrays.fill(agg, init); + for(int col = 0; col < _colIndexes.size(); col++) { + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + for(int seg = 0; seg + 1 < bp.length; seg++) { + double m = slopes[seg]; + double b = intercepts[seg]; + for(int r = bp[seg]; r < bp[seg + 1]; r++) + agg[r] = builtin.execute(agg[r], m * r + b); + } + } + return agg; } + /** Two piecewise linear groups have the same index structure if they are both piecewise linear. */ @Override public boolean sameIndexStructure(AColGroupCompressed that) { - throw new NotImplementedException(); + return that instanceof ColGroupPiecewiseLinearCompressed; } + /** + * Computes the transpose self-matrix multiplication (t(A) %*% A) using closed-form arithmetic series. + * For each pair of columns i, j, merges their breakpoint sequences and sums segment cross-products analytically. + */ @Override protected void tsmm(double[] result, int numColumns, int nRows) { - throw new NotImplementedException(); + final int numCols = _colIndexes.size(); + for(int i = 0; i < numCols; i++) { + final int gcol_i = _colIndexes.get(i); + for(int j = i; j < numCols; j++) { + final int gcol_j = _colIndexes.get(j); + double dotProduct = crossColDotProduct(i, j); + result[gcol_i * numColumns + gcol_j] += dotProduct; + } + } + } + /** + * Computes sum_r val_i(r) * val_j(r) by merging breakpoints of columns i and j. + * Within each merged interval, the product of two linear functions has a closed form. + */ + private double crossColDotProduct(int i, int j) { + int[] bp_i = breakpointsPerCol[i]; + int[] bp_j = breakpointsPerCol[j]; + double[] slopes_i = slopesPerCol[i]; + double[] intercepts_i = interceptsPerCol[i]; + double[] slopes_j = slopesPerCol[j]; + double[] intercepts_j = interceptsPerCol[j]; + + double dot = 0.0; + int si = 0, sj = 0; + int a = 0; + + while(si < slopes_i.length && sj < slopes_j.length) { + int end_i = bp_i[si + 1]; + int end_j = bp_j[sj + 1]; + int b = Math.min(end_i, end_j); + + double m_i = slopes_i[si]; + double b_i = intercepts_i[si]; + double m_j = slopes_j[sj]; + double b_j = intercepts_j[sj]; + + int len = b - a; + if(len > 0) { + double sumI = (double) len * (2.0 * a + (len - 1)) / 2.0; + double sumI2 = sumOfSquares(a, b); + dot += m_i * m_j * sumI2 + (m_i * b_j + m_j * b_i) * sumI + b_i * b_j * len; + } + + a = b; + if(b >= end_i) + si++; + if(b >= end_j) + sj++; + } + return dot; } + /** Returns a copy of this group with new column indices. */ @Override public AColGroup copyAndSet(IColIndex colIndexes) { - throw new NotImplementedException(); + return new ColGroupPiecewiseLinearCompressed(colIndexes, breakpointsPerCol, slopesPerCol, interceptsPerCol, + numRows); } + /** + * Decompresses rows [rl, ru) into the DenseBlock in transposed form: + * db row = _colIndexes.get(col), db column = original row index. + */ @Override public void decompressToDenseBlockTransposed(DenseBlock db, int rl, int ru) { - throw new NotImplementedException(); - + for(int col = 0; col < _colIndexes.size(); col++) { + final int gcol = _colIndexes.get(col); + final double[] c = db.values(gcol); + final int off = db.pos(gcol); + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + for(int seg = 0; seg + 1 < bp.length; seg++) { + int segStart = Math.max(bp[seg], rl); + int segEnd = Math.min(bp[seg + 1], ru); + double m = slopes[seg]; + double b = intercepts[seg]; + for(int r = segStart; r < segEnd; r++) + c[off + r] += m * r + b; + } + } } @Override public void decompressToSparseBlockTransposed(SparseBlockMCSR sb, int nColOut) { - throw new NotImplementedException(); - + throw new NotImplementedException("decompressToSparseBlockTransposed not supported for PiecewiseLinear"); } + /** Decompresses rows [rl, ru) into a SparseBlock, iterating row-first to satisfy column-order append requirement. */ @Override public void decompressToSparseBlock(SparseBlock sb, int rl, int ru, int offR, int offC) { - throw new NotImplementedException(); - + final int numCols = _colIndexes.size(); + for(int row = rl; row < ru; row++) { + for(int col = 0; col < numCols; col++) { + double v = getIdx(row, col); + if(v != 0) + sb.append(row + offR, _colIndexes.get(col) + offC, v); + } + } } + /** + * Right-multiplies this column group by the given matrix, returning an uncompressed column group. + * For each output column j and each input segment, accumulates the weighted sum row-by-row. + */ @Override public AColGroup rightMultByMatrix(MatrixBlock right, IColIndex allCols, int k) { - throw new NotImplementedException(); + final int nColR = right.getNumColumns(); + final IColIndex outputCols = allCols != null ? allCols : ColIndexFactory.create(nColR); + final MatrixBlock result = new MatrixBlock(numRows, nColR, false); + result.allocateDenseBlock(); + final double[] resultValues = result.getDenseBlockValues(); + + for(int col = 0; col < _colIndexes.size(); col++) { + final int gcol = _colIndexes.get(col); + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + + for(int seg = 0; seg + 1 < bp.length; seg++) { + double m = slopes[seg]; + double b = intercepts[seg]; + for(int j = 0; j < nColR; j++) { + double w = right.get(gcol, j); + if(w == 0) + continue; + for(int r = bp[seg]; r < bp[seg + 1]; r++) + resultValues[r * nColR + j] += w * (m * r + b); + } + } + } + result.recomputeNonZeros(); + return ColGroupUncompressed.create(result, outputCols); } + /** + * Left-multiplies a sub-range of the given matrix by this column group. + * Rows [rl, ru) of matrix, columns [cl, cu) are multiplied against this group's rows [cl, cu). + */ @Override public void leftMultByMatrixNoPreAgg(MatrixBlock matrix, MatrixBlock result, int rl, int ru, int cl, int cu) { - throw new NotImplementedException(); + final int numCols = _colIndexes.size(); + for(int col = 0; col < numCols; col++) { + final int gcol = _colIndexes.get(col); + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + for(int mRow = rl; mRow < ru; mRow++) { + double sum = 0.0; + for(int seg = 0; seg + 1 < bp.length; seg++) { + int segStart = Math.max(bp[seg], cl); + int segEnd = Math.min(bp[seg + 1], cu); + if(segStart >= segEnd) + continue; + double m = slopes[seg]; + double b = intercepts[seg]; + for(int r = segStart; r < segEnd; r++) + sum += matrix.get(mRow, r) * (m * r + b); + } + result.set(mRow, gcol, result.get(mRow, gcol) + sum); + } + } } + /** + * Left-multiplies by another column group: computes t(lhs) %*% this and accumulates into result. + * Iterates over all rows, using getIdx to decompress both sides. + */ @Override public void leftMultByAColGroup(AColGroup lhs, MatrixBlock result, int nRows) { - throw new NotImplementedException(); + if(lhs instanceof ColGroupEmpty) + return; + final int lhsNumCols = lhs.getNumCols(); + final int rhsNumCols = _colIndexes.size(); + final double[] resValues = result.getDenseBlockValues(); + final int resCols = result.getNumColumns(); + + for(int col = 0; col < rhsNumCols; col++) { + final int gcol = _colIndexes.get(col); + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + for(int seg = 0; seg + 1 < bp.length; seg++) { + double m = slopes[seg]; + double b = intercepts[seg]; + for(int r = bp[seg]; r < bp[seg + 1]; r++) { + double rhsVal = m * r + b; + if(rhsVal == 0) + continue; + for(int lhsCol = 0; lhsCol < lhsNumCols; lhsCol++) { + double lhsVal = lhs.getIdx(r, lhsCol); + if(lhsVal != 0) + resValues[lhsCol * resCols + gcol] += lhsVal * rhsVal; + } + } + } + } } @Override public void tsmmAColGroup(AColGroup other, MatrixBlock result) { - throw new NotImplementedException(); - + throw new DMLCompressionException("tsmmAColGroup should not be called on PiecewiseLinear"); } + /** Returns a new group with only column at index idx. */ @Override protected AColGroup sliceSingleColumn(int idx) { - throw new NotImplementedException(); + IColIndex newCols = ColIndexFactory.create(1); + return new ColGroupPiecewiseLinearCompressed(newCols, + new int[][] {breakpointsPerCol[idx].clone()}, + new double[][] {slopesPerCol[idx].clone()}, + new double[][] {interceptsPerCol[idx].clone()}, + numRows); } + /** Returns a new group with columns [idStart, idEnd), mapped to outputCols. */ @Override protected AColGroup sliceMultiColumns(int idStart, int idEnd, IColIndex outputCols) { - throw new NotImplementedException(); + int numSelected = idEnd - idStart; + int[][] newBp = new int[numSelected][]; + double[][] newSlopes = new double[numSelected][]; + double[][] newIntercepts = new double[numSelected][]; + for(int i = 0; i < numSelected; i++) { + int src = idStart + i; + newBp[i] = breakpointsPerCol[src].clone(); + newSlopes[i] = slopesPerCol[src].clone(); + newIntercepts[i] = interceptsPerCol[src].clone(); + } + return new ColGroupPiecewiseLinearCompressed(outputCols, newBp, newSlopes, newIntercepts, numRows); } + /** + * Returns a new group covering rows [rl, ru) only. + * Breakpoints are shifted to start at 0; intercepts are adjusted so that + * value at new row r' = value at original row r' + rl. + */ @Override public AColGroup sliceRows(int rl, int ru) { - throw new NotImplementedException(); + int numCols = _colIndexes.size(); + int newNumRows = ru - rl; + int[][] newBp = new int[numCols][]; + double[][] newSlopes = new double[numCols][]; + double[][] newIntercepts = new double[numCols][]; + + for(int col = 0; col < numCols; col++) { + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + + List bpList = new ArrayList<>(); + List slopeList = new ArrayList<>(); + List interceptList = new ArrayList<>(); + bpList.add(0); + + for(int seg = 0; seg + 1 < bp.length; seg++) { + if(bp[seg + 1] <= rl) + continue; + if(bp[seg] >= ru) + break; + int newEnd = Math.min(bp[seg + 1], ru) - rl; + slopeList.add(slopes[seg]); + // adjust intercept: original value at r = m*r + b; at new index r'=r-rl: m*(r'+rl)+b = m*r' + (m*rl+b) + interceptList.add(intercepts[seg] + slopes[seg] * rl); + bpList.add(newEnd); + } + + if(bpList.size() == 1) { + slopeList.add(0.0); + interceptList.add(0.0); + bpList.add(newNumRows); + } + + newBp[col] = bpList.stream().mapToInt(Integer::intValue).toArray(); + newSlopes[col] = slopeList.stream().mapToDouble(Double::doubleValue).toArray(); + newIntercepts[col] = interceptList.stream().mapToDouble(Double::doubleValue).toArray(); + } + + return new ColGroupPiecewiseLinearCompressed(_colIndexes, newBp, newSlopes, newIntercepts, newNumRows); } + /** + * Counts non-zero decompressed values. For constant segments (slope=0) the answer is trivial. + * For linear segments, the zero crossing can occur at most once per segment. + */ @Override public long getNumberNonZeros(int nRows) { - throw new NotImplementedException(); + long nnz = 0; + for(int col = 0; col < _colIndexes.size(); col++) { + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + for(int seg = 0; seg + 1 < bp.length; seg++) { + int start = bp[seg]; + int end = bp[seg + 1]; + int len = end - start; + if(len <= 0) + continue; + double m = slopes[seg]; + double b = intercepts[seg]; + if(m == 0) { + if(b != 0) + nnz += len; + } + else { + // linear: zero at r = -b/m; at most one integer zero crossing + double zeroAt = -b / m; + int zi = (int) Math.round(zeroAt); + if(zi >= start && zi < end && Math.abs(m * zi + b) < 1e-12) + nnz += len - 1; + else + nnz += len; + } + } + } + return nnz; } + /** Computes central moment by iterating all decompressed values row by row. */ @Override public CmCovObject centralMoment(CMOperator op, int nRows) { - throw new NotImplementedException(); + CmCovObject ret = new CmCovObject(); + for(int col = 0; col < _colIndexes.size(); col++) { + int[] bp = breakpointsPerCol[col]; + double[] slopes = slopesPerCol[col]; + double[] intercepts = interceptsPerCol[col]; + for(int seg = 0; seg + 1 < bp.length; seg++) { + double m = slopes[seg]; + double b = intercepts[seg]; + for(int r = bp[seg]; r < bp[seg + 1]; r++) + op.fn.execute(ret, m * r + b, 1); + } + } + return ret; } @Override public AColGroup rexpandCols(int max, boolean ignore, boolean cast, int nRows) { - throw new NotImplementedException(); + throw new NotImplementedException("rexpandCols not supported for PiecewiseLinear"); } @Override public double getCost(ComputationCostEstimator e, int nRows) { - throw new NotImplementedException(); + final int nCols = getNumCols(); + final int nVals = getNumValues(); + return e.getCost(nRows, nRows, nCols, nVals, 1.0); } + /** Returns null to indicate the group cannot be merged; the framework will use the generic append path. */ @Override public AColGroup append(AColGroup g) { - throw new NotImplementedException(); + return null; } + /** + * Appends multiple piecewise linear blocks vertically. groups[0] == this. + * Each block i covers rows [i*blen, min((i+1)*blen, rlen)). + * Breakpoints are shifted by the block offset; intercepts are adjusted accordingly. + */ @Override protected AColGroup appendNInternal(AColGroup[] groups, int blen, int rlen) { - throw new NotImplementedException(); + final int numCols = _colIndexes.size(); + int[][] mergedBp = new int[numCols][]; + double[][] mergedSlopes = new double[numCols][]; + double[][] mergedIntercepts = new double[numCols][]; + + for(int col = 0; col < numCols; col++) { + List bpList = new ArrayList<>(); + List slopeList = new ArrayList<>(); + List interceptList = new ArrayList<>(); + bpList.add(0); + + int offset = 0; + for(AColGroup g : groups) { + if(g instanceof ColGroupPiecewiseLinearCompressed) { + ColGroupPiecewiseLinearCompressed plg = (ColGroupPiecewiseLinearCompressed) g; + int[] gbp = plg.breakpointsPerCol[col]; + double[] gSlopes = plg.slopesPerCol[col]; + double[] gIntercepts = plg.interceptsPerCol[col]; + for(int seg = 0; seg + 1 < gbp.length; seg++) { + slopeList.add(gSlopes[seg]); + // new intercept = original_intercept - slope * offset (so value at global row R=r+offset is + // m*(R-offset)+b = m*R + (b - m*offset)) + interceptList.add(gIntercepts[seg] - gSlopes[seg] * offset); + bpList.add(gbp[seg + 1] + offset); + } + offset += plg.numRows; + } + else { + throw new NotImplementedException( + "appendNInternal: cannot append " + g.getClass().getSimpleName() + " into PiecewiseLinear"); + } + } + + mergedBp[col] = bpList.stream().mapToInt(Integer::intValue).toArray(); + mergedSlopes[col] = slopeList.stream().mapToDouble(Double::doubleValue).toArray(); + mergedIntercepts[col] = interceptList.stream().mapToDouble(Double::doubleValue).toArray(); + } + + return new ColGroupPiecewiseLinearCompressed(_colIndexes, mergedBp, mergedSlopes, mergedIntercepts, rlen); } + /** No scheme available for piecewise linear compression; returns null. */ @Override public ICLAScheme getCompressionScheme() { - throw new NotImplementedException(); + return null; } + /** No recompression needed; returns this. */ @Override public AColGroup recompress() { - throw new NotImplementedException(); + return this; } @Override public CompressedSizeInfoColGroup getCompressionInfo(int nRow) { - throw new NotImplementedException(); + throw new NotImplementedException("getCompressionInfo not implemented for PiecewiseLinear"); } + /** Returns a new group with columns reordered according to the reordering array. */ @Override protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) { - throw new NotImplementedException(); + final int numCols = newColIndex.size(); + int[][] newBp = new int[numCols][]; + double[][] newSlopes = new double[numCols][]; + double[][] newIntercepts = new double[numCols][]; + for(int i = 0; i < numCols; i++) { + int old = reordering[i]; + newBp[i] = breakpointsPerCol[old].clone(); + newSlopes[i] = slopesPerCol[old].clone(); + newIntercepts[i] = interceptsPerCol[old].clone(); + } + return new ColGroupPiecewiseLinearCompressed(newColIndex, newBp, newSlopes, newIntercepts, numRows); } + /** Piecewise linear column groups cannot be reduced to fewer columns; returns null. */ @Override public AColGroup reduceCols() { - throw new NotImplementedException(); + return null; } + /** All decompressed values are in general non-zero; returns 1.0. */ @Override public double getSparsity() { - throw new NotImplementedException(); + return 1.0; } + /** + * For each output row r in [rl, ru): finds the single "1" in row r of the sparse selection matrix, + * reads its column index as the compressed row to copy, then decompresses that row into output row r. + */ @Override protected void sparseSelection(MatrixBlock selection, ColGroupUtils.P[] points, MatrixBlock ret, int rl, int ru) { - throw new NotImplementedException(); + final SparseBlock sb = selection.getSparseBlock(); + final SparseBlock retB = ret.getSparseBlock(); + for(int r = rl; r < ru; r++) { + if(sb.isEmpty(r)) + continue; + final int sPos = sb.pos(r); + final int rowCompressed = sb.indexes(r)[sPos]; + decompressToSparseBlock(retB, rowCompressed, rowCompressed + 1, r - rowCompressed, 0); + } } + /** + * For each output row r in [rl, ru): finds the single "1" in row r of the sparse selection matrix, + * reads its column index as the compressed row to copy, then decompresses that row into the dense output at row r. + */ @Override protected void denseSelection(MatrixBlock selection, ColGroupUtils.P[] points, MatrixBlock ret, int rl, int ru) { - throw new NotImplementedException(); + final SparseBlock sb = selection.getSparseBlock(); + final DenseBlock retB = ret.getDenseBlock(); + for(int r = rl; r < ru; r++) { + if(sb.isEmpty(r)) + continue; + final int sPos = sb.pos(r); + final int rowCompressed = sb.indexes(r)[sPos]; + decompressToDenseBlock(retB, rowCompressed, rowCompressed + 1, r - rowCompressed, 0); + } } + /** + * Splits this column group for a row-wise reshape from (nRow x nColOrg) to (nRow/multiplier x nColOrg*multiplier). + * Each block of nRow/multiplier rows becomes new columns shifted by i*nColOrg. + * Returns a single ColGroupPiecewiseLinearCompressed covering all multiplier*numCols new columns. + */ @Override public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { - throw new NotImplementedException(); + final int numCols = _colIndexes.size(); + final int newNRow = nRow / multiplier; + final int totalNewCols = numCols * multiplier; + + final int[] newColIndices = new int[totalNewCols]; + for(int i = 0; i < multiplier; i++) + for(int c = 0; c < numCols; c++) + newColIndices[i * numCols + c] = _colIndexes.get(c) + i * nColOrg; + + final int[][] newBp = new int[totalNewCols][]; + final double[][] newSlopes = new double[totalNewCols][]; + final double[][] newIntercepts = new double[totalNewCols][]; + + for(int i = 0; i < multiplier; i++) { + final int rl = i * newNRow; + final int ru = rl + newNRow; + + for(int c = 0; c < numCols; c++) { + final int[] bp = breakpointsPerCol[c]; + final double[] slopes = slopesPerCol[c]; + final double[] intercepts = interceptsPerCol[c]; + + final List bpList = new ArrayList<>(); + final List slopeList = new ArrayList<>(); + final List interceptList = new ArrayList<>(); + bpList.add(0); + + for(int seg = 0; seg + 1 < bp.length; seg++) { + if(bp[seg + 1] <= rl) + continue; + if(bp[seg] >= ru) + break; + final int newEnd = Math.min(bp[seg + 1], ru) - rl; + slopeList.add(slopes[seg]); + // adjust intercept: value at new row r' = m*(r'+rl)+b = m*r' + (b + m*rl) + interceptList.add(intercepts[seg] + slopes[seg] * rl); + bpList.add(newEnd); + } + + if(bpList.size() == 1) { + slopeList.add(0.0); + interceptList.add(0.0); + bpList.add(newNRow); + } + + final int newColIdx = i * numCols + c; + newBp[newColIdx] = bpList.stream().mapToInt(Integer::intValue).toArray(); + newSlopes[newColIdx] = slopeList.stream().mapToDouble(Double::doubleValue).toArray(); + newIntercepts[newColIdx] = interceptList.stream().mapToDouble(Double::doubleValue).toArray(); + } + } + + return new AColGroup[] {new ColGroupPiecewiseLinearCompressed(ColIndexFactory.create(newColIndices), newBp, + newSlopes, newIntercepts, newNRow)}; } } - diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java index cc92113d0e9..3eeea1cb233 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java @@ -274,20 +274,13 @@ public static List computeBreakpointSuccessive(double[] y, CompressionS double sse = Math.max(0.0,sumY2 - alpha * sumY - beta * sumXY); //sum of least squares if(sse > segmentLength * targetMSE){ - int bestEnd = (segmentLength > 2) ? n-1 : n; - breakpoints.add(bestEnd); - - if (bestEnd == n - 1) { - segmentLength = 1; - sumX = x; - sumY = Y; - sumX2 = x * x; - sumY2 = Y * Y; - sumXY = x * Y; - } else { - segmentLength = 0; - sumX = sumY = sumX2 = sumY2 = sumXY = 0.0; - } + breakpoints.add(n); + segmentLength = 1; + sumX = x; + sumY = Y; + sumX2 = x * x; + sumY2 = Y * Y; + sumXY = x * Y; } } } diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java index 53ae3a1277c..980482a0adb 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java @@ -131,9 +131,12 @@ private MatrixBlock applyBinaryRowOpRight(MatrixBlock mb, BinaryOperator op, dou @Test public void testComputeSum() { - double[] sumsComp = new double[numCols]; + double[] sumsComp = new double[1]; piecewiseLinearColGroup.computeSum(sumsComp, numRows); - assertArrayEquals(sumsComp, computeSums(decompressedMB), DELTA); + double expectedTotal = 0; + for(double s : computeSums(decompressedMB)) + expectedTotal += s; + assertEquals(expectedTotal, sumsComp[0], DELTA); } @Test From a649eb03a8a1a5fea9ec69f4b14d6f7f40d0db92 Mon Sep 17 00:00:00 2001 From: Ugway Date: Tue, 23 Jun 2026 20:42:05 +0200 Subject: [PATCH 05/10] new replace & unitaryOperation --- .../ColGroupPiecewiseLinearCompressed.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java index b87c581264d..fd0ddffeb42 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java @@ -457,14 +457,24 @@ private boolean colContainsValue(int col, double pattern) { return false; } + private AColGroup decompress(){ + IColIndex columns = ColIndexFactory.create(numRows); + + MatrixBlock mb = new MatrixBlock(numRows, getNumCols(), false); + decompressToDenseBlock(mb.getDenseBlock(), 0, numRows, 0, 0); + return ColGroupUncompressed.create(mb, columns); + } + @Override public AColGroup unaryOperation(UnaryOperator op) { - throw new NotImplementedException("unaryOperation not supported for PiecewiseLinear"); + AColGroup uncompressed = decompress(); + return uncompressed.unaryOperation(op); } @Override public AColGroup replace(double pattern, double replace) { - throw new NotImplementedException("replace not supported for PiecewiseLinear"); + AColGroup uncompressed = decompress(); + return uncompressed.replace(pattern, replace); } /** From 3df7122a2d608fc03cf92893654094b03d932ae8 Mon Sep 17 00:00:00 2001 From: Ugway Date: Wed, 24 Jun 2026 08:12:21 +0200 Subject: [PATCH 06/10] ColGroupIO support --- .../runtime/compress/colgroup/AColGroup.java | 4 +- .../compress/colgroup/ColGroupFactory.java | 5 +- .../runtime/compress/colgroup/ColGroupIO.java | 2 + .../ColGroupPiecewiseLinearCompressed.java | 65 ++++++++++++++++++- ...ecewiseLinearCompressedOperationsTest.java | 2 + ...ColGroupPiecewiseLinearCompressedTest.java | 4 +- 6 files changed, 72 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java index fc14f866a09..af3a14f8123 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java @@ -66,7 +66,7 @@ public abstract class AColGroup implements Serializable { /** Public super types of compression ColGroups supported */ public static enum CompressionType { UNCOMPRESSED, RLE, OLE, DDC, CONST, EMPTY, SDC, SDCFOR, DDCFOR, DeltaDDC, DDCLZW, LinearFunctional, - PiecewiseLinear, PiecewiseLinearSuccessive; + PiecewiseLinearCompressed; public boolean isDense() { return this == DDC || this == CONST || this == DDCFOR || this == DDCFOR || this == DDCLZW; @@ -88,7 +88,7 @@ public boolean isSDC() { */ protected static enum ColGroupType { UNCOMPRESSED, RLE, OLE, DDC, CONST, EMPTY, SDC, SDCSingle, SDCSingleZeros, SDCZeros, SDCFOR, DDCFOR, DeltaDDC, - LinearFunctional, PiecewiseLinear, DDCLZW; + LinearFunctional, PiecewiseLinearCompressed, DDCLZW; } /** The ColGroup indexes contained in the ColGroup */ diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java index 59a0a456ad6..acf4e414c6f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java @@ -308,12 +308,9 @@ else if(ct == CompressionType.LinearFunctional) { return compressLinearFunctional(colIndexes, in, cs); } } - else if(ct == CompressionType.PiecewiseLinear) { + else if(ct == CompressionType.PiecewiseLinearCompressed) { return compressPiecewiseLinearFunctional(colIndexes, in, cs); } - else if(ct == CompressionType.PiecewiseLinearSuccessive) { - return compressPiecewiseLinearFunctionalSuccessive(colIndexes, in, cs); - } else if(ct == CompressionType.DDCFOR) { AColGroup g = directCompressDDC(colIndexes, cg); if(g instanceof ColGroupDDC) diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupIO.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupIO.java index f4e9007575c..1d587ccd57a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupIO.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupIO.java @@ -111,6 +111,8 @@ public static AColGroup readColGroup(DataInput in, int nRows) throws IOException return ColGroupDeltaDDC.read(in); case DDCLZW: return ColGroupDDCLZW.read(in); + case PiecewiseLinearCompressed: + return ColGroupPiecewiseLinearCompressed.read(in); case OLE: return ColGroupOLE.read(in, nRows); case RLE: diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java index fd0ddffeb42..a62f5f4cbf4 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java @@ -2,6 +2,8 @@ import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.runtime.compress.DMLCompressionException; +import org.apache.sysds.runtime.compress.colgroup.dictionary.DictionaryFactory; +import org.apache.sysds.runtime.compress.colgroup.dictionary.IDictionary; import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme; @@ -19,6 +21,9 @@ import org.apache.sysds.runtime.matrix.operators.UnaryOperator; import org.apache.sysds.utils.MemoryEstimates; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -281,12 +286,12 @@ public void computeColSums(double[] c, int nRows) { @Override public CompressionType getCompType() { - return CompressionType.PiecewiseLinear; + return CompressionType.PiecewiseLinearCompressed; } @Override protected ColGroupType getColGroupType() { - return ColGroupType.PiecewiseLinear; + return ColGroupType.PiecewiseLinearCompressed; } /** @@ -477,6 +482,62 @@ public AColGroup replace(double pattern, double replace) { return uncompressed.replace(pattern, replace); } + + public static int[][] read2DIntegerArray( DataInput in, int numRows) throws IOException{ + int[][] twoDimArray = new int[numRows][]; + for (int i = 0; i < numRows; i++){ + int twoDimArray_lenght = in.readInt(); + for (int j = 0; j < twoDimArray_lenght; j++){ + twoDimArray[i][j] = in.readInt(); + } + } + return twoDimArray; + } + public static double[][] read2DDoubleArray( DataInput in, int numRows) throws IOException{ + double[][] twoDimArray = new double[numRows][]; + for (int i = 0; i < numRows; i++){ + int twoDimArray_lenght = in.readInt(); + for (int j = 0; j < twoDimArray_lenght; j++){ + twoDimArray[i][j] = in.readDouble(); + } + } + return twoDimArray; + } + + public static ColGroupPiecewiseLinearCompressed read(DataInput in) throws IOException { + IColIndex colIndices = ColIndexFactory.read(in); + int numRows = in.readInt(); + int[][] breakpointsPerCol=read2DIntegerArray(in, numRows); + double[][] slopesPerCol = read2DDoubleArray(in, numRows); + double [][] interceptsPerCol = read2DDoubleArray(in, numRows); + + return new ColGroupPiecewiseLinearCompressed(colIndices, breakpointsPerCol, slopesPerCol, interceptsPerCol, numRows); + } + + @Override + public void write(DataOutput out) throws IOException { + super.write(out); + out.writeInt(numRows); + for(int[] breakpoints : breakpointsPerCol){ + out.writeInt(breakpoints.length); + for (int i : breakpoints) { + out.writeInt(i); + } + } + for(double[] slopes : slopesPerCol) { + out.writeInt(slopes.length); + for (double i : slopes) { + out.writeDouble(i); + } + } + for(double[] intercepts : interceptsPerCol) { + out.writeInt(intercepts.length); + for (double i : intercepts) { + out.writeDouble(i); + } + } + } + /** * Computes global min or max over all decompressed values. For each linear segment the extreme is at one endpoint. */ diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java index 980482a0adb..ce24db79c18 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java @@ -20,6 +20,8 @@ import org.junit.Before; import org.junit.Test; +q + import java.util.Random; import static org.junit.Assert.*; diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedTest.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedTest.java index e05745bc97d..7a8e040782e 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedTest.java @@ -339,7 +339,7 @@ private CompressedSizeInfo createTestCompressedSizeInfo() { EstimationFactors facts = new EstimationFactors(2, 10); CompressedSizeInfoColGroup info = new CompressedSizeInfoColGroup(cols, facts, - AColGroup.CompressionType.PiecewiseLinear); + AColGroup.CompressionType.PiecewiseLinearCompressed); List infos = Arrays.asList(info); CompressedSizeInfo csi = new CompressedSizeInfo(infos); @@ -357,7 +357,7 @@ public void testCompressPiecewiseLinearViaRealAPI() { } CompressionSettings cs = new CompressionSettingsBuilder().addValidCompression( - AColGroup.CompressionType.PiecewiseLinear).create(); + AColGroup.CompressionType.PiecewiseLinearCompressed).create(); CompressedSizeInfo csi = createTestCompressedSizeInfo(); From a294bf25ccda57024970338116ec37be118a4957 Mon Sep 17 00:00:00 2001 From: Ugway Date: Tue, 7 Jul 2026 17:26:08 +0200 Subject: [PATCH 07/10] [Minor] fixed mistake in previous commit --- .../ColGroupPiecewiseLinearCompressedOperationsTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java index ce24db79c18..980482a0adb 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java @@ -20,8 +20,6 @@ import org.junit.Before; import org.junit.Test; -q - import java.util.Random; import static org.junit.Assert.*; From fcb5fbc58fcd96cae0b69ed957ade326122c81c9 Mon Sep 17 00:00:00 2001 From: Mancer1 Date: Sun, 12 Jul 2026 22:28:01 +0200 Subject: [PATCH 08/10] Performance tests and Benchmarks for sucussive Piece wise algorithm and downstream CLA operations --- ...ewiseLinearCompressionPerformanceTest.java | 122 +++++++++++++++++- 1 file changed, 117 insertions(+), 5 deletions(-) diff --git a/src/test/java/org/apache/sysds/performance/PiecewiseLinearCompressionPerformanceTest.java b/src/test/java/org/apache/sysds/performance/PiecewiseLinearCompressionPerformanceTest.java index 25b9d5df7e0..bd1db57866c 100644 --- a/src/test/java/org/apache/sysds/performance/PiecewiseLinearCompressionPerformanceTest.java +++ b/src/test/java/org/apache/sysds/performance/PiecewiseLinearCompressionPerformanceTest.java @@ -7,9 +7,14 @@ import org.apache.sysds.runtime.compress.colgroup.ColGroupPiecewiseLinearCompressed; import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; +import org.apache.sysds.runtime.functionobjects.Multiply; import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.operators.RightScalarOperator; +import org.apache.sysds.runtime.matrix.operators.ScalarOperator; import org.apache.sysds.utils.stats.Timing; +import org.apache.sysds.runtime.compress.colgroup.functional.PiecewiseLinearUtils; import java.util.Random; +import java.util.List; /** * Performance benchmark for piecewise linear compression. @@ -53,6 +58,26 @@ private static MatrixBlock generateTestMatrix(int nr, int nc) { } return mb; } + + /** + * generate of a perfectly linear matrix to have a realistic test set up + * @param nr number of rows + * @param nc number of columns + * @return matrix with random generated data + */ + private static MatrixBlock generateLinearMatrix(int nr, int nc) { + MatrixBlock mb = new MatrixBlock(nr, nc, true); + mb.allocateDenseBlock(); + for(int c = 0; c < nc; c++) { + double slope = 0.5 * (c + 1); + double intercept = 10.0 - c; + for(int r = 0; r < nr; r++) { + mb.set(r, c, slope * r + intercept); + } + } + return mb; + } + /// returns a average number of segments per column private static double avgSegments(AColGroup cg) { int[][] breakpoints = ((ColGroupPiecewiseLinearCompressed) cg).getBreakpointsPerCol(); @@ -112,11 +137,95 @@ private static void benchmarkSuccessive(MatrixBlock mb, double loss) { loss, avgSegments(cg), size / 1e6, saving, time, reconstructionMSE(mb, cg)); } + /** + * Comparing the runtime speeds between the old Piece-wise linear compression + * and new successive Piece-wise linear compression algorithms + * @param mb original matrix to compress + * @param loss target loss param + */ + private static void compareDPvsSuccessive(MatrixBlock mb , double loss) { + System.out.println("\n--- Strategy Head-to-Head: DP vs Successive ---"); + CompressionSettings cs = new CompressionSettingsBuilder().create(); + cs.setPiecewiseTargetLoss(loss); + + double[] colData = PiecewiseLinearUtils.getColumn(mb, 0); + + // Benchmark DP + Timing tDP = new Timing(); + tDP.start(); + List dpBreaks = PiecewiseLinearUtils.computeBreakpoints(cs, colData); + double timeDP = tDP.stop(); + + // Benchmark Successive + Timing tSucc = new Timing(); + tSucc.start(); + List succBreaks = PiecewiseLinearUtils.computeBreakpointSuccessive(colData, cs); + double timeSucc = tSucc.stop(); + + System.out.printf(" DP: %5.1f ms (%2d segs) | Successive: %5.1f ms (%2d segs)%n", + timeDP, dpBreaks.size() - 1, timeSucc, succBreaks.size() - 1); + + } + + /** + * Verifying whether new successive algorithm follow an O(N) time complexity + * irrespective to the distribution of a matrix + * @param mb original matrix to compress + * @param loss target loss param + * @param nr number of rows + * @param nc number of columns + */ + private static void testDistributions(MatrixBlock mb, int nr, int nc, double loss) { + System.out.println("\n--- Testing Data Distributions ---\n"); + MatrixBlock linear = generateLinearMatrix(nr, nc); + + System.out.println("\n1. Perfect Linear Dataset:\n"); + benchmarkSuccessive(linear, loss); + + System.out.println("\n2. Volatile Level-Shift Dataset:\n"); + benchmarkSuccessive(mb, loss); + } + + /** + * Verifies performance delta over different data distributions + * @param mb original matrix to compress + * @param loss target loss param + * @param nr number of rows + * @param nc number of columns + */ + private static void benchmarkOperations(MatrixBlock mb, int nr, int nc, double loss) { + System.out.println("\n--- Benchmarking SystemDS CLA Matrix Operations ---\n"); + CompressionSettings cs = new CompressionSettingsBuilder().create(); + cs.setPiecewiseTargetLoss(loss); + IColIndex colIndexes = ColIndexFactory.create(nc); + + AColGroup cg = ColGroupFactory.compressPiecewiseLinearFunctional(colIndexes, mb, cs); + int scalarReps = 100; + int analyticalReps = 50; + // 1. Benchmark Compressed Scalar Operations + double scalarFactor = 2.5; + ScalarOperator sop = new RightScalarOperator(Multiply.getMultiplyFnObject(), scalarFactor); + Timing tScalar = new Timing(); + tScalar.start(); + for(int i = 0; i < scalarReps; i++) { + cg.scalarOperation(sop); + } + System.out.printf("\n Scalar Multiplication (100 reps): %6.2f ms%n", tScalar.stop()); + + // 2. Benchmark Compressed Analytical Operations (Closed-form loops) + Timing tSums = new Timing(); + tSums.start(); + for(int i = 0; i < analyticalReps; i++) { + cg.computeColSums(new double[nc], nr); + } + System.out.printf("\n Analytical Column Sums (50 reps): %6.2f ms%n", tSums.stop()); + } + public static void main(String[] args) { System.out.println("=== Piecewise Linear Compression Benchmark ===\n"); // Successive scalability across large matrices - System.out.println("=== Successive: scalability ==="); + System.out.println("=== Successive: scalability ===\n"); int[][] configs = {{1000, 10}, {1000, 100}, {1000, 500}, {5000, 10}, {5000, 100}, {5000, 500}, {10000, 10}, {10000, 100}, {10000, 500}}; @@ -124,10 +233,13 @@ public static void main(String[] args) { for(int[] cfg : configs) { int nr = cfg[0], nc = cfg[1]; MatrixBlock mb = generateTestMatrix(nr, nc); - System.out.printf("%nnrows=%d ncols=%d original=%.2f MB%n", - nr, nc, mb.getInMemorySize() / 1e6); - for(double loss : LOSSES) - benchmarkSuccessive(mb, loss); + System.out.printf("%nnrows=%d ncols=%d original=%.2f MB%n\n", + nr, nc, mb.getInMemorySize() / 1e6); + for (double loss : LOSSES) { + testDistributions(mb, nr, nc, loss); + compareDPvsSuccessive(mb, loss); + benchmarkOperations(mb, nr, nc, loss); + } } } } From a02ed6fe70b69a4969e29aac507c1c936f7c29f8 Mon Sep 17 00:00:00 2001 From: Ugway Date: Wed, 15 Jul 2026 05:03:29 +0200 Subject: [PATCH 09/10] added operation tests --- .../ColGroupPiecewiseLinearCompressed.java | 22 ++-- ...ecewiseLinearCompressedOperationsTest.java | 124 ++++++++++++++++-- 2 files changed, 126 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java index a62f5f4cbf4..02ce9638bb0 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java @@ -2,8 +2,6 @@ import org.apache.commons.lang3.NotImplementedException; import org.apache.sysds.runtime.compress.DMLCompressionException; -import org.apache.sysds.runtime.compress.colgroup.dictionary.DictionaryFactory; -import org.apache.sysds.runtime.compress.colgroup.dictionary.IDictionary; import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; import org.apache.sysds.runtime.compress.colgroup.scheme.ICLAScheme; @@ -462,24 +460,28 @@ private boolean colContainsValue(int col, double pattern) { return false; } - private AColGroup decompress(){ + private AColGroup decompress() { IColIndex columns = ColIndexFactory.create(numRows); - MatrixBlock mb = new MatrixBlock(numRows, getNumCols(), false); + + mb.allocateDenseBlock(); decompressToDenseBlock(mb.getDenseBlock(), 0, numRows, 0, 0); + mb.recomputeNonZeros(); + return ColGroupUncompressed.create(mb, columns); } @Override - public AColGroup unaryOperation(UnaryOperator op) { - AColGroup uncompressed = decompress(); - return uncompressed.unaryOperation(op); + public AColGroup unaryOperation(UnaryOperator op){ + AColGroup cg_unc = decompress(); + return cg_unc.unaryOperation(op); } @Override - public AColGroup replace(double pattern, double replace) { - AColGroup uncompressed = decompress(); - return uncompressed.replace(pattern, replace); + public AColGroup replace(double pattern, double replace){ + AColGroup cg_unc = decompress(); + return cg_unc.replace(pattern, replace); + } diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java index 980482a0adb..cc10e272ad6 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java @@ -5,21 +5,23 @@ import org.apache.sysds.runtime.compress.colgroup.AColGroup; import org.apache.sysds.runtime.compress.colgroup.ColGroupFactory; import org.apache.sysds.runtime.compress.colgroup.ColGroupPiecewiseLinearCompressed; +import org.apache.sysds.runtime.compress.colgroup.ColGroupUncompressed; import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory; import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex; -import org.apache.sysds.runtime.functionobjects.Divide; -import org.apache.sysds.runtime.functionobjects.Minus; -import org.apache.sysds.runtime.functionobjects.Multiply; -import org.apache.sysds.runtime.functionobjects.Plus; +import org.apache.sysds.runtime.data.DenseBlock; +import org.apache.sysds.runtime.data.DenseBlockFP64; +import org.apache.sysds.runtime.functionobjects.*; import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.runtime.matrix.operators.BinaryOperator; import org.apache.sysds.runtime.matrix.operators.RightScalarOperator; import org.apache.sysds.runtime.matrix.operators.ScalarOperator; +import org.apache.sysds.runtime.matrix.operators.UnaryOperator; import org.apache.sysds.runtime.util.DataConverter; import org.apache.sysds.test.AutomatedTestBase; import org.junit.Before; import org.junit.Test; +import java.util.Arrays; import java.util.Random; import static org.junit.Assert.*; @@ -37,11 +39,11 @@ public class ColGroupPiecewiseLinearCompressedOperationsTest extends AutomatedTe private static final long SEED = 42L; private static final int NROWS = 50; private static final int NCOLS = 3; - private static final double TARGET_LOSS = 1e-8; + private static final double TARGET_LOSS = 50; private static final double DELTA = 1e-9; private ColGroupPiecewiseLinearCompressed piecewiseLinearColGroup; - private MatrixBlock orignalMB; + private MatrixBlock originalMB; private MatrixBlock decompressedMB; private IColIndex colIndexes; private int numRows; @@ -53,9 +55,9 @@ public void setUp() { numCols = NCOLS; /// generate random matrix - double[][] data = getRandomMatrix(numRows, numCols, -3, 3, 1.0, SEED); - orignalMB = DataConverter.convertToMatrixBlock(data); - orignalMB.allocateDenseBlock(); + double[][] data = getRandomMatrix(numRows, numCols, -30, 30, 1.0, SEED); + originalMB = DataConverter.convertToMatrixBlock(data); + originalMB.allocateDenseBlock(); colIndexes = ColIndexFactory.create(buildColArray(numCols)); @@ -63,7 +65,7 @@ public void setUp() { cs.setPiecewiseTargetLoss(TARGET_LOSS); /// create ColGroupPiecewiseLinearCompressed instance - AColGroup result = ColGroupFactory.compressPiecewiseLinearFunctional(colIndexes, orignalMB, cs); + AColGroup result = ColGroupFactory.compressPiecewiseLinearFunctional(colIndexes, originalMB, cs); assertTrue(result instanceof ColGroupPiecewiseLinearCompressed); piecewiseLinearColGroup = (ColGroupPiecewiseLinearCompressed) result; @@ -308,4 +310,106 @@ public double[][] getRandomMatrix(int rows, int cols, double min, double max, do data[r][c] = min + rng.nextDouble() * (max - min); return data; } + + + @Test + public void testCreate() { + ColGroupPiecewiseLinearCompressed plc =(ColGroupPiecewiseLinearCompressed) piecewiseLinearColGroup; + + AColGroup result = ColGroupPiecewiseLinearCompressed.create(plc.getColIndices(), plc.getBreakpointsPerCol(),plc.getSlopesPerCol(), plc.getInterceptsPerCol(), NROWS); + assertTrue(result instanceof ColGroupPiecewiseLinearCompressed); + + // equal to piecewiseLinearColGroup instance + assertArrayEquals(((ColGroupPiecewiseLinearCompressed) result).getBreakpointsPerCol(),plc.getBreakpointsPerCol()); + assertArrayEquals(((ColGroupPiecewiseLinearCompressed) result).getSlopesPerCol(),plc.getSlopesPerCol()); + assertArrayEquals(((ColGroupPiecewiseLinearCompressed) result).getInterceptsPerCol(),plc.getInterceptsPerCol()); + } + @Test + public void testDecompressToDenseBlock() { + MatrixBlock mb_compare = new MatrixBlock(originalMB); + mb_compare.recomputeNonZeros(); + DenseBlock db_compare = mb_compare.getDenseBlock(); + + MatrixBlock mb_result = new MatrixBlock(NROWS, NCOLS, false); + mb_result.allocateDenseBlock(); + mb_result.recomputeNonZeros(); + piecewiseLinearColGroup.decompressToDenseBlock(mb_result.getDenseBlock(), 0, 3, 0, 0); + DenseBlock db_result = mb_result.getDenseBlock(); + + + // DenseBlockFP64 is just one large 1 dim array + assertTrue(db_result instanceof DenseBlockFP64); + assertTrue(db_compare instanceof DenseBlockFP64); + + assertArrayEquals(db_result.values(NCOLS), db_compare.values(NCOLS), TARGET_LOSS); + } + + private double highest_loss(MatrixBlock result, MatrixBlock compare){ + // recompute non zeros + result.recomputeNonZeros(); + compare.recomputeNonZeros(); + + // asserEquals size correct + assertEquals(result.getNumRows(), compare.getNumRows()); + assertEquals(result.getNumColumns(), compare.getNumColumns()); + + // MatrixBlock diff + MatrixBlock diff = new MatrixBlock(NCOLS, NROWS, false); + + // binary Operation Minus + ValueFunction fn = Minus.getMinusFnObject(); + BinaryOperator op = new BinaryOperator( fn); + result.binaryOperations(op, compare, diff); + + // get max and min + double max = diff.max(); + double min = diff.min(); + + // choose max absolute value + return Math.max(Math.abs(max), Math.abs(min)); + } + + @Test + public void testUnaryOperationMultiply2() { + MatrixBlock compare = new MatrixBlock(originalMB); + ValueFunction fn = Multiply2.getMultiply2FnObject(); + AColGroup result = piecewiseLinearColGroup.unaryOperation(new UnaryOperator(fn)); + assertTrue(result instanceof ColGroupUncompressed); + + MatrixBlock resultMB = ((ColGroupUncompressed) result ).getData(); + MatrixBlock compareMB =compare; + + // do unaryOperation on compare + MatrixBlock compare_final = compare.unaryOperations(new UnaryOperator( fn)); + + // check if highest_loss smaller than worst case expected loss + double biggest_loss = highest_loss (resultMB, compareMB); + assertEquals(TARGET_LOSS * 2, Math.max(biggest_loss, TARGET_LOSS * 2), 0.0); + } + + @Test + public void testUnaryOperationPower2() { + MatrixBlock compare = new MatrixBlock(originalMB); + ValueFunction fn = Power2.getPower2FnObject(); + AColGroup result = piecewiseLinearColGroup.unaryOperation(new UnaryOperator(fn)); + assertTrue(result instanceof ColGroupUncompressed); + + MatrixBlock resultMB = ((ColGroupUncompressed) result ).getData(); + MatrixBlock compareMB =compare; + + // do unaryOperation on compare + MatrixBlock compare_final = compare.unaryOperations(new UnaryOperator( fn)); + + // check if highest_loss smaller than worst case expected loss + double biggest_loss = highest_loss (resultMB, compareMB); + assertEquals(TARGET_LOSS * TARGET_LOSS, Math.max(biggest_loss, TARGET_LOSS * TARGET_LOSS), 0.0); + } + + @Test + public void testReplace() { + // correct Data Type + AColGroup result = piecewiseLinearColGroup.replace(5.0, 1.0); + assertTrue(result instanceof ColGroupUncompressed); + } + } From 255fee602bef9ceb3c0271949c15e5cb4b29a184 Mon Sep 17 00:00:00 2001 From: Ugway Date: Sun, 19 Jul 2026 14:56:53 +0200 Subject: [PATCH 10/10] using tab instead of spaces for intentations --- .../ColGroupPiecewiseLinearCompressed.java | 459 +++++++------- .../functional/PiecewiseLinearUtils.java | 571 +++++++++--------- ...ecewiseLinearCompressedOperationsTest.java | 99 +-- 3 files changed, 589 insertions(+), 540 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java index 02ce9638bb0..42192583c87 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupPiecewiseLinearCompressed.java @@ -46,7 +46,7 @@ protected ColGroupPiecewiseLinearCompressed(IColIndex colIndices) { } public ColGroupPiecewiseLinearCompressed(IColIndex colIndices, int[][] breakpoints, double[][] slopes, - double[][] intercepts, int numRows) { + double[][] intercepts, int numRows) { super(colIndices); this.breakpointsPerCol = breakpoints; this.slopesPerCol = slopes.clone(); @@ -67,20 +67,20 @@ public ColGroupPiecewiseLinearCompressed(IColIndex colIndices, int[][] breakpoin */ public static AColGroup create(IColIndex colIndices, int[][] breakpointsPerCol, double[][] slopesPerCol, - double[][] interceptsPerCol, int numRows) { + double[][] interceptsPerCol, int numRows) { final int numCols = colIndices.size(); - if(breakpointsPerCol.length != numCols) + if (breakpointsPerCol.length != numCols) throw new IllegalArgumentException( - "bp.length=" + breakpointsPerCol.length + " != colIndices.size()=" + numCols); + "bp.length=" + breakpointsPerCol.length + " != colIndices.size()=" + numCols); - for(int c = 0; c < numCols; c++) { - if(breakpointsPerCol[c].length < 1 || breakpointsPerCol[c][0] != 0 || - breakpointsPerCol[c][breakpointsPerCol[c].length - 1] != numRows) + for (int c = 0; c < numCols; c++) { + if (breakpointsPerCol[c].length < 1 || breakpointsPerCol[c][0] != 0 || + breakpointsPerCol[c][breakpointsPerCol[c].length - 1] != numRows) throw new IllegalArgumentException( - "Invalid breakpoints for col " + c + ": must start=0, end=numRows, >=1 pts"); + "Invalid breakpoints for col " + c + ": must start=0, end=numRows, >=1 pts"); - if(slopesPerCol[c].length != interceptsPerCol[c].length || - slopesPerCol[c].length != breakpointsPerCol[c].length - 1) + if (slopesPerCol[c].length != interceptsPerCol[c].length || + slopesPerCol[c].length != breakpointsPerCol[c].length - 1) throw new IllegalArgumentException("Inconsistent array lengths col " + c); } @@ -88,7 +88,7 @@ public static AColGroup create(IColIndex colIndices, int[][] breakpointsPerCol, double[][] slopeCopy = new double[numCols][]; double[][] interceptCopy = new double[numCols][]; // defensive copy to prevent external modification - for(int c = 0; c < numCols; c++) { + for (int c = 0; c < numCols; c++) { bpCopy[c] = Arrays.copyOf(breakpointsPerCol[c], breakpointsPerCol[c].length); slopeCopy[c] = Arrays.copyOf(slopesPerCol[c], slopesPerCol[c].length); interceptCopy[c] = Arrays.copyOf(interceptsPerCol[c], interceptsPerCol[c].length); @@ -110,20 +110,20 @@ public static AColGroup create(IColIndex colIndices, int[][] breakpointsPerCol, */ @Override public void decompressToDenseBlock(DenseBlock db, int rl, int ru, int offR, int offC) { - if(db == null || _colIndexes == null || _colIndexes.size() == 0 || breakpointsPerCol == null || - slopesPerCol == null || interceptsPerCol == null) { + if (db == null || _colIndexes == null || _colIndexes.size() == 0 || breakpointsPerCol == null || + slopesPerCol == null || interceptsPerCol == null) { return; } - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { final int colIndex = _colIndexes.get(col); int[] breakpoints = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; // per segment in this column - for(int seg = 0; seg + 1 < breakpoints.length; seg++) { + for (int seg = 0; seg + 1 < breakpoints.length; seg++) { int segStart = breakpoints[seg]; int segEnd = breakpoints[seg + 1]; - if(segStart >= segEnd) + if (segStart >= segEnd) continue; double currentSlopeInSegment = slopes[seg]; @@ -132,16 +132,16 @@ public void decompressToDenseBlock(DenseBlock db, int rl, int ru, int offR, int int rowStart = Math.max(segStart, rl); int rowEnd = Math.min(segEnd, ru); - if(rowStart >= rowEnd) + if (rowStart >= rowEnd) continue; //Fill DenseBlock für this column and Segment - for(int row = rowStart; row < rowEnd; row++) { + for (int row = rowStart; row < rowEnd; row++) { double yhat = currentSlopeInSegment * row + currentInterceptInSegment; int dbRow = offR + row; int dbCol = offC + colIndex; - if(dbRow >= 0 && dbRow < db.numRows() && dbCol >= 0 && dbCol < db.numCols()) { + if (dbRow >= 0 && dbRow < db.numRows() && dbCol >= 0 && dbCol < db.numCols()) { db.set(dbRow, dbCol, yhat); } } @@ -173,7 +173,7 @@ public double[][] getInterceptsPerCol() { @Override public double getIdx(int r, int colIdx) { //safety check - if(r < 0 || r >= numRows || colIdx < 0 || colIdx >= _colIndexes.size()) { + if (r < 0 || r >= numRows || colIdx < 0 || colIdx >= _colIndexes.size()) { return 0.0; } int[] breakpoints = breakpointsPerCol[colIdx]; @@ -182,12 +182,11 @@ public double getIdx(int r, int colIdx) { // binary search for the segment containing row r int lowerBound = 0; int higherBound = breakpoints.length - 2; - while(lowerBound <= higherBound) { + while (lowerBound <= higherBound) { int mid = (lowerBound + higherBound) / 2; - if(r < breakpoints[mid + 1]) { + if (r < breakpoints[mid + 1]) { higherBound = mid - 1; - } - else + } else lowerBound = mid + 1; } int segment = Math.min(lowerBound, breakpoints.length - 2); @@ -203,7 +202,7 @@ public double getIdx(int r, int colIdx) { @Override public int getNumValues() { int total = 0; - for(int c = 0; c < _colIndexes.size(); c++) { + for (int c = 0; c < _colIndexes.size(); c++) { total += breakpointsPerCol[c].length + slopesPerCol[c].length + interceptsPerCol[c].length; } return total; @@ -222,7 +221,7 @@ public long getExactSizeOnDisk() { ret += 24L * 3; // outer array headers ret += 4L; //numRows field - for(int c = 0; c < numCols; c++) { + for (int c = 0; c < numCols; c++) { ret += (long) MemoryEstimates.intArrayCost(breakpointsPerCol[c].length); ret += (long) MemoryEstimates.doubleArrayCost(slopesPerCol[c].length); ret += (long) MemoryEstimates.doubleArrayCost(interceptsPerCol[c].length); @@ -239,19 +238,21 @@ public long getExactSizeOnDisk() { * @param c output array to accumulate column sums into * @param nRows number of rows, which is used because it is covered by the breakpoints */ - /** Accumulates the sum of all decompressed values across all columns into c[0]. */ + /** + * Accumulates the sum of all decompressed values across all columns into c[0]. + */ @Override public void computeSum(double[] c, int nRows) { - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { int[] breakpoints = breakpointsPerCol[col]; double[] intercepts = interceptsPerCol[col]; double[] slopes = slopesPerCol[col]; - for(int seg = 0; seg < slopes.length; seg++) { + for (int seg = 0; seg < slopes.length; seg++) { int start = breakpoints[seg]; int end = breakpoints[seg + 1]; int len = end - start; - if(len <= 0) + if (len <= 0) continue; double sumX = (double) len * (2.0 * start + (len - 1)) / 2.0; @@ -260,20 +261,22 @@ public void computeSum(double[] c, int nRows) { } } - /** Accumulates the sum for each column into c[_colIndexes.get(col)]. */ + /** + * Accumulates the sum for each column into c[_colIndexes.get(col)]. + */ @Override public void computeColSums(double[] c, int nRows) { - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { int gcol = _colIndexes.get(col); int[] breakpoints = breakpointsPerCol[col]; double[] intercepts = interceptsPerCol[col]; double[] slopes = slopesPerCol[col]; - for(int seg = 0; seg < slopes.length; seg++) { + for (int seg = 0; seg < slopes.length; seg++) { int start = breakpoints[seg]; int end = breakpoints[seg + 1]; int len = end - start; - if(len <= 0) + if (len <= 0) continue; double sumX = (double) len * (2.0 * start + (len - 1)) / 2.0; @@ -304,25 +307,24 @@ protected ColGroupType getColGroupType() { public AColGroup scalarOperation(ScalarOperator op) { final int numCols = _colIndexes.size(); - if(!(op.fn instanceof Plus || op.fn instanceof Minus || op.fn instanceof Multiply || op.fn instanceof Divide)) { + if (!(op.fn instanceof Plus || op.fn instanceof Minus || op.fn instanceof Multiply || op.fn instanceof Divide)) { throw new NotImplementedException("Unsupported scalar op: " + op.fn.getClass().getSimpleName()); } double[][] newIntercepts = new double[numCols][]; double[][] newSlopes = new double[numCols][]; - for(int col = 0; col < numCols; col++) { + for (int col = 0; col < numCols; col++) { final int numSegments = interceptsPerCol[col].length; newIntercepts[col] = new double[numSegments]; newSlopes[col] = new double[numSegments]; - for(int seg = 0; seg < numSegments; seg++) { - if(op.fn instanceof Plus || op.fn instanceof Minus) { + for (int seg = 0; seg < numSegments; seg++) { + if (op.fn instanceof Plus || op.fn instanceof Minus) { // only intercepts changes newSlopes[col][seg] = slopesPerCol[col][seg]; newIntercepts[col][seg] = op.executeScalar(interceptsPerCol[col][seg]); - } - else { // Multiply/Divide + } else { // Multiply/Divide newSlopes[col][seg] = op.executeScalar(slopesPerCol[col][seg]); newIntercepts[col][seg] = op.executeScalar(interceptsPerCol[col][seg]); } @@ -350,10 +352,10 @@ public AColGroup binaryRowOpLeft(BinaryOperator op, double[] v, boolean isRowSaf double[][] newSlopes = new double[numCols][]; final boolean isAddSub = op.fn instanceof Plus || op.fn instanceof Minus; - if(!isAddSub && !(op.fn instanceof Multiply || op.fn instanceof Divide)) + if (!isAddSub && !(op.fn instanceof Multiply || op.fn instanceof Divide)) throw new NotImplementedException("Unsupported binary op: " + op.fn.getClass().getSimpleName()); - for(int col = 0; col < numCols; col++) { + for (int col = 0; col < numCols; col++) { double rowValue = v[_colIndexes.get(col)]; int numSegs = interceptsPerCol[col].length; newIntercepts[col] = new double[numSegs]; @@ -361,9 +363,9 @@ public AColGroup binaryRowOpLeft(BinaryOperator op, double[] v, boolean isRowSaf // Plus/Minus: slope is translation-invariant, only intercept shifts newSlopes[col] = isAddSub ? slopesPerCol[col].clone() : new double[numSegs]; - for(int seg = 0; seg < numSegs; seg++) { + for (int seg = 0; seg < numSegs; seg++) { newIntercepts[col][seg] = op.fn.execute(rowValue, interceptsPerCol[col][seg]); - if(!isAddSub) + if (!isAddSub) newSlopes[col][seg] = op.fn.execute(rowValue, slopesPerCol[col][seg]); } } @@ -385,22 +387,22 @@ public AColGroup binaryRowOpRight(BinaryOperator op, double[] v, boolean isRowSa final int numCols = _colIndexes.size(); final boolean isAddSub = op.fn instanceof Plus || op.fn instanceof Minus; - if(!isAddSub && !(op.fn instanceof Multiply || op.fn instanceof Divide)) + if (!isAddSub && !(op.fn instanceof Multiply || op.fn instanceof Divide)) throw new NotImplementedException("Unsupported scalar op: " + op.fn.getClass().getSimpleName()); double[][] newSlopes = new double[numCols][]; double[][] newIntercepts = new double[numCols][]; - for(int col = 0; col < numCols; col++) { + for (int col = 0; col < numCols; col++) { double val = v[_colIndexes.get(col)]; int numSegs = interceptsPerCol[col].length; // Plus/Minus shifts intercept only, slopes are unchanged newSlopes[col] = isAddSub ? slopesPerCol[col].clone() : new double[numSegs]; newIntercepts[col] = new double[numSegs]; - for(int seg = 0; seg < numSegs; seg++) { + for (int seg = 0; seg < numSegs; seg++) { newIntercepts[col][seg] = op.fn.execute(interceptsPerCol[col][seg], val); - if(!isAddSub) + if (!isAddSub) newSlopes[col][seg] = op.fn.execute(slopesPerCol[col][seg], val); } } @@ -415,8 +417,8 @@ public AColGroup binaryRowOpRight(BinaryOperator op, double[] v, boolean isRowSa */ @Override public boolean containsValue(double pattern) { - for(int col = 0; col < _colIndexes.size(); col++) { - if(colContainsValue(col, pattern)) + for (int col = 0; col < _colIndexes.size(); col++) { + if (colContainsValue(col, pattern)) return true; } return false; @@ -435,18 +437,18 @@ private boolean colContainsValue(int col, double pattern) { int[] breakpoints = breakpointsPerCol[col]; double[] intercepts = interceptsPerCol[col]; double[] slopes = slopesPerCol[col]; - for(int seg = 0; seg < breakpoints.length - 1; seg++) { + for (int seg = 0; seg < breakpoints.length - 1; seg++) { int start = breakpoints[seg]; int len = breakpoints[seg + 1] - start; - if(len <= 0) + if (len <= 0) continue; double b = intercepts[seg]; double m = slopes[seg]; - if(m == 0.0) { + if (m == 0.0) { // constant segment: all values equal b - if(Double.compare(b, pattern) == 0) + if (Double.compare(b, pattern) == 0) return true; continue; } @@ -454,13 +456,13 @@ private boolean colContainsValue(int col, double pattern) { // check if pattern lies on the line: solve m*x + b = pattern for x double x = (pattern - b) / m; int xi = (int) x; - if(xi >= start && xi < start + len && Double.compare(m * xi + b, pattern) == 0) + if (xi >= start && xi < start + len && Double.compare(m * xi + b, pattern) == 0) return true; } return false; } - private AColGroup decompress() { + private AColGroup decompress() { IColIndex columns = ColIndexFactory.create(numRows); MatrixBlock mb = new MatrixBlock(numRows, getNumCols(), false); @@ -472,34 +474,35 @@ private AColGroup decompress() { } @Override - public AColGroup unaryOperation(UnaryOperator op){ - AColGroup cg_unc = decompress(); + public AColGroup unaryOperation(UnaryOperator op) { + AColGroup cg_unc = decompress(); return cg_unc.unaryOperation(op); } @Override - public AColGroup replace(double pattern, double replace){ - AColGroup cg_unc = decompress(); + public AColGroup replace(double pattern, double replace) { + AColGroup cg_unc = decompress(); return cg_unc.replace(pattern, replace); } - public static int[][] read2DIntegerArray( DataInput in, int numRows) throws IOException{ + public static int[][] read2DIntegerArray(DataInput in, int numRows) throws IOException { int[][] twoDimArray = new int[numRows][]; - for (int i = 0; i < numRows; i++){ + for (int i = 0; i < numRows; i++) { int twoDimArray_lenght = in.readInt(); - for (int j = 0; j < twoDimArray_lenght; j++){ + for (int j = 0; j < twoDimArray_lenght; j++) { twoDimArray[i][j] = in.readInt(); } } return twoDimArray; } - public static double[][] read2DDoubleArray( DataInput in, int numRows) throws IOException{ + + public static double[][] read2DDoubleArray(DataInput in, int numRows) throws IOException { double[][] twoDimArray = new double[numRows][]; - for (int i = 0; i < numRows; i++){ + for (int i = 0; i < numRows; i++) { int twoDimArray_lenght = in.readInt(); - for (int j = 0; j < twoDimArray_lenght; j++){ + for (int j = 0; j < twoDimArray_lenght; j++) { twoDimArray[i][j] = in.readDouble(); } } @@ -509,9 +512,9 @@ public static double[][] read2DDoubleArray( DataInput in, int numRows) throws I public static ColGroupPiecewiseLinearCompressed read(DataInput in) throws IOException { IColIndex colIndices = ColIndexFactory.read(in); int numRows = in.readInt(); - int[][] breakpointsPerCol=read2DIntegerArray(in, numRows); + int[][] breakpointsPerCol = read2DIntegerArray(in, numRows); double[][] slopesPerCol = read2DDoubleArray(in, numRows); - double [][] interceptsPerCol = read2DDoubleArray(in, numRows); + double[][] interceptsPerCol = read2DDoubleArray(in, numRows); return new ColGroupPiecewiseLinearCompressed(colIndices, breakpointsPerCol, slopesPerCol, interceptsPerCol, numRows); } @@ -520,19 +523,19 @@ public static ColGroupPiecewiseLinearCompressed read(DataInput in) throws IOExce public void write(DataOutput out) throws IOException { super.write(out); out.writeInt(numRows); - for(int[] breakpoints : breakpointsPerCol){ + for (int[] breakpoints : breakpointsPerCol) { out.writeInt(breakpoints.length); for (int i : breakpoints) { out.writeInt(i); } } - for(double[] slopes : slopesPerCol) { + for (double[] slopes : slopesPerCol) { out.writeInt(slopes.length); for (double i : slopes) { out.writeDouble(i); } } - for(double[] intercepts : interceptsPerCol) { + for (double[] intercepts : interceptsPerCol) { out.writeInt(intercepts.length); for (double i : intercepts) { out.writeDouble(i); @@ -545,14 +548,14 @@ public void write(DataOutput out) throws IOException { */ @Override protected double computeMxx(double c, Builtin builtin) { - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { int start = bp[seg]; int end = bp[seg + 1] - 1; // last row index in this segment - if(start > end) + if (start > end) continue; double valStart = slopes[seg] * start + intercepts[seg]; double valEnd = slopes[seg] * end + intercepts[seg]; @@ -563,18 +566,20 @@ protected double computeMxx(double c, Builtin builtin) { return c; } - /** Computes per-column min or max, storing in c[_colIndexes.get(col)]. */ + /** + * Computes per-column min or max, storing in c[_colIndexes.get(col)]. + */ @Override protected void computeColMxx(double[] c, Builtin builtin) { - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { int gcol = _colIndexes.get(col); int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { int start = bp[seg]; int end = bp[seg + 1] - 1; - if(start > end) + if (start > end) continue; double valStart = slopes[seg] * start + intercepts[seg]; double valEnd = slopes[seg] * end + intercepts[seg]; @@ -591,15 +596,17 @@ protected void computeColMxx(double[] c, Builtin builtin) { @Override protected void computeSumSq(double[] c, int nRows) { double total = 0.0; - for(int col = 0; col < _colIndexes.size(); col++) + for (int col = 0; col < _colIndexes.size(); col++) total += segmentSumSq(col); c[0] += total; } - /** Computes per-column sum of squares. */ + /** + * Computes per-column sum of squares. + */ @Override protected void computeColSumsSq(double[] c, int nRows) { - for(int col = 0; col < _colIndexes.size(); col++) + for (int col = 0; col < _colIndexes.size(); col++) c[_colIndexes.get(col)] += segmentSumSq(col); } @@ -608,11 +615,11 @@ private double segmentSumSq(int col) { int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { int start = bp[seg]; int end = bp[seg + 1]; int len = end - start; - if(len <= 0) + if (len <= 0) continue; double m = slopes[seg]; double b = intercepts[seg]; @@ -623,43 +630,51 @@ private double segmentSumSq(int col) { return total; } - /** sum_{i=start}^{end-1} i^2, using the closed form end*(end-1)*(2*end-1)/6 - start*(start-1)*(2*start-1)/6 */ + /** + * sum_{i=start}^{end-1} i^2, using the closed form end*(end-1)*(2*end-1)/6 - start*(start-1)*(2*start-1)/6 + */ private static double sumOfSquares(int start, int end) { double s = 0; - if(end > 0) + if (end > 0) s += (double) end * (end - 1) * (2 * end - 1) / 6.0; - if(start > 0) + if (start > 0) s -= (double) start * (start - 1) * (2 * start - 1) / 6.0; return s; } - /** Adds preAgg[rix] to c[rix] for each row in [rl, ru). preAgg is the row-sum across all columns. */ + /** + * Adds preAgg[rix] to c[rix] for each row in [rl, ru). preAgg is the row-sum across all columns. + */ @Override protected void computeRowSums(double[] c, int rl, int ru, double[] preAgg) { - for(int rix = rl; rix < ru; rix++) + for (int rix = rl; rix < ru; rix++) c[rix] += preAgg[rix]; } - /** Applies builtin(c[rix], preAgg[rix]) for each row in [rl, ru). preAgg is the row min/max across columns. */ + /** + * Applies builtin(c[rix], preAgg[rix]) for each row in [rl, ru). preAgg is the row min/max across columns. + */ @Override protected void computeRowMxx(double[] c, Builtin builtin, int rl, int ru, double[] preAgg) { - for(int rix = rl; rix < ru; rix++) + for (int rix = rl; rix < ru; rix++) c[rix] = builtin.execute(c[rix], preAgg[rix]); } - /** Computes the product of all decompressed values, accumulated into c[0]. */ + /** + * Computes the product of all decompressed values, accumulated into c[0]. + */ @Override protected void computeProduct(double[] c, int nRows) { - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { double m = slopes[seg]; double b = intercepts[seg]; - for(int r = bp[seg]; r < bp[seg + 1]; r++) { + for (int r = bp[seg]; r < bp[seg + 1]; r++) { double v = m * r + b; - if(v == 0) { + if (v == 0) { c[0] = 0; return; } @@ -669,68 +684,76 @@ protected void computeProduct(double[] c, int nRows) { } } - /** Multiplies c[rix] by the product of all column values at row rix (from preAgg). */ + /** + * Multiplies c[rix] by the product of all column values at row rix (from preAgg). + */ @Override protected void computeRowProduct(double[] c, int rl, int ru, double[] preAgg) { - for(int rix = rl; rix < ru; rix++) + for (int rix = rl; rix < ru; rix++) c[rix] *= preAgg[rix]; } - /** Computes per-column product of all decompressed values. */ + /** + * Computes per-column product of all decompressed values. + */ @Override protected void computeColProduct(double[] c, int nRows) { - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { int gcol = _colIndexes.get(col); int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { double m = slopes[seg]; double b = intercepts[seg]; - for(int r = bp[seg]; r < bp[seg + 1]; r++) { + for (int r = bp[seg]; r < bp[seg + 1]; r++) { double v = m * r + b; - if(v == 0) { + if (v == 0) { c[gcol] = 0; break; } c[gcol] *= v; } - if(c[gcol] == 0) + if (c[gcol] == 0) break; } } } - /** Returns array[r] = sum of all column values at row r (used by computeRowSums). */ + /** + * Returns array[r] = sum of all column values at row r (used by computeRowSums). + */ @Override protected double[] preAggSumRows() { double[] agg = new double[numRows]; - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { double m = slopes[seg]; double b = intercepts[seg]; - for(int r = bp[seg]; r < bp[seg + 1]; r++) + for (int r = bp[seg]; r < bp[seg + 1]; r++) agg[r] += m * r + b; } } return agg; } - /** Returns array[r] = sum of squared column values at row r (used by computeRowSums for SumSq). */ + /** + * Returns array[r] = sum of squared column values at row r (used by computeRowSums for SumSq). + */ @Override protected double[] preAggSumSqRows() { double[] agg = new double[numRows]; - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { double m = slopes[seg]; double b = intercepts[seg]; - for(int r = bp[seg]; r < bp[seg + 1]; r++) { + for (int r = bp[seg]; r < bp[seg + 1]; r++) { double v = m * r + b; agg[r] += v * v; } @@ -739,47 +762,53 @@ protected double[] preAggSumSqRows() { return agg; } - /** Returns array[r] = product of all column values at row r (used by computeRowProduct). */ + /** + * Returns array[r] = product of all column values at row r (used by computeRowProduct). + */ @Override protected double[] preAggProductRows() { double[] agg = new double[numRows]; Arrays.fill(agg, 1.0); - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { double m = slopes[seg]; double b = intercepts[seg]; - for(int r = bp[seg]; r < bp[seg + 1]; r++) + for (int r = bp[seg]; r < bp[seg + 1]; r++) agg[r] *= m * r + b; } } return agg; } - /** Returns array[r] = builtin applied across all column values at row r (used by computeRowMxx). */ + /** + * Returns array[r] = builtin applied across all column values at row r (used by computeRowMxx). + */ @Override protected double[] preAggBuiltinRows(Builtin builtin) { double init = builtin.getBuiltinCode() == Builtin.BuiltinCode.MAX ? Double.NEGATIVE_INFINITY - : Double.POSITIVE_INFINITY; + : Double.POSITIVE_INFINITY; double[] agg = new double[numRows]; Arrays.fill(agg, init); - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { double m = slopes[seg]; double b = intercepts[seg]; - for(int r = bp[seg]; r < bp[seg + 1]; r++) + for (int r = bp[seg]; r < bp[seg + 1]; r++) agg[r] = builtin.execute(agg[r], m * r + b); } } return agg; } - /** Two piecewise linear groups have the same index structure if they are both piecewise linear. */ + /** + * Two piecewise linear groups have the same index structure if they are both piecewise linear. + */ @Override public boolean sameIndexStructure(AColGroupCompressed that) { return that instanceof ColGroupPiecewiseLinearCompressed; @@ -792,9 +821,9 @@ public boolean sameIndexStructure(AColGroupCompressed that) { @Override protected void tsmm(double[] result, int numColumns, int nRows) { final int numCols = _colIndexes.size(); - for(int i = 0; i < numCols; i++) { + for (int i = 0; i < numCols; i++) { final int gcol_i = _colIndexes.get(i); - for(int j = i; j < numCols; j++) { + for (int j = i; j < numCols; j++) { final int gcol_j = _colIndexes.get(j); double dotProduct = crossColDotProduct(i, j); result[gcol_i * numColumns + gcol_j] += dotProduct; @@ -818,7 +847,7 @@ private double crossColDotProduct(int i, int j) { int si = 0, sj = 0; int a = 0; - while(si < slopes_i.length && sj < slopes_j.length) { + while (si < slopes_i.length && sj < slopes_j.length) { int end_i = bp_i[si + 1]; int end_j = bp_j[sj + 1]; int b = Math.min(end_i, end_j); @@ -829,26 +858,28 @@ private double crossColDotProduct(int i, int j) { double b_j = intercepts_j[sj]; int len = b - a; - if(len > 0) { + if (len > 0) { double sumI = (double) len * (2.0 * a + (len - 1)) / 2.0; double sumI2 = sumOfSquares(a, b); dot += m_i * m_j * sumI2 + (m_i * b_j + m_j * b_i) * sumI + b_i * b_j * len; } a = b; - if(b >= end_i) + if (b >= end_i) si++; - if(b >= end_j) + if (b >= end_j) sj++; } return dot; } - /** Returns a copy of this group with new column indices. */ + /** + * Returns a copy of this group with new column indices. + */ @Override public AColGroup copyAndSet(IColIndex colIndexes) { return new ColGroupPiecewiseLinearCompressed(colIndexes, breakpointsPerCol, slopesPerCol, interceptsPerCol, - numRows); + numRows); } /** @@ -857,19 +888,19 @@ public AColGroup copyAndSet(IColIndex colIndexes) { */ @Override public void decompressToDenseBlockTransposed(DenseBlock db, int rl, int ru) { - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { final int gcol = _colIndexes.get(col); final double[] c = db.values(gcol); final int off = db.pos(gcol); int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { int segStart = Math.max(bp[seg], rl); int segEnd = Math.min(bp[seg + 1], ru); double m = slopes[seg]; double b = intercepts[seg]; - for(int r = segStart; r < segEnd; r++) + for (int r = segStart; r < segEnd; r++) c[off + r] += m * r + b; } } @@ -880,14 +911,16 @@ public void decompressToSparseBlockTransposed(SparseBlockMCSR sb, int nColOut) { throw new NotImplementedException("decompressToSparseBlockTransposed not supported for PiecewiseLinear"); } - /** Decompresses rows [rl, ru) into a SparseBlock, iterating row-first to satisfy column-order append requirement. */ + /** + * Decompresses rows [rl, ru) into a SparseBlock, iterating row-first to satisfy column-order append requirement. + */ @Override public void decompressToSparseBlock(SparseBlock sb, int rl, int ru, int offR, int offC) { final int numCols = _colIndexes.size(); - for(int row = rl; row < ru; row++) { - for(int col = 0; col < numCols; col++) { + for (int row = rl; row < ru; row++) { + for (int col = 0; col < numCols; col++) { double v = getIdx(row, col); - if(v != 0) + if (v != 0) sb.append(row + offR, _colIndexes.get(col) + offC, v); } } @@ -905,20 +938,20 @@ public AColGroup rightMultByMatrix(MatrixBlock right, IColIndex allCols, int k) result.allocateDenseBlock(); final double[] resultValues = result.getDenseBlockValues(); - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { final int gcol = _colIndexes.get(col); int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { double m = slopes[seg]; double b = intercepts[seg]; - for(int j = 0; j < nColR; j++) { + for (int j = 0; j < nColR; j++) { double w = right.get(gcol, j); - if(w == 0) + if (w == 0) continue; - for(int r = bp[seg]; r < bp[seg + 1]; r++) + for (int r = bp[seg]; r < bp[seg + 1]; r++) resultValues[r * nColR + j] += w * (m * r + b); } } @@ -934,22 +967,22 @@ public AColGroup rightMultByMatrix(MatrixBlock right, IColIndex allCols, int k) @Override public void leftMultByMatrixNoPreAgg(MatrixBlock matrix, MatrixBlock result, int rl, int ru, int cl, int cu) { final int numCols = _colIndexes.size(); - for(int col = 0; col < numCols; col++) { + for (int col = 0; col < numCols; col++) { final int gcol = _colIndexes.get(col); int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int mRow = rl; mRow < ru; mRow++) { + for (int mRow = rl; mRow < ru; mRow++) { double sum = 0.0; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { int segStart = Math.max(bp[seg], cl); int segEnd = Math.min(bp[seg + 1], cu); - if(segStart >= segEnd) + if (segStart >= segEnd) continue; double m = slopes[seg]; double b = intercepts[seg]; - for(int r = segStart; r < segEnd; r++) + for (int r = segStart; r < segEnd; r++) sum += matrix.get(mRow, r) * (m * r + b); } result.set(mRow, gcol, result.get(mRow, gcol) + sum); @@ -963,29 +996,29 @@ public void leftMultByMatrixNoPreAgg(MatrixBlock matrix, MatrixBlock result, int */ @Override public void leftMultByAColGroup(AColGroup lhs, MatrixBlock result, int nRows) { - if(lhs instanceof ColGroupEmpty) + if (lhs instanceof ColGroupEmpty) return; final int lhsNumCols = lhs.getNumCols(); final int rhsNumCols = _colIndexes.size(); final double[] resValues = result.getDenseBlockValues(); final int resCols = result.getNumColumns(); - for(int col = 0; col < rhsNumCols; col++) { + for (int col = 0; col < rhsNumCols; col++) { final int gcol = _colIndexes.get(col); int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { double m = slopes[seg]; double b = intercepts[seg]; - for(int r = bp[seg]; r < bp[seg + 1]; r++) { + for (int r = bp[seg]; r < bp[seg + 1]; r++) { double rhsVal = m * r + b; - if(rhsVal == 0) + if (rhsVal == 0) continue; - for(int lhsCol = 0; lhsCol < lhsNumCols; lhsCol++) { + for (int lhsCol = 0; lhsCol < lhsNumCols; lhsCol++) { double lhsVal = lhs.getIdx(r, lhsCol); - if(lhsVal != 0) + if (lhsVal != 0) resValues[lhsCol * resCols + gcol] += lhsVal * rhsVal; } } @@ -998,25 +1031,29 @@ public void tsmmAColGroup(AColGroup other, MatrixBlock result) { throw new DMLCompressionException("tsmmAColGroup should not be called on PiecewiseLinear"); } - /** Returns a new group with only column at index idx. */ + /** + * Returns a new group with only column at index idx. + */ @Override protected AColGroup sliceSingleColumn(int idx) { IColIndex newCols = ColIndexFactory.create(1); return new ColGroupPiecewiseLinearCompressed(newCols, - new int[][] {breakpointsPerCol[idx].clone()}, - new double[][] {slopesPerCol[idx].clone()}, - new double[][] {interceptsPerCol[idx].clone()}, - numRows); + new int[][]{breakpointsPerCol[idx].clone()}, + new double[][]{slopesPerCol[idx].clone()}, + new double[][]{interceptsPerCol[idx].clone()}, + numRows); } - /** Returns a new group with columns [idStart, idEnd), mapped to outputCols. */ + /** + * Returns a new group with columns [idStart, idEnd), mapped to outputCols. + */ @Override protected AColGroup sliceMultiColumns(int idStart, int idEnd, IColIndex outputCols) { int numSelected = idEnd - idStart; int[][] newBp = new int[numSelected][]; double[][] newSlopes = new double[numSelected][]; double[][] newIntercepts = new double[numSelected][]; - for(int i = 0; i < numSelected; i++) { + for (int i = 0; i < numSelected; i++) { int src = idStart + i; newBp[i] = breakpointsPerCol[src].clone(); newSlopes[i] = slopesPerCol[src].clone(); @@ -1038,7 +1075,7 @@ public AColGroup sliceRows(int rl, int ru) { double[][] newSlopes = new double[numCols][]; double[][] newIntercepts = new double[numCols][]; - for(int col = 0; col < numCols; col++) { + for (int col = 0; col < numCols; col++) { int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; @@ -1048,10 +1085,10 @@ public AColGroup sliceRows(int rl, int ru) { List interceptList = new ArrayList<>(); bpList.add(0); - for(int seg = 0; seg + 1 < bp.length; seg++) { - if(bp[seg + 1] <= rl) + for (int seg = 0; seg + 1 < bp.length; seg++) { + if (bp[seg + 1] <= rl) continue; - if(bp[seg] >= ru) + if (bp[seg] >= ru) break; int newEnd = Math.min(bp[seg + 1], ru) - rl; slopeList.add(slopes[seg]); @@ -1060,7 +1097,7 @@ public AColGroup sliceRows(int rl, int ru) { bpList.add(newEnd); } - if(bpList.size() == 1) { + if (bpList.size() == 1) { slopeList.add(0.0); interceptList.add(0.0); bpList.add(newNumRows); @@ -1081,27 +1118,26 @@ public AColGroup sliceRows(int rl, int ru) { @Override public long getNumberNonZeros(int nRows) { long nnz = 0; - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { int start = bp[seg]; int end = bp[seg + 1]; int len = end - start; - if(len <= 0) + if (len <= 0) continue; double m = slopes[seg]; double b = intercepts[seg]; - if(m == 0) { - if(b != 0) + if (m == 0) { + if (b != 0) nnz += len; - } - else { + } else { // linear: zero at r = -b/m; at most one integer zero crossing double zeroAt = -b / m; int zi = (int) Math.round(zeroAt); - if(zi >= start && zi < end && Math.abs(m * zi + b) < 1e-12) + if (zi >= start && zi < end && Math.abs(m * zi + b) < 1e-12) nnz += len - 1; else nnz += len; @@ -1111,18 +1147,20 @@ public long getNumberNonZeros(int nRows) { return nnz; } - /** Computes central moment by iterating all decompressed values row by row. */ + /** + * Computes central moment by iterating all decompressed values row by row. + */ @Override public CmCovObject centralMoment(CMOperator op, int nRows) { CmCovObject ret = new CmCovObject(); - for(int col = 0; col < _colIndexes.size(); col++) { + for (int col = 0; col < _colIndexes.size(); col++) { int[] bp = breakpointsPerCol[col]; double[] slopes = slopesPerCol[col]; double[] intercepts = interceptsPerCol[col]; - for(int seg = 0; seg + 1 < bp.length; seg++) { + for (int seg = 0; seg + 1 < bp.length; seg++) { double m = slopes[seg]; double b = intercepts[seg]; - for(int r = bp[seg]; r < bp[seg + 1]; r++) + for (int r = bp[seg]; r < bp[seg + 1]; r++) op.fn.execute(ret, m * r + b, 1); } } @@ -1141,7 +1179,9 @@ public double getCost(ComputationCostEstimator e, int nRows) { return e.getCost(nRows, nRows, nCols, nVals, 1.0); } - /** Returns null to indicate the group cannot be merged; the framework will use the generic append path. */ + /** + * Returns null to indicate the group cannot be merged; the framework will use the generic append path. + */ @Override public AColGroup append(AColGroup g) { return null; @@ -1159,20 +1199,20 @@ protected AColGroup appendNInternal(AColGroup[] groups, int blen, int rlen) { double[][] mergedSlopes = new double[numCols][]; double[][] mergedIntercepts = new double[numCols][]; - for(int col = 0; col < numCols; col++) { + for (int col = 0; col < numCols; col++) { List bpList = new ArrayList<>(); List slopeList = new ArrayList<>(); List interceptList = new ArrayList<>(); bpList.add(0); int offset = 0; - for(AColGroup g : groups) { - if(g instanceof ColGroupPiecewiseLinearCompressed) { + for (AColGroup g : groups) { + if (g instanceof ColGroupPiecewiseLinearCompressed) { ColGroupPiecewiseLinearCompressed plg = (ColGroupPiecewiseLinearCompressed) g; int[] gbp = plg.breakpointsPerCol[col]; double[] gSlopes = plg.slopesPerCol[col]; double[] gIntercepts = plg.interceptsPerCol[col]; - for(int seg = 0; seg + 1 < gbp.length; seg++) { + for (int seg = 0; seg + 1 < gbp.length; seg++) { slopeList.add(gSlopes[seg]); // new intercept = original_intercept - slope * offset (so value at global row R=r+offset is // m*(R-offset)+b = m*R + (b - m*offset)) @@ -1180,10 +1220,9 @@ protected AColGroup appendNInternal(AColGroup[] groups, int blen, int rlen) { bpList.add(gbp[seg + 1] + offset); } offset += plg.numRows; - } - else { + } else { throw new NotImplementedException( - "appendNInternal: cannot append " + g.getClass().getSimpleName() + " into PiecewiseLinear"); + "appendNInternal: cannot append " + g.getClass().getSimpleName() + " into PiecewiseLinear"); } } @@ -1195,13 +1234,17 @@ protected AColGroup appendNInternal(AColGroup[] groups, int blen, int rlen) { return new ColGroupPiecewiseLinearCompressed(_colIndexes, mergedBp, mergedSlopes, mergedIntercepts, rlen); } - /** No scheme available for piecewise linear compression; returns null. */ + /** + * No scheme available for piecewise linear compression; returns null. + */ @Override public ICLAScheme getCompressionScheme() { return null; } - /** No recompression needed; returns this. */ + /** + * No recompression needed; returns this. + */ @Override public AColGroup recompress() { return this; @@ -1212,14 +1255,16 @@ public CompressedSizeInfoColGroup getCompressionInfo(int nRow) { throw new NotImplementedException("getCompressionInfo not implemented for PiecewiseLinear"); } - /** Returns a new group with columns reordered according to the reordering array. */ + /** + * Returns a new group with columns reordered according to the reordering array. + */ @Override protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) { final int numCols = newColIndex.size(); int[][] newBp = new int[numCols][]; double[][] newSlopes = new double[numCols][]; double[][] newIntercepts = new double[numCols][]; - for(int i = 0; i < numCols; i++) { + for (int i = 0; i < numCols; i++) { int old = reordering[i]; newBp[i] = breakpointsPerCol[old].clone(); newSlopes[i] = slopesPerCol[old].clone(); @@ -1228,13 +1273,17 @@ protected AColGroup fixColIndexes(IColIndex newColIndex, int[] reordering) { return new ColGroupPiecewiseLinearCompressed(newColIndex, newBp, newSlopes, newIntercepts, numRows); } - /** Piecewise linear column groups cannot be reduced to fewer columns; returns null. */ + /** + * Piecewise linear column groups cannot be reduced to fewer columns; returns null. + */ @Override public AColGroup reduceCols() { return null; } - /** All decompressed values are in general non-zero; returns 1.0. */ + /** + * All decompressed values are in general non-zero; returns 1.0. + */ @Override public double getSparsity() { return 1.0; @@ -1248,8 +1297,8 @@ public double getSparsity() { protected void sparseSelection(MatrixBlock selection, ColGroupUtils.P[] points, MatrixBlock ret, int rl, int ru) { final SparseBlock sb = selection.getSparseBlock(); final SparseBlock retB = ret.getSparseBlock(); - for(int r = rl; r < ru; r++) { - if(sb.isEmpty(r)) + for (int r = rl; r < ru; r++) { + if (sb.isEmpty(r)) continue; final int sPos = sb.pos(r); final int rowCompressed = sb.indexes(r)[sPos]; @@ -1265,8 +1314,8 @@ protected void sparseSelection(MatrixBlock selection, ColGroupUtils.P[] points, protected void denseSelection(MatrixBlock selection, ColGroupUtils.P[] points, MatrixBlock ret, int rl, int ru) { final SparseBlock sb = selection.getSparseBlock(); final DenseBlock retB = ret.getDenseBlock(); - for(int r = rl; r < ru; r++) { - if(sb.isEmpty(r)) + for (int r = rl; r < ru; r++) { + if (sb.isEmpty(r)) continue; final int sPos = sb.pos(r); final int rowCompressed = sb.indexes(r)[sPos]; @@ -1286,19 +1335,19 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { final int totalNewCols = numCols * multiplier; final int[] newColIndices = new int[totalNewCols]; - for(int i = 0; i < multiplier; i++) - for(int c = 0; c < numCols; c++) + for (int i = 0; i < multiplier; i++) + for (int c = 0; c < numCols; c++) newColIndices[i * numCols + c] = _colIndexes.get(c) + i * nColOrg; final int[][] newBp = new int[totalNewCols][]; final double[][] newSlopes = new double[totalNewCols][]; final double[][] newIntercepts = new double[totalNewCols][]; - for(int i = 0; i < multiplier; i++) { + for (int i = 0; i < multiplier; i++) { final int rl = i * newNRow; final int ru = rl + newNRow; - for(int c = 0; c < numCols; c++) { + for (int c = 0; c < numCols; c++) { final int[] bp = breakpointsPerCol[c]; final double[] slopes = slopesPerCol[c]; final double[] intercepts = interceptsPerCol[c]; @@ -1308,10 +1357,10 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { final List interceptList = new ArrayList<>(); bpList.add(0); - for(int seg = 0; seg + 1 < bp.length; seg++) { - if(bp[seg + 1] <= rl) + for (int seg = 0; seg + 1 < bp.length; seg++) { + if (bp[seg + 1] <= rl) continue; - if(bp[seg] >= ru) + if (bp[seg] >= ru) break; final int newEnd = Math.min(bp[seg + 1], ru) - rl; slopeList.add(slopes[seg]); @@ -1320,7 +1369,7 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { bpList.add(newEnd); } - if(bpList.size() == 1) { + if (bpList.size() == 1) { slopeList.add(0.0); interceptList.add(0.0); bpList.add(newNRow); @@ -1333,8 +1382,8 @@ public AColGroup[] splitReshape(int multiplier, int nRow, int nColOrg) { } } - return new AColGroup[] {new ColGroupPiecewiseLinearCompressed(ColIndexFactory.create(newColIndices), newBp, - newSlopes, newIntercepts, newNRow)}; + return new AColGroup[]{new ColGroupPiecewiseLinearCompressed(ColIndexFactory.create(newColIndices), newBp, + newSlopes, newIntercepts, newNRow)}; } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java index 3eeea1cb233..72f1ad4e67c 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/functional/PiecewiseLinearUtils.java @@ -1,296 +1,295 @@ package org.apache.sysds.runtime.compress.colgroup.functional; + import org.apache.sysds.runtime.compress.CompressionSettings; import org.apache.sysds.runtime.matrix.data.MatrixBlock; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; public class PiecewiseLinearUtils { - /** - * Utility methods for piecewise linear compression of matric columns - * supports compression used the segmented least squares algorithm which is implemented with dynamic programming - * and a successive method, which puts all values in a segment till the target loss is exceeded - */ - - private PiecewiseLinearUtils() { - - } - - public static final class SegmentedRegression { - private final int[] breakpoints; - private final double[] slopes; - private final double[] intercepts; - - public SegmentedRegression(int[] breakpoints, double[] slopes, double[] intercepts) { - this.breakpoints = breakpoints; - this.slopes = slopes; - this.intercepts = intercepts; - } - - public int[] getBreakpoints() { - return breakpoints; - } - - public double[] getSlopes() { - return slopes; - } - - public double[] getIntercepts() { - return intercepts; - } - } - - public static double[] getColumn(MatrixBlock in, int colIndex) { - final int numRows = in.getNumRows(); - final double[] column = new double[numRows]; - - for (int row = 0; row < numRows; row++) { - column[row] = in.get(row, colIndex); - } - return column; - } - - public static SegmentedRegression compressSuccessivePiecewiseLinear(double[] column, CompressionSettings cs) { - //compute Breakpoints for a Column with a sukzessive breakpoints algorithm - - final List breakpointsList = computeBreakpointSuccessive(column, cs); - final int[] breakpoints = breakpointsList.stream().mapToInt(Integer::intValue).toArray(); - - //get values for Regression - final int numSeg = breakpoints.length - 1; - final double[] slopes = new double[numSeg]; - final double[] intercepts = new double[numSeg]; - - // Regress per Segment - for (int seg = 0; seg < numSeg; seg++) { - final int segstart = breakpoints[seg]; - final int segEnd = breakpoints[seg + 1]; - final double[] line = regressSegment(column, segstart, segEnd); - slopes[seg] = line[0]; - intercepts[seg] = line[1]; - } - return new SegmentedRegression(breakpoints, slopes, intercepts); - } - - /** - * Computes breakpoints for a column using segmented least squares with dynamic programming - * Iteratively reduces lambda to increase the number of segments until the target MSE is met. - * - * @param cs compression settings containing the target loss - * @param column the column values to segment - * @return list of breakpoint indices, starting with 0 - */ - public static List computeBreakpoints(CompressionSettings cs, double[] column) { - final int numElements = column.length; - final double targetMSE = cs.getPiecewiseTargetLoss(); - final double sseMax = numElements * targetMSE; // max allowed total SSE - - //start with high lambda an reduce iteratively - double lambda = Math.max(10.0, sseMax * 2.0); - List bestBreaks = Arrays.asList(0, numElements); - double bestSSE = computeTotalSSE(column, bestBreaks); - - for (int iter = 0; iter < 50; iter++) { - List breaks = computeBreakpointsLambda(column, lambda); - double totalSSE = computeTotalSSE(column, breaks); - int numSegs = breaks.size() - 1; - - if (totalSSE < bestSSE) { - bestSSE = totalSSE; - bestBreaks = new ArrayList<>(breaks); - } - //target loss reached - if (bestSSE <= sseMax) { - return bestBreaks; - } - - // only one segment left, break condition - if (numSegs <= 1) { - break; - } - // reducing lambda to allow more segments in next iteration - lambda *= 0.8; - } - - return bestBreaks; - } - - /** - * Computes optimal breakpoints, each segment has a SEE plus a - */ - - public static List computeBreakpointsLambda(double[] column, double lambda) { - final int n = column.length; - final double[] costs = new double[n + 1]; // min cost to reach i - final int[] prev = new int[n + 1]; - - Arrays.fill(costs, Double.POSITIVE_INFINITY); - costs[0] = 0.0; - // precompute all segment costs to avoid recomputation in dynamic programming - double[][] segCosts = new double[n + 1][n + 1]; - for (int i = 0; i < n; i++) { - for (int j = i + 1; j <= n; j++) { - segCosts[i][j] = computeSegmentCost(column, i, j); - } - } - // for each point j, find the cheapest previous breakpoint i - for (int j = 1; j <= n; j++) { - for (int i = 0; i < j; i++) { - // cost equals the SSE of segment [i,j] plus penalty plus best costs - double cost = costs[i] + segCosts[i][j] + lambda; - if (cost < costs[j]) { - costs[j] = cost; - prev[j] = i; - } - } - } - - // Backtrack to previous points to recover the breakpoints - List breaks = new ArrayList<>(); - int j = n; - while (j > 0) { - breaks.add(j); - j = prev[j]; - } - breaks.add(0); - Collections.reverse(breaks); - return breaks; - } - - /** - * computes the segment cost - * - * @param column column values - * @param start start index - * @param end end index - * @return SSE of the regression line over the segment - */ - public static double computeSegmentCost(double[] column, int start, int end) { - final int segSize = end - start; - if (segSize <= 1) - return 0.0; - - final double[] ab = regressSegment(column, start, end); - final double slope = ab[0]; - final double intercept = ab[1]; - - double sse = 0.0; - for (int i = start; i < end; i++) { - double err = column[i] - (slope * i + intercept); - sse += err * err; - } - return sse; - } - - /** - * computes the total SSE over all segments defined by the given breakpoints - * - * @param column - * @param breaks - * @return sum of the total SSE - */ - public static double computeTotalSSE(double[] column, List breaks) { - double total = 0.0; - for (int s = 0; s < breaks.size() - 1; s++) { - final int start = breaks.get(s); - final int end = breaks.get(s + 1); - total += computeSegmentCost(column, start, end); - } - return total; - } - - public static double[] regressSegment(double[] column, int start, int end) { - final int numElements = end - start; - if (numElements <= 0) - return new double[]{0.0, 0.0}; - - double sumOfRowIndices = 0, sumOfColumnValues = 0, sumOfRowIndicesSquared = 0, productRowIndexTimesColumnValue = 0; - for (int i = start; i < end; i++) { - sumOfRowIndices += i; - sumOfColumnValues += column[i]; - sumOfRowIndicesSquared += i * i; - productRowIndexTimesColumnValue += i * column[i]; - } - - - final double denominatorForSlope = - numElements * sumOfRowIndicesSquared - sumOfRowIndices * sumOfRowIndices; - final double slope; - final double intercept; - if (denominatorForSlope == 0) { - slope = 0.0; - intercept = sumOfColumnValues / numElements; - } else { - slope = (numElements * productRowIndexTimesColumnValue - sumOfRowIndices * sumOfColumnValues) / - denominatorForSlope; - intercept = (sumOfColumnValues - slope * sumOfRowIndices) / numElements; - } - return new double[]{slope, intercept}; - } - - /** - * computes breakpoints for a y using a successive algorithm - * extends each segment until the SEE reaches the target loss, then start a new segment - * - * @param y y values - * @param cs compression setting for setting the target loss - * @return list of breakpoint indices - */ - public static List computeBreakpointSuccessive(double[] y, CompressionSettings cs) { - final int numElements = y.length; - final double targetMSE = cs.getPiecewiseTargetLoss(); - if (Double.isNaN(targetMSE) || targetMSE <= 0) { - return Arrays.asList(0, numElements); // fallback single segment - } - - List breakpoints = new ArrayList<>(); - breakpoints.add(0); - double sumX = 0.0, sumY = 0.0, sumXY = 0.0, sumX2 = 0.0, sumY2 = 0.0; - int segmentLength = 0; - double beta, alpha; - - for (int n = 0; n < numElements; n++){ - double x = n; - double Y = y[n]; - sumX += x; - sumY += Y; - sumX2 += x * x; - sumY2 += Y * Y; - sumXY += x * Y; - segmentLength++; - - if(segmentLength > 1) { - - final double alphaBetaDenominator = segmentLength * sumX2 - sumX * sumX; - if (alphaBetaDenominator == 0.0){ - beta = 0.0; - alpha = sumY / segmentLength; - } - else{ - beta = (segmentLength * sumXY - sumX * sumY )/alphaBetaDenominator; - alpha = (sumY * sumX2 - sumX * sumXY)/alphaBetaDenominator; - } - - double sse = Math.max(0.0,sumY2 - alpha * sumY - beta * sumXY); //sum of least squares - if(sse > segmentLength * targetMSE){ - breakpoints.add(n); - segmentLength = 1; - sumX = x; - sumY = Y; - sumX2 = x * x; - sumY2 = Y * Y; - sumXY = x * Y; - } - } - } - - // make sure, that the last breakpoint equals numElements - int last = breakpoints.get(breakpoints.size() - 1); - if (last != numElements) { - breakpoints.add(numElements); - } - - return breakpoints; - } + /** + * Utility methods for piecewise linear compression of matric columns + * supports compression used the segmented least squares algorithm which is implemented with dynamic programming + * and a successive method, which puts all values in a segment till the target loss is exceeded + */ + + private PiecewiseLinearUtils() { + } + + public static final class SegmentedRegression { + private final int[] breakpoints; + private final double[] slopes; + private final double[] intercepts; + + public SegmentedRegression(int[] breakpoints, double[] slopes, double[] intercepts) { + this.breakpoints = breakpoints; + this.slopes = slopes; + this.intercepts = intercepts; + } + + public int[] getBreakpoints() { + return breakpoints; + } + + public double[] getSlopes() { + return slopes; + } + + public double[] getIntercepts() { + return intercepts; + } + } + + public static double[] getColumn(MatrixBlock in, int colIndex) { + final int numRows = in.getNumRows(); + final double[] column = new double[numRows]; + for (int row = 0; row < numRows; row++) { + column[row] = in.get(row, colIndex); + } + return column; + } + + public static SegmentedRegression compressSuccessivePiecewiseLinear(double[] column, CompressionSettings cs) { + //compute Breakpoints for a Column with a sukzessive breakpoints algorithm + + final List breakpointsList = computeBreakpointSuccessive(column, cs); + final int[] breakpoints = breakpointsList.stream().mapToInt(Integer::intValue).toArray(); + + //get values for Regression + final int numSeg = breakpoints.length - 1; + final double[] slopes = new double[numSeg]; + final double[] intercepts = new double[numSeg]; + + // Regress per Segment + for (int seg = 0; seg < numSeg; seg++) { + final int segstart = breakpoints[seg]; + final int segEnd = breakpoints[seg + 1]; + final double[] line = regressSegment(column, segstart, segEnd); + slopes[seg] = line[0]; + intercepts[seg] = line[1]; + } + return new SegmentedRegression(breakpoints, slopes, intercepts); + } + + /** + * Computes breakpoints for a column using segmented least squares with dynamic programming + * Iteratively reduces lambda to increase the number of segments until the target MSE is met. + * + * @param cs compression settings containing the target loss + * @param column the column values to segment + * @return list of breakpoint indices, starting with 0 + */ + public static List computeBreakpoints(CompressionSettings cs, double[] column) { + final int numElements = column.length; + final double targetMSE = cs.getPiecewiseTargetLoss(); + final double sseMax = numElements * targetMSE; // max allowed total SSE + + //start with high lambda an reduce iteratively + double lambda = Math.max(10.0, sseMax * 2.0); + List bestBreaks = Arrays.asList(0, numElements); + double bestSSE = computeTotalSSE(column, bestBreaks); + + for (int iter = 0; iter < 50; iter++) { + List breaks = computeBreakpointsLambda(column, lambda); + double totalSSE = computeTotalSSE(column, breaks); + int numSegs = breaks.size() - 1; + + if (totalSSE < bestSSE) { + bestSSE = totalSSE; + bestBreaks = new ArrayList<>(breaks); + } + //target loss reached + if (bestSSE <= sseMax) { + return bestBreaks; + } + + // only one segment left, break condition + if (numSegs <= 1) { + break; + } + // reducing lambda to allow more segments in next iteration + lambda *= 0.8; + } + + return bestBreaks; + } + + /** + * Computes optimal breakpoints, each segment has a SEE plus a + */ + + public static List computeBreakpointsLambda(double[] column, double lambda) { + final int n = column.length; + final double[] costs = new double[n + 1]; // min cost to reach i + final int[] prev = new int[n + 1]; + + Arrays.fill(costs, Double.POSITIVE_INFINITY); + costs[0] = 0.0; + // precompute all segment costs to avoid recomputation in dynamic programming + double[][] segCosts = new double[n + 1][n + 1]; + for (int i = 0; i < n; i++) { + for (int j = i + 1; j <= n; j++) { + segCosts[i][j] = computeSegmentCost(column, i, j); + } + } + // for each point j, find the cheapest previous breakpoint i + for (int j = 1; j <= n; j++) { + for (int i = 0; i < j; i++) { + // cost equals the SSE of segment [i,j] plus penalty plus best costs + double cost = costs[i] + segCosts[i][j] + lambda; + if (cost < costs[j]) { + costs[j] = cost; + prev[j] = i; + } + } + } + + // Backtrack to previous points to recover the breakpoints + List breaks = new ArrayList<>(); + int j = n; + while (j > 0) { + breaks.add(j); + j = prev[j]; + } + breaks.add(0); + Collections.reverse(breaks); + return breaks; + } + + /** + * computes the segment cost + * + * @param column column values + * @param start start index + * @param end end index + * @return SSE of the regression line over the segment + */ + public static double computeSegmentCost(double[] column, int start, int end) { + final int segSize = end - start; + if (segSize <= 1) + return 0.0; + + final double[] ab = regressSegment(column, start, end); + final double slope = ab[0]; + final double intercept = ab[1]; + + double sse = 0.0; + for (int i = start; i < end; i++) { + double err = column[i] - (slope * i + intercept); + sse += err * err; + } + return sse; + } + + /** + * computes the total SSE over all segments defined by the given breakpoints + * + * @param column + * @param breaks + * @return sum of the total SSE + */ + public static double computeTotalSSE(double[] column, List breaks) { + double total = 0.0; + for (int s = 0; s < breaks.size() - 1; s++) { + final int start = breaks.get(s); + final int end = breaks.get(s + 1); + total += computeSegmentCost(column, start, end); + } + return total; + } + + public static double[] regressSegment(double[] column, int start, int end) { + final int numElements = end - start; + if (numElements <= 0) + return new double[]{0.0, 0.0}; + + double sumOfRowIndices = 0, sumOfColumnValues = 0, sumOfRowIndicesSquared = 0, productRowIndexTimesColumnValue = 0; + for (int i = start; i < end; i++) { + sumOfRowIndices += i; + sumOfColumnValues += column[i]; + sumOfRowIndicesSquared += i * i; + productRowIndexTimesColumnValue += i * column[i]; + } + + + final double denominatorForSlope = + numElements * sumOfRowIndicesSquared - sumOfRowIndices * sumOfRowIndices; + final double slope; + final double intercept; + if (denominatorForSlope == 0) { + slope = 0.0; + intercept = sumOfColumnValues / numElements; + } else { + slope = (numElements * productRowIndexTimesColumnValue - sumOfRowIndices * sumOfColumnValues) / + denominatorForSlope; + intercept = (sumOfColumnValues - slope * sumOfRowIndices) / numElements; + } + return new double[]{slope, intercept}; + } + + /** + * computes breakpoints for a y using a successive algorithm + * extends each segment until the SEE reaches the target loss, then start a new segment + * + * @param y y values + * @param cs compression setting for setting the target loss + * @return list of breakpoint indices + */ + public static List computeBreakpointSuccessive(double[] y, CompressionSettings cs) { + final int numElements = y.length; + final double targetMSE = cs.getPiecewiseTargetLoss(); + if (Double.isNaN(targetMSE) || targetMSE <= 0) { + return Arrays.asList(0, numElements); // fallback single segment + } + + List breakpoints = new ArrayList<>(); + breakpoints.add(0); + double sumX = 0.0, sumY = 0.0, sumXY = 0.0, sumX2 = 0.0, sumY2 = 0.0; + int segmentLength = 0; + double beta, alpha; + + for (int n = 0; n < numElements; n++) { + double x = n; + double Y = y[n]; + sumX += x; + sumY += Y; + sumX2 += x * x; + sumY2 += Y * Y; + sumXY += x * Y; + segmentLength++; + + if (segmentLength > 1) { + + final double alphaBetaDenominator = segmentLength * sumX2 - sumX * sumX; + if (alphaBetaDenominator == 0.0) { + beta = 0.0; + alpha = sumY / segmentLength; + } else { + beta = (segmentLength * sumXY - sumX * sumY) / alphaBetaDenominator; + alpha = (sumY * sumX2 - sumX * sumXY) / alphaBetaDenominator; + } + + double sse = Math.max(0.0, sumY2 - alpha * sumY - beta * sumXY); //sum of least squares + if (sse > segmentLength * targetMSE) { + breakpoints.add(n); + segmentLength = 1; + sumX = x; + sumY = Y; + sumX2 = x * x; + sumY2 = Y * Y; + sumXY = x * Y; + } + } + } + + // make sure, that the last breakpoint equals numElements + int last = breakpoints.get(breakpoints.size() - 1); + if (last != numElements) { + breakpoints.add(numElements); + } + + return breakpoints; + } } diff --git a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java index cc10e272ad6..bf2b6e22a87 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java +++ b/src/test/java/org/apache/sysds/test/component/compress/colgroup/ColGroupPiecewiseLinearCompressedOperationsTest.java @@ -82,18 +82,18 @@ private MatrixBlock decompress(AColGroup cg) { /// check elementwise to compare results from compressed and decompressed matrixblock private void checkMatrixEquals(String msg, MatrixBlock mb1, MatrixBlock mb2) { - if(mb1.getNumRows() != mb2.getNumRows() || mb1.getNumColumns() != mb2.getNumColumns()) + if (mb1.getNumRows() != mb2.getNumRows() || mb1.getNumColumns() != mb2.getNumColumns()) fail(msg + " dimension mismatch"); - for(int r = 0; r < numRows; r++) - for(int c = 0; c < numCols; c++) + for (int r = 0; r < numRows; r++) + for (int c = 0; c < numCols; c++) assertEquals(msg + "[" + r + "," + c + "]", mb1.get(r, c), mb2.get(r, c), DELTA); } /// compute column sum to validate private double[] computeSums(MatrixBlock mb) { double[] sums = new double[numCols]; - for(int c = 0; c < numCols; c++) - for(int r = 0; r < numRows; r++) + for (int c = 0; c < numCols; c++) + for (int r = 0; r < numRows; r++) sums[c] += mb.get(r, c); return sums; } @@ -101,14 +101,14 @@ private double[] computeSums(MatrixBlock mb) { /// create row vector private double[] buildRowVector() { double[] v = new double[numCols]; - for(int i = 0; i < numCols; i++) + for (int i = 0; i < numCols; i++) v[i] = 0.5 * (i + 1); return v; } private int[] buildColArray(int n) { int[] cols = new int[n]; - for(int i = 0; i < n; i++) + for (int i = 0; i < n; i++) cols[i] = i; return cols; } @@ -116,8 +116,8 @@ private int[] buildColArray(int n) { private MatrixBlock applyBinaryRowOpLeft(MatrixBlock mb, BinaryOperator op, double[] v) { MatrixBlock result = new MatrixBlock(numRows, numCols, false); result.allocateDenseBlock(); - for(int r = 0; r < numRows; r++) - for(int c = 0; c < numCols; c++) + for (int r = 0; r < numRows; r++) + for (int c = 0; c < numCols; c++) result.getDenseBlock().set(r, c, op.fn.execute(v[c], mb.get(r, c))); return result; } @@ -125,8 +125,8 @@ private MatrixBlock applyBinaryRowOpLeft(MatrixBlock mb, BinaryOperator op, doub private MatrixBlock applyBinaryRowOpRight(MatrixBlock mb, BinaryOperator op, double[] v) { MatrixBlock result = new MatrixBlock(numRows, numCols, false); result.allocateDenseBlock(); - for(int r = 0; r < numRows; r++) - for(int c = 0; c < numCols; c++) + for (int r = 0; r < numRows; r++) + for (int c = 0; c < numCols; c++) result.getDenseBlock().set(r, c, op.fn.execute(mb.get(r, c), v[c])); return result; } @@ -136,7 +136,7 @@ public void testComputeSum() { double[] sumsComp = new double[1]; piecewiseLinearColGroup.computeSum(sumsComp, numRows); double expectedTotal = 0; - for(double s : computeSums(decompressedMB)) + for (double s : computeSums(decompressedMB)) expectedTotal += s; assertEquals(expectedTotal, sumsComp[0], DELTA); } @@ -151,12 +151,12 @@ public void testComputeColSums() { private void testScalarOp(ScalarOperator op, double scalar) { MatrixBlock expected = new MatrixBlock(numRows, numCols, false); expected.allocateDenseBlock(); - for(int r = 0; r < numRows; r++) - for(int c = 0; c < numCols; c++) + for (int r = 0; r < numRows; r++) + for (int c = 0; c < numCols; c++) expected.getDenseBlock().set(r, c, op.fn.execute(decompressedMB.get(r, c), scalar)); checkMatrixEquals("scalarOp " + op.fn.getClass().getSimpleName(), expected, - decompress(piecewiseLinearColGroup.scalarOperation(op))); + decompress(piecewiseLinearColGroup.scalarOperation(op))); } @Test @@ -184,7 +184,7 @@ public void testBinaryRowOpLeftPlus() { BinaryOperator op = new BinaryOperator(Plus.getPlusFnObject()); double[] v = buildRowVector(); checkMatrixEquals("binaryRowOpLeft Plus", applyBinaryRowOpLeft(decompressedMB, op, v), - decompress(piecewiseLinearColGroup.binaryRowOpLeft(op, v, false))); + decompress(piecewiseLinearColGroup.binaryRowOpLeft(op, v, false))); } @Test @@ -192,7 +192,7 @@ public void testBinaryRowOpLeftMultiply() { BinaryOperator op = new BinaryOperator(Multiply.getMultiplyFnObject()); double[] v = buildRowVector(); checkMatrixEquals("binaryRowOpLeft Multiply", applyBinaryRowOpLeft(decompressedMB, op, v), - decompress(piecewiseLinearColGroup.binaryRowOpLeft(op, v, false))); + decompress(piecewiseLinearColGroup.binaryRowOpLeft(op, v, false))); } @Test @@ -200,7 +200,7 @@ public void testBinaryRowOpRightMinus() { BinaryOperator op = new BinaryOperator(Minus.getMinusFnObject()); double[] v = buildRowVector(); checkMatrixEquals("binaryRowOpRight Minus", applyBinaryRowOpRight(decompressedMB, op, v), - decompress(piecewiseLinearColGroup.binaryRowOpRight(op, v, false))); + decompress(piecewiseLinearColGroup.binaryRowOpRight(op, v, false))); } @Test @@ -208,7 +208,7 @@ public void testBinaryRowOpRightDivide() { BinaryOperator op = new BinaryOperator(Divide.getDivideFnObject()); double[] v = buildRowVector(); checkMatrixEquals("binaryRowOpRight Divide", applyBinaryRowOpRight(decompressedMB, op, v), - decompress(piecewiseLinearColGroup.binaryRowOpRight(op, v, false))); + decompress(piecewiseLinearColGroup.binaryRowOpRight(op, v, false))); } @Test @@ -222,7 +222,7 @@ public void testContainsValueEndpoint() { int[] breakpoints = piecewiseLinearColGroup.getBreakpointsPerCol()[0]; double[] intercepts = piecewiseLinearColGroup.getInterceptsPerCol()[0]; double[] slopes = piecewiseLinearColGroup.getSlopesPerCol()[0]; - if(breakpoints.length > 1) { + if (breakpoints.length > 1) { double pattern = intercepts[0] + slopes[0] * (breakpoints[1] - breakpoints[0] - 1); assertTrue("endpoint of col 0 seg 0 should exist", piecewiseLinearColGroup.containsValue(pattern)); } @@ -231,8 +231,8 @@ public void testContainsValueEndpoint() { @Test public void testContainsValueConstantSegment() { ColGroupPiecewiseLinearCompressed cg = (ColGroupPiecewiseLinearCompressed) ColGroupPiecewiseLinearCompressed.create( - ColIndexFactory.create(new int[] {0}), new int[][] {{0, numRows}}, new double[][] {{0.0}}, - new double[][] {{1.23}}, numRows); + ColIndexFactory.create(new int[]{0}), new int[][]{{0, numRows}}, new double[][]{{0.0}}, + new double[][]{{1.23}}, numRows); assertTrue("constant value 1.23 should exist", cg.containsValue(1.23)); assertFalse("value 2.0 should not exist", cg.containsValue(2.0)); @@ -246,10 +246,10 @@ public void testContainsValueOutsideRange() { @Test public void testGetIdxMatchesDecompress() { - for(int c = 0; c < numCols; c++) - for(int r = 0; r < numRows; r++) + for (int c = 0; c < numCols; c++) + for (int r = 0; r < numRows; r++) assertEquals("getIdx(" + r + "," + c + ")", decompressedMB.get(r, c), - piecewiseLinearColGroup.getIdx(r, c), 1e-10); + piecewiseLinearColGroup.getIdx(r, c), 1e-10); } @Test @@ -263,7 +263,7 @@ public void testGetIdxInvalidBounds() { @Test public void testGetNumValues() { int expected = 0; - for(int c = 0; c < numCols; c++) { + for (int c = 0; c < numCols; c++) { int breakpointsLen = piecewiseLinearColGroup.getBreakpointsPerCol()[c].length; int slopesLen = piecewiseLinearColGroup.getSlopesPerCol()[c].length; int interceptsLen = piecewiseLinearColGroup.getInterceptsPerCol()[c].length; @@ -283,19 +283,19 @@ public void testGetExactSizeOnDisk() { int[] breakpoints = new int[numSegs + 1]; breakpoints[0] = 0; breakpoints[numSegs] = rows; - for(int s = 1; s < numSegs; s++) + for (int s = 1; s < numSegs; s++) breakpoints[s] = rng.nextInt(rows * 2 / 3) + rows / 10; double[] slopes = new double[numSegs]; double[] intercepts = new double[numSegs]; - for(int s = 0; s < numSegs; s++) { + for (int s = 0; s < numSegs; s++) { slopes[s] = rng.nextDouble() * 4 - 2; intercepts[s] = rng.nextDouble() * 4 - 2; } /// PLC Piecewise Linear Compressed AColGroup colGroupPLC = ColGroupPiecewiseLinearCompressed.create( - ColIndexFactory.create(new int[] {rng.nextInt(20)}), new int[][] {breakpoints}, new double[][] {slopes}, - new double[][] {intercepts}, rows); + ColIndexFactory.create(new int[]{rng.nextInt(20)}), new int[][]{breakpoints}, new double[][]{slopes}, + new double[][]{intercepts}, rows); assertTrue("disk size should be positive", colGroupPLC.getExactSizeOnDisk() > 0); assertTrue("num values should be positive", colGroupPLC.getNumValues() > 0); @@ -305,8 +305,8 @@ public void testGetExactSizeOnDisk() { public double[][] getRandomMatrix(int rows, int cols, double min, double max, double sparsity, long seed) { Random rng = new Random(seed); double[][] data = new double[rows][cols]; - for(int r = 0; r < rows; r++) - for(int c = 0; c < cols; c++) + for (int r = 0; r < rows; r++) + for (int c = 0; c < cols; c++) data[r][c] = min + rng.nextDouble() * (max - min); return data; } @@ -314,19 +314,20 @@ public double[][] getRandomMatrix(int rows, int cols, double min, double max, do @Test public void testCreate() { - ColGroupPiecewiseLinearCompressed plc =(ColGroupPiecewiseLinearCompressed) piecewiseLinearColGroup; + ColGroupPiecewiseLinearCompressed plc = (ColGroupPiecewiseLinearCompressed) piecewiseLinearColGroup; - AColGroup result = ColGroupPiecewiseLinearCompressed.create(plc.getColIndices(), plc.getBreakpointsPerCol(),plc.getSlopesPerCol(), plc.getInterceptsPerCol(), NROWS); + AColGroup result = ColGroupPiecewiseLinearCompressed.create(plc.getColIndices(), plc.getBreakpointsPerCol(), plc.getSlopesPerCol(), plc.getInterceptsPerCol(), NROWS); assertTrue(result instanceof ColGroupPiecewiseLinearCompressed); // equal to piecewiseLinearColGroup instance - assertArrayEquals(((ColGroupPiecewiseLinearCompressed) result).getBreakpointsPerCol(),plc.getBreakpointsPerCol()); - assertArrayEquals(((ColGroupPiecewiseLinearCompressed) result).getSlopesPerCol(),plc.getSlopesPerCol()); - assertArrayEquals(((ColGroupPiecewiseLinearCompressed) result).getInterceptsPerCol(),plc.getInterceptsPerCol()); + assertArrayEquals(((ColGroupPiecewiseLinearCompressed) result).getBreakpointsPerCol(), plc.getBreakpointsPerCol()); + assertArrayEquals(((ColGroupPiecewiseLinearCompressed) result).getSlopesPerCol(), plc.getSlopesPerCol()); + assertArrayEquals(((ColGroupPiecewiseLinearCompressed) result).getInterceptsPerCol(), plc.getInterceptsPerCol()); } + @Test public void testDecompressToDenseBlock() { - MatrixBlock mb_compare = new MatrixBlock(originalMB); + MatrixBlock mb_compare = new MatrixBlock(originalMB); mb_compare.recomputeNonZeros(); DenseBlock db_compare = mb_compare.getDenseBlock(); @@ -344,13 +345,13 @@ public void testDecompressToDenseBlock() { assertArrayEquals(db_result.values(NCOLS), db_compare.values(NCOLS), TARGET_LOSS); } - private double highest_loss(MatrixBlock result, MatrixBlock compare){ + private double highest_loss(MatrixBlock result, MatrixBlock compare) { // recompute non zeros result.recomputeNonZeros(); compare.recomputeNonZeros(); // asserEquals size correct - assertEquals(result.getNumRows(), compare.getNumRows()); + assertEquals(result.getNumRows(), compare.getNumRows()); assertEquals(result.getNumColumns(), compare.getNumColumns()); // MatrixBlock diff @@ -358,7 +359,7 @@ private double highest_loss(MatrixBlock result, MatrixBlock compare){ // binary Operation Minus ValueFunction fn = Minus.getMinusFnObject(); - BinaryOperator op = new BinaryOperator( fn); + BinaryOperator op = new BinaryOperator(fn); result.binaryOperations(op, compare, diff); // get max and min @@ -376,14 +377,14 @@ public void testUnaryOperationMultiply2() { AColGroup result = piecewiseLinearColGroup.unaryOperation(new UnaryOperator(fn)); assertTrue(result instanceof ColGroupUncompressed); - MatrixBlock resultMB = ((ColGroupUncompressed) result ).getData(); - MatrixBlock compareMB =compare; + MatrixBlock resultMB = ((ColGroupUncompressed) result).getData(); + MatrixBlock compareMB = compare; // do unaryOperation on compare - MatrixBlock compare_final = compare.unaryOperations(new UnaryOperator( fn)); + MatrixBlock compare_final = compare.unaryOperations(new UnaryOperator(fn)); // check if highest_loss smaller than worst case expected loss - double biggest_loss = highest_loss (resultMB, compareMB); + double biggest_loss = highest_loss(resultMB, compareMB); assertEquals(TARGET_LOSS * 2, Math.max(biggest_loss, TARGET_LOSS * 2), 0.0); } @@ -394,14 +395,14 @@ public void testUnaryOperationPower2() { AColGroup result = piecewiseLinearColGroup.unaryOperation(new UnaryOperator(fn)); assertTrue(result instanceof ColGroupUncompressed); - MatrixBlock resultMB = ((ColGroupUncompressed) result ).getData(); - MatrixBlock compareMB =compare; + MatrixBlock resultMB = ((ColGroupUncompressed) result).getData(); + MatrixBlock compareMB = compare; // do unaryOperation on compare - MatrixBlock compare_final = compare.unaryOperations(new UnaryOperator( fn)); + MatrixBlock compare_final = compare.unaryOperations(new UnaryOperator(fn)); // check if highest_loss smaller than worst case expected loss - double biggest_loss = highest_loss (resultMB, compareMB); + double biggest_loss = highest_loss(resultMB, compareMB); assertEquals(TARGET_LOSS * TARGET_LOSS, Math.max(biggest_loss, TARGET_LOSS * TARGET_LOSS), 0.0); }