Merge pull request #3716 from crazy-max/history-fix-finalize

history: finalize export records on their owning node

Author: tonistiigi

commands/history/export.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"io"
 	"os"
-	"slices"
 
 	"github.com/containerd/console"
 	"github.com/containerd/platforms"
@@ -53,37 +52,52 @@ func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) e
 			return errors.Errorf("no record found for ref %q", ref)
 		}
 
+		toExport := recs
+		if !opts.all && ref == "" {
+			latestRef := recs[0].Ref
+			recCount := 0
+			for _, rec := range recs {
+				if rec.Ref != latestRef {
+					break
+				}
+				recCount++
+			}
+			toExport = recs[:recCount]
+		}
 		if opts.finalize {
+			seen := make(map[string]struct{}, len(toExport))
 			var finalized bool
-			for _, rec := range recs {
-				if rec.Trace == nil {
-					finalized = true
-					if err := finalizeRecord(ctx, rec.Ref, nodes); err != nil {
-						return err
-					}
+			for _, rec := range toExport {
+				if rec.node == nil || rec.Trace != nil {
+					continue
+				}
+				key := rec.node.Builder + "\x00" + rec.node.Name + "\x00" + rec.Ref
+				if _, ok := seen[key]; ok {
+					continue
+				}
+				seen[key] = struct{}{}
+				finalized = true
+				if err := finalizeRecord(ctx, rec.Ref, *rec.node); err != nil {
+					return err
 				}
 			}
 			if finalized {
-				recs, err = queryRecords(ctx, ref, nodes, &queryOptions{
+				queryRef := ref
+				if !opts.all {
+					queryRef = toExport[0].Ref
+				}
+				recs, err = queryRecords(ctx, queryRef, nodes, &queryOptions{
 					CompletedOnly: true,
 				})
 				if err != nil {
 					return err
 				}
+				toExport = recs
 			}
 		}
-
-		if ref == "" {
-			slices.SortFunc(recs, func(a, b historyRecord) int {
-				return b.CreatedAt.AsTime().Compare(a.CreatedAt.AsTime())
-			})
-		}
-
+		res = append(res, toExport...)
 		if opts.all {
-			res = append(res, recs...)
 			break
-		} else {
-			res = append(res, recs[0])
 		}
 	}
 

commands/history/trace.go

@@ -56,7 +56,7 @@ func loadTrace(ctx context.Context, ref string, nodes []builder.Node) (string, [
 		// build is complete but no trace yet. try to finalize the trace
 		time.Sleep(1 * time.Second) // give some extra time for last parts of trace to be written
 
-		err := finalizeRecord(ctx, rec.Ref, []builder.Node{*rec.node})
+		err := finalizeRecord(ctx, rec.Ref, *rec.node)
 		if err != nil {
 			return "", nil, err
 		}

commands/history/utils.go

@@ -257,25 +257,19 @@ func queryRecords(ctx context.Context, ref string, nodes []builder.Node, opts *q
 	return out, nil
 }
 
-func finalizeRecord(ctx context.Context, ref string, nodes []builder.Node) error {
-	eg, ctx := errgroup.WithContext(ctx)
-	for _, node := range nodes {
-		eg.Go(func() error {
-			if node.Driver == nil {
-				return nil
-			}
-			c, err := node.Driver.Client(ctx)
-			if err != nil {
-				return err
-			}
-			_, err = c.ControlClient().UpdateBuildHistory(ctx, &controlapi.UpdateBuildHistoryRequest{
-				Ref:      ref,
-				Finalize: true,
-			})
-			return err
-		})
+func finalizeRecord(ctx context.Context, ref string, node builder.Node) error {
+	if node.Driver == nil {
+		return nil
 	}
-	return eg.Wait()
+	c, err := node.Driver.Client(ctx)
+	if err != nil {
+		return err
+	}
+	_, err = c.ControlClient().UpdateBuildHistory(ctx, &controlapi.UpdateBuildHistoryRequest{
+		Ref:      ref,
+		Finalize: true,
+	})
+	return err
 }
 
 func formatDuration(d time.Duration) string {

hack/test-driver

@@ -134,6 +134,8 @@ buildxCmd build ${buildPlatformFlag} \
   --metadata-file="${context}/metadata-build.json" \
   "${context}"
 cat "${context}/metadata-build.json"
+buildRef=$(awk -F'"' '/"buildx.build.ref"/ {print $4; exit}' "${context}/metadata-build.json")
+buildID=${buildRef##*/}
 
 # load to docker store
 if [ "$DRIVER" != "docker" ]; then
@@ -143,6 +145,15 @@ if [ "$DRIVER" != "docker" ]; then
     "${context}"
 fi
 
+# list build records
+buildxCmd --builder="${builderName}" history ls
+
+# export build records
+buildxCmd --builder="${builderName}" history export --finalize "${buildID}" --output "${context}/record.dockerbuild"
+file "${context}/record.dockerbuild"
+buildxCmd --builder="${builderName}" history export --finalize --all --output "${context}/records.dockerbuild"
+file "${context}/records.dockerbuild"
+
 # create bake def
 cat > "${bakedef}" <<EOL
 group "default" {