Go: pass source root from autobuilder to extractor

This ensures the extractor can resolve the relative paths for files
changed in the overlay.

Author: nickrolfe

go/extractor/cli/go-autobuilder/go-autobuilder.go

@@ -448,7 +448,7 @@ func installDependencies(workspace project.GoWorkspace) {
 }
 
 // Run the extractor.
-func extract(workspace project.GoWorkspace) bool {
+func extract(workspace project.GoWorkspace, sourceRoot string) bool {
 	extractor, err := util.GetExtractorPath()
 	if err != nil {
 		log.Fatalf("Could not determine path of extractor: %v.\n", err)
@@ -459,6 +459,12 @@ func extract(workspace project.GoWorkspace) bool {
 		extractorArgs = append(extractorArgs, workspace.ModMode.ArgsForGoVersion(toolchain.GetEnvGoSemVer())...)
 	}
 
+	if util.IsOverlayExtraction() {
+		// When we are extracting an overlay, pass the source root to the extractor so that it knows
+		// how to resolve the relative paths in the list of changed files.
+		extractorArgs = append(extractorArgs, "--source-root", sourceRoot)
+	}
+
 	if len(workspace.Modules) == 0 {
 		// There may be no modules if we are using e.g. Dep or Glide
 		extractorArgs = append(extractorArgs, "./...")
@@ -587,6 +593,12 @@ func installDependenciesAndBuild() {
 		buildWithCustomCommands(inst)
 	}
 
+	// The autobuilder is invoked with its working directory set to the source directory.
+	sourceRoot, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("Failed to get current working directory: %s\n", err.Error())
+	}
+
 	// Attempt to extract all workspaces; we will tolerate individual extraction failures here
 	for i, workspace := range workspaces {
 		if workspace.ModMode == project.ModVendor {
@@ -607,7 +619,7 @@ func installDependenciesAndBuild() {
 			}
 		}
 
-		workspaces[i].Extracted = extract(workspace)
+		workspaces[i].Extracted = extract(workspace, sourceRoot)
 
 		if !workspaces[i].Extracted {
 			unsuccessfulProjects = append(unsuccessfulProjects, workspace.BaseDir)
@@ -632,6 +644,8 @@ func installDependenciesAndBuild() {
 	} else {
 		log.Printf("Success: extraction succeeded for all %d discovered project(s).\n", len(workspaces))
 	}
+
+	util.WriteOverlayBaseMetadata()
 }
 
 func main() {

go/extractor/cli/go-extractor/go-extractor.go

@@ -24,9 +24,10 @@ func usage() {
 
 // extractTests is set (a) if we were manually commanded to extract tests via the relevant
 // environment variable / extractor option, or (b) we're mimicking a `go test` command.
-func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []string, bool) {
+func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []string, bool, string) {
 	i := 0
 	buildFlags := []string{}
+	var sourceRoot string
 	for ; i < len(args) && strings.HasPrefix(args[i], "-"); i++ {
 		if args[i] == "--" {
 			i++
@@ -61,6 +62,18 @@ func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []strin
 				} else {
 					log.Fatalf("--mimic requires an argument, e.g. --mimic go")
 				}
+			case "--source-root":
+				// The extractor can be called by the autobuilder with the working directory set to
+				// the directory containing the workspace we're extracting, and this may be a
+				// subdirectory of the actual source root. This argument lets us resolve paths that
+				// are relative to that source root, e.g. for the list of overlay changed files.
+				if i+1 < len(args) {
+					i++
+					sourceRoot = args[i]
+					log.Printf("Source root is %s", sourceRoot)
+				} else {
+					log.Fatalf("--source-root requires an argument, e.g. --source-root /path/to/root")
+				}
 			}
 		}
 
@@ -93,14 +106,14 @@ func parseFlags(args []string, mimic bool, extractTests bool) ([]string, []strin
 	cpuprofile = os.Getenv("CODEQL_EXTRACTOR_GO_CPU_PROFILE")
 	memprofile = os.Getenv("CODEQL_EXTRACTOR_GO_MEM_PROFILE")
 
-	return buildFlags, args[i:], extractTests
+	return buildFlags, args[i:], extractTests, sourceRoot
 }
 
 func main() {
 	util.SetLogLevel()
 
 	extractTestsDefault := os.Getenv("CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_TESTS") == "true"
-	buildFlags, patterns, extractTests := parseFlags(os.Args[1:], false, extractTestsDefault)
+	buildFlags, patterns, extractTests, sourceRoot := parseFlags(os.Args[1:], false, extractTestsDefault)
 
 	if cpuprofile != "" {
 		f, err := os.Create(cpuprofile)
@@ -120,7 +133,7 @@ func main() {
 	}
 
 	log.Printf("Build flags: '%s'; patterns: '%s'\n", strings.Join(buildFlags, " "), strings.Join(patterns, " "))
-	err := extractor.ExtractWithFlags(buildFlags, patterns, extractTests)
+	err := extractor.ExtractWithFlags(buildFlags, patterns, extractTests, sourceRoot)
 	if err != nil {
 		errString := err.Error()
 		if strings.Contains(errString, "unexpected directory layout:") {

go/extractor/extractor.go

@@ -58,16 +58,11 @@ func init() {
 	}
 }
 
-// Extract extracts the packages specified by the given patterns
-func Extract(patterns []string) error {
-	return ExtractWithFlags(nil, patterns, false)
-}
-
 // ExtractWithFlags extracts the packages specified by the given patterns and build flags
-func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool) error {
+func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool, sourceRoot string) error {
 	startTime := time.Now()
 
-	extraction := NewExtraction(buildFlags, patterns)
+	extraction := NewExtraction(buildFlags, patterns, sourceRoot)
 	defer extraction.StatWriter.Close()
 
 	modEnabled := os.Getenv("GO111MODULE") != "off"
@@ -311,8 +306,6 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool)
 
 	extraction.WaitGroup.Wait()
 
-	util.WriteOverlayBaseMetadata()
-
 	log.Println("Done extracting packages.")
 
 	t := time.Now()
@@ -370,7 +363,7 @@ func (extraction *Extraction) GetNextErr(path string) int {
 	return res
 }
 
-func NewExtraction(buildFlags []string, patterns []string) *Extraction {
+func NewExtraction(buildFlags []string, patterns []string, sourceRoot string) *Extraction {
 	hash := md5.New()
 	io.WriteString(hash, "go")
 	for _, buildFlag := range buildFlags {
@@ -382,7 +375,7 @@ func NewExtraction(buildFlags []string, patterns []string) *Extraction {
 	}
 	sum := hash.Sum(nil)
 
-	overlayChangeList := util.GetOverlayChanges()
+	overlayChangeList := util.GetOverlayChanges(sourceRoot)
 	var overlayChanges map[string]bool
 	if overlayChangeList == nil {
 		overlayChanges = nil

go/extractor/util/overlays.go

@@ -4,12 +4,18 @@ import (
 	"encoding/json"
 	"log"
 	"os"
+	"path/filepath"
 )
 
+func IsOverlayExtraction() bool {
+	_, present := os.LookupEnv("CODEQL_EXTRACTOR_GO_OVERLAY_METADATA_IN")
+	return present
+}
+
 // If the relevant environment variable is set, indicating that we are extracting an overlay
 // database, GetOverlayChanges returns the list of relative paths of files that have changed (or
 // been deleted). Otherwise, it returns `nil`.
-func GetOverlayChanges() []string {
+func GetOverlayChanges(sourceRoot string) []string {
 	if overlayChangesJsonPath, present := os.LookupEnv("CODEQL_EXTRACTOR_GO_OVERLAY_CHANGES"); present {
 		log.Printf("Reading overlay changes from: %s", overlayChangesJsonPath)
 
@@ -28,7 +34,19 @@ func GetOverlayChanges() []string {
 			log.Fatalf("Failed to decode overlay changes JSON file: %s", err)
 		}
 
-		return overlayData.Changes
+		absPaths := make([]string, 0, len(overlayData.Changes))
+		if sourceRoot == "" {
+			// This shouldn't happen, because it implies the extractor was invoked in some way other
+			// than from the autobuilder. However, we'll only attempt to build an overlay if there
+			// exists an overlay _base_, and only the autobuilder writes the metadata file that
+			// ensures a database is created as an overlay-base.
+			log.Fatalf("Extractor is running in overlay mode, but --source-root was not provided")
+		}
+		for _, relPath := range overlayData.Changes {
+			absPaths = append(absPaths, filepath.Clean(sourceRoot+"/"+relPath))
+		}
+
+		return absPaths
 	} else {
 		return nil
 	}