diff --git a/api/v1beta1/spec.go b/api/v1beta1/spec.go index b363d90f..fd319c88 100644 --- a/api/v1beta1/spec.go +++ b/api/v1beta1/spec.go @@ -861,8 +861,13 @@ type RemoteURL struct { // Sveltos fetches the content on every reconciliation and redeploys if the // content hash has changed. // Supported schemes: - // "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - // "oci://" — OCI registry artifact containing YAML manifests + // "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + // document, a gzip-compressed document, an + // uncompressed tar, or a gzip-compressed tar of + // .yaml/.yml/.json files + // "oci://" — OCI registry artifact whose layers are accepted + // in the same shapes: raw YAML/JSON, gzip-compressed + // YAML/JSON, uncompressed tar, or gzip-compressed tar // +kubebuilder:validation:Pattern=`^(https?|oci)://` URL string `json:"url"` diff --git a/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml b/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml index 8b72804d..ee597a98 100644 --- a/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml +++ b/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml @@ -1223,8 +1223,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: diff --git a/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml b/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml index b92aab89..7a8836c7 100644 --- a/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml +++ b/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml @@ -1125,8 +1125,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: @@ -2571,8 +2576,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: @@ -2989,8 +2999,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: diff --git a/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml b/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml index da5a5332..175e9265 100644 --- a/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml +++ b/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml @@ -1262,8 +1262,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: diff --git a/config/crd/bases/config.projectsveltos.io_profiles.yaml b/config/crd/bases/config.projectsveltos.io_profiles.yaml index 9159af72..34f26eb4 100644 --- a/config/crd/bases/config.projectsveltos.io_profiles.yaml +++ b/config/crd/bases/config.projectsveltos.io_profiles.yaml @@ -1223,8 +1223,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: diff --git a/controllers/handlers_kustomize.go b/controllers/handlers_kustomize.go index 6a9456b8..145bdf71 100644 --- a/controllers/handlers_kustomize.go +++ b/controllers/handlers_kustomize.go @@ -18,6 +18,7 @@ package controllers import ( archivetar "archive/tar" + "bufio" "bytes" "compress/gzip" "context" @@ -479,14 +480,15 @@ func getHashFromKustomizationRef(ctx context.Context, c client.Client, clusterSu } // getHashFromRemoteKustomizeURL fetches the content referenced by a KustomizationRef's -// RemoteURL and returns a hash covering it. It reuses fetchContent (rather than -// fetchContentToDir) since only a byte stream that changes when the content changes is -// needed here; the directory structure is only relevant when the content is actually -// extracted for a kustomize build. +// RemoteURL and returns a hash covering it. It uses fetchContentForHash, which returns +// the same raw bytes fetchContentToDir extracts to disk for deployment (rather than +// fetchContent's flattened .yaml/.yml/.json-only view), so the hash changes whenever +// anything fetchContentToDir would actually deploy changes — including non-manifest +// files a kustomize build depends on (patches, generators, etc.) and directory structure. func getHashFromRemoteKustomizeURL(ctx context.Context, remoteURL *configv1beta1.RemoteKustomizeURL, clusterSummary *configv1beta1.ClusterSummary, logger logr.Logger) ([]byte, error) { - body, err := fetchContent(ctx, remoteURL.URL, remoteURL.SecretRef, + body, err := fetchContentForHash(ctx, remoteURL.URL, remoteURL.SecretRef, clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, clusterSummary.Spec.ClusterType, logger) if err != nil { @@ -1247,6 +1249,11 @@ func deployEachKustomizeRefs(ctx context.Context, c client.Client, remoteRestCon return localResourceReports, remoteResourceReports, err } +// extractTarGz extracts the tar archive at src into dest, preserving directory +// structure. The archive may be gzip-compressed (the layout produced by tools +// such as `flux push artifact`) or an uncompressed tar; the gzip magic bytes +// are sniffed to tell which. macOS AppleDouble sidecar entries and PAX header +// entries are skipped (see isSkippableTarEntry) rather than extracted. func extractTarGz(src, dest string) error { // Open the tarball for reading tarball, err := os.Open(src) @@ -1255,15 +1262,25 @@ func extractTarGz(src, dest string) error { } defer tarball.Close() - // Create a gzip reader to decompress the tarball - gzipReader, err := gzip.NewReader(io.LimitReader(tarball, maxSize)) - if err != nil { + bufReader := bufio.NewReader(io.LimitReader(tarball, maxSize)) + magic, err := bufReader.Peek(len(gzipMagicBytes)) + if err != nil && !errors.Is(err, io.EOF) { return err } - defer gzipReader.Close() - // Create a tar reader to read the uncompressed tarball - tarReader := archivetar.NewReader(gzipReader) + // tarSource yields the uncompressed tar stream: the gzip-decompressed + // reader if the archive is gzip-compressed, or bufReader itself otherwise. + tarSource := io.Reader(bufReader) + if bytes.Equal(magic, gzipMagicBytes) { + gzipReader, err := gzip.NewReader(bufReader) + if err != nil { + return err + } + defer gzipReader.Close() + tarSource = gzipReader + } + + tarReader := archivetar.NewReader(tarSource) // Iterate over each file in the tarball and extract it to the destination for { @@ -1275,6 +1292,10 @@ func extractTarGz(src, dest string) error { return err } + if isSkippableTarEntry(header.Name) { + continue + } + target := filepath.Join(dest, filepath.Clean(header.Name)) if !strings.HasPrefix(target, dest) { return fmt.Errorf("tar archive entry %q is outside of destination directory", header.Name) diff --git a/controllers/handlers_kustomize_test.go b/controllers/handlers_kustomize_test.go index 70be9017..116452d3 100644 --- a/controllers/handlers_kustomize_test.go +++ b/controllers/handlers_kustomize_test.go @@ -18,6 +18,7 @@ package controllers_test import ( "archive/tar" + "bytes" "compress/gzip" "context" "fmt" @@ -247,6 +248,55 @@ var _ = Describe("KustomizeRefs", func() { _, err = os.Stat(extraFilePath) Expect(os.IsNotExist(err)).To(BeTrue()) }) + + It("extractTarGz extracts an uncompressed tar archive", func() { + srcDir, err := os.MkdirTemp("", "test") + Expect(err).To(BeNil()) + defer os.RemoveAll(srcDir) + + src := filepath.Join(srcDir, "plain.tar") + writeTarArchive(src, false, map[string]string{ + "overlay.yaml": "namePrefix: uncompressed-\n", + }) + + dest, err := os.MkdirTemp("", "extract") + Expect(err).To(BeNil()) + defer os.RemoveAll(dest) + + Expect(controllers.ExtractTarGz(src, dest)).To(Succeed()) + + contents, err := os.ReadFile(filepath.Join(dest, "overlay.yaml")) + Expect(err).To(BeNil()) + Expect(string(contents)).To(ContainSubstring("namePrefix")) + }) + + It("extractTarGz skips AppleDouble and PAX header entries", func() { + srcDir, err := os.MkdirTemp("", "test") + Expect(err).To(BeNil()) + defer os.RemoveAll(srcDir) + + src := filepath.Join(srcDir, "with-junk.tar.gz") + writeTarArchive(src, true, map[string]string{ + "kustomization.yaml": "resources:\n- deploy.yaml\n", + "._kustomization.yaml": "\x00\x05\x16\x07AppleDouble binary junk", + "PaxHeader/deploy.yaml": "not a real manifest", + }) + + dest, err := os.MkdirTemp("", "extract") + Expect(err).To(BeNil()) + defer os.RemoveAll(dest) + + Expect(controllers.ExtractTarGz(src, dest)).To(Succeed()) + + _, err = os.ReadFile(filepath.Join(dest, "kustomization.yaml")) + Expect(err).To(BeNil()) + + _, err = os.Stat(filepath.Join(dest, "._kustomization.yaml")) + Expect(os.IsNotExist(err)).To(BeTrue()) + + _, err = os.Stat(filepath.Join(dest, "PaxHeader", "deploy.yaml")) + Expect(os.IsNotExist(err)).To(BeTrue()) + }) }) var _ = Describe("Hash methods", func() { @@ -611,6 +661,37 @@ var _ = Describe("Hash methods", func() { }) }) +// writeTarArchive builds a tar archive from entries (name -> content) and writes it to +// dest. When gz is true the archive is gzip-compressed first. +func writeTarArchive(dest string, gz bool, entries map[string]string) { + var tarBuf bytes.Buffer + tw := tar.NewWriter(&tarBuf) + for name, content := range entries { + hdr := &tar.Header{ + Name: name, + Typeflag: tar.TypeReg, + Mode: 0600, + Size: int64(len(content)), + } + Expect(tw.WriteHeader(hdr)).To(Succeed()) + _, err := tw.Write([]byte(content)) + Expect(err).To(BeNil()) + } + Expect(tw.Close()).To(Succeed()) + + raw := tarBuf.Bytes() + if gz { + var gzBuf bytes.Buffer + gw := gzip.NewWriter(&gzBuf) + _, err := gw.Write(raw) + Expect(err).To(BeNil()) + Expect(gw.Close()).To(Succeed()) + raw = gzBuf.Bytes() + } + + Expect(os.WriteFile(dest, raw, 0600)).To(Succeed()) +} + func createTarGz(dest string) { // Create the test directory and some test files. err := os.MkdirAll("testdata/testdir", 0755) diff --git a/controllers/url_source.go b/controllers/url_source.go index 121d551b..92ebda3e 100644 --- a/controllers/url_source.go +++ b/controllers/url_source.go @@ -19,6 +19,7 @@ package controllers import ( "archive/tar" "bytes" + "compress/gzip" "context" "crypto/tls" "crypto/x509" @@ -131,8 +132,12 @@ func fetchURL(ctx context.Context, rawURL string, secretRef *corev1.SecretRefere } // fetchContent retrieves raw manifest bytes from a remote source. -// It dispatches on the URL scheme: "oci://" routes to fetchOCI; everything -// else is handled by fetchURL. +// It dispatches on the URL scheme: "oci://" routes to fetchOCI; everything else is +// fetched via fetchURL. Either way, the fetched bytes are normalized through +// extractYAMLFromLayer, so an http(s) endpoint may return a raw YAML/JSON document +// (the original contract), a gzip-compressed YAML/JSON document, an uncompressed tar, +// or a gzip-compressed tar of multiple .yaml/.yml/.json files — the same set of shapes +// already accepted per-layer for oci://. // The secretRef Name and Namespace are treated as Go templates and instantiated // against the target cluster, following the same convention as PolicyRef ConfigMap/Secret // references. When Namespace is empty it defaults to clusterNamespace. @@ -143,13 +148,20 @@ func fetchContent(ctx context.Context, rawURL string, secretRef *corev1.SecretRe if strings.HasPrefix(rawURL, "oci://") { return fetchOCI(ctx, rawURL, secretRef, clusterNamespace, clusterName, clusterType, logger) } - return fetchURL(ctx, rawURL, secretRef, clusterNamespace, clusterName, clusterType, logger) + + body, err := fetchURL(ctx, rawURL, secretRef, clusterNamespace, clusterName, clusterType, logger) + if err != nil { + return nil, err + } + return extractYAMLFromLayer(body, rawURL, logger) } // fetchOCI pulls an OCI artifact and returns its YAML content. -// The artifact layers are read as gzipped tar archives; if a layer is not a -// valid tar archive it is used as-is (raw YAML blob). All .yaml/.yml/.json -// files found across all layers are concatenated and returned. +// Each layer may be a gzip-compressed tar (e.g. the layout produced by +// `flux push artifact`), an uncompressed tar, or a raw YAML/JSON blob; +// gzip-compressed layers are decompressed before the tar is read. All +// .yaml/.yml/.json files found across all layers (or the raw blob itself) +// are concatenated and returned. // Supported Secret keys: "token" (pre-obtained bearer token), "username"+"password" (basic auth // exchanged for a registry token), "caFile" (PEM CA for TLS verification). // The secretRef Name and Namespace support Go templating against the target cluster. @@ -278,36 +290,50 @@ func pullOCIArtifactLayers(ctx context.Context, rawURL string, secretRef *corev1 } // extractYAMLFromLayer extracts YAML/JSON manifest bytes from a single OCI layer. -// It first attempts to parse the bytes as a tar archive and concatenates all -// .yaml/.yml/.json files found inside. If the bytes are not a valid tar archive -// (e.g. a raw YAML blob), they are returned as-is. +// If the layer is gzip-compressed, it is decompressed first. It then attempts +// to parse the (possibly decompressed) bytes as a tar archive and concatenates +// all .yaml/.yml/.json files found inside, skipping AppleDouble sidecar files +// and PAX header entries (see isYAMLManifestEntry). If the bytes are not a +// valid tar archive (e.g. a raw YAML blob), they are returned as-is. func extractYAMLFromLayer(raw []byte, rawURL string, logger logr.Logger) ([]byte, error) { - tr := tar.NewReader(bytes.NewReader(raw)) + content := raw + if isGzip(raw) { + decompressed, err := gunzipBytes(raw) + if err != nil { + // Magic bytes matched but the stream isn't valid gzip — fall back + // to treating the original bytes as raw content, consistent with + // how a non-tar layer is handled below. + logger.V(logs.LogDebug).Info(fmt.Sprintf( + "OCI layer from %s has a gzip header but failed to decompress, using as raw content: %v", + rawURL, err)) + return raw, nil + } + content = decompressed + } + + tr := tar.NewReader(bytes.NewReader(content)) hdr, err := tr.Next() if err == io.EOF { // Valid tar archive with no entries. return []byte{}, nil } if err != nil { - // Not a tar archive — treat the raw bytes as YAML/JSON content directly. + // Not a tar archive — treat the bytes as YAML/JSON content directly. logger.V(logs.LogDebug).Info(fmt.Sprintf( "OCI layer from %s is not a tar archive, using as raw content", rawURL)) - return raw, nil + return content, nil } var buf bytes.Buffer for { - if hdr.Typeflag == tar.TypeReg { - ext := strings.ToLower(filepath.Ext(hdr.Name)) - if ext == ".yaml" || ext == ".yml" || ext == ".json" { - content, err := io.ReadAll(tr) - if err != nil { - return nil, fmt.Errorf("failed to read %s from OCI artifact %s: %w", - hdr.Name, rawURL, err) - } - buf.Write(content) - buf.WriteString("\n---\n") + if hdr.Typeflag == tar.TypeReg && isYAMLManifestEntry(hdr.Name) { + content, err := io.ReadAll(tr) + if err != nil { + return nil, fmt.Errorf("failed to read %s from OCI artifact %s: %w", + hdr.Name, rawURL, err) } + buf.Write(content) + buf.WriteString("\n---\n") } hdr, err = tr.Next() if err == io.EOF { @@ -320,6 +346,57 @@ func extractYAMLFromLayer(raw []byte, rawURL string, logger logr.Logger) ([]byte return buf.Bytes(), nil } +var ( + // gzipMagicBytes is the two-byte header identifying a gzip stream (RFC 1952). + gzipMagicBytes = []byte{0x1f, 0x8b} +) + +// isGzip reports whether raw starts with the gzip magic bytes. +func isGzip(raw []byte) bool { + return bytes.HasPrefix(raw, gzipMagicBytes) +} + +// isSkippableTarEntry reports whether a tar entry is not real archive +// content and should never be extracted or read, regardless of its name's +// extension: +// - macOS AppleDouble sidecar files (e.g. "._ns.yaml"), written alongside +// the real file by bsdtar's default extended-attribute handling; they +// can carry the same extension as the real file but their content is +// binary AppleDouble data. +// - PAX extended header entries some tar implementations materialize as +// ordinary path entries (e.g. under a "PaxHeader/" prefix) instead of +// merging them into the following entry. +func isSkippableTarEntry(name string) bool { + if strings.HasPrefix(filepath.Base(name), "._") { + return true + } + return strings.Contains(name, "PaxHeader") +} + +// isYAMLManifestEntry reports whether a tar entry name is a genuine +// YAML/JSON manifest file, filtering out skippable entries (see +// isSkippableTarEntry) that happen to share the extension but aren't +// manifest content. +func isYAMLManifestEntry(name string) bool { + if isSkippableTarEntry(name) { + return false + } + + ext := strings.ToLower(filepath.Ext(name)) + return ext == ".yaml" || ext == ".yml" || ext == ".json" +} + +// gunzipBytes decompresses a gzip-compressed byte slice in full. +func gunzipBytes(raw []byte) ([]byte, error) { + gr, err := gzip.NewReader(bytes.NewReader(raw)) + if err != nil { + return nil, err + } + defer gr.Close() + + return io.ReadAll(gr) +} + // deployContentOfURL fetches YAML/JSON content from a remote source and deploys it // to the destination cluster using the same pipeline as ConfigMap/Secret sources. func deployContentOfURL(ctx context.Context, deployingToMgmtCluster bool, destConfig *rest.Config, @@ -403,8 +480,9 @@ func minKustomizeURLInterval(refs []configv1beta1.KustomizationRef) time.Duratio // extracts it into destDir, preserving the relative paths of every file so that // kustomize build can resolve files referenced by relative path (bases, resources, // patches, generator files, and so on). -// It dispatches on the URL scheme: "oci://" routes to fetchOCIToDir; everything else -// is fetched via fetchURL and treated as a gzipped tarball (.tar.gz). +// It dispatches on the URL scheme: "oci://" pulls and concatenates every OCI layer; +// everything else is fetched via fetchURL as a single archive. Either a gzip-compressed +// or an uncompressed tar is accepted (see extractTarGz). func fetchContentToDir(ctx context.Context, rawURL string, secretRef *corev1.SecretReference, clusterNamespace, clusterName string, clusterType libsveltosv1beta1.ClusterType, destDir string, logger logr.Logger) error { @@ -429,6 +507,30 @@ func fetchContentToDir(ctx context.Context, rawURL string, secretRef *corev1.Sec return extractTarGzBytes(body, destDir, 0) } +// fetchContentForHash retrieves the same raw bytes that fetchContentToDir would extract +// for rawURL — every OCI layer concatenated in manifest order, or the single HTTP/HTTPS +// response body — without extracting anything to disk. Hashing these bytes (rather than +// re-deriving a separate representation of the content) keeps drift-detection hashes for +// KustomizationRef.RemoteURL in sync with exactly what fetchContentToDir will deploy. +func fetchContentForHash(ctx context.Context, rawURL string, secretRef *corev1.SecretReference, + clusterNamespace, clusterName string, clusterType libsveltosv1beta1.ClusterType, + logger logr.Logger) ([]byte, error) { + + if strings.HasPrefix(rawURL, "oci://") { + layers, err := pullOCIArtifactLayers(ctx, rawURL, secretRef, clusterNamespace, clusterName, clusterType, logger) + if err != nil { + return nil, err + } + var buf bytes.Buffer + for _, raw := range layers { + buf.Write(raw) + } + return buf.Bytes(), nil + } + + return fetchURL(ctx, rawURL, secretRef, clusterNamespace, clusterName, clusterType, logger) +} + // extractTarGzBytes writes a gzipped tarball's raw bytes to a temporary file inside // destDir and extracts it in place using extractTarGz, preserving the archive's // directory structure. index disambiguates the temp file name when extracting diff --git a/controllers/url_source_test.go b/controllers/url_source_test.go index fd99dabc..1fd80d2e 100644 --- a/controllers/url_source_test.go +++ b/controllers/url_source_test.go @@ -19,16 +19,20 @@ package controllers import ( "archive/tar" "bytes" + "compress/gzip" + "context" + "net/http" + "net/http/httptest" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/go-logr/logr" + + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" ) // buildTar creates an uncompressed tar archive containing the given files. -// extractYAMLFromLayer receives already-uncompressed layer bytes, so no gzip -// wrapper is needed in tests. func buildTar(files map[string]string) []byte { var buf bytes.Buffer tw := tar.NewWriter(&buf) @@ -46,6 +50,17 @@ func buildTar(files map[string]string) []byte { return buf.Bytes() } +// gzipBytes gzip-compresses raw, mirroring the layout produced by tools such +// as `flux push artifact`. +func gzipBytes(raw []byte) []byte { + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + _, err := gw.Write(raw) + Expect(err).ToNot(HaveOccurred()) + Expect(gw.Close()).To(Succeed()) + return buf.Bytes() +} + var _ = Describe("extractYAMLFromLayer", func() { var logger logr.Logger const rawURL = "oci://example.com/unit/space/slug:head" @@ -105,4 +120,127 @@ var _ = Describe("extractYAMLFromLayer", func() { Expect(err).ToNot(HaveOccurred()) Expect(got).To(BeEmpty()) }) + + It("extracts .yaml files from a gzip-compressed tar archive", func() { + svc := "apiVersion: v1\nkind: Service\nmetadata:\n name: gzipped\n" + raw := gzipBytes(buildTar(map[string]string{ + "svc.yaml": svc, + "notes.md": "not a manifest", + })) + + got, err := extractYAMLFromLayer(raw, rawURL, logger) + Expect(err).ToNot(HaveOccurred()) + s := string(got) + Expect(s).To(ContainSubstring(svc)) + Expect(s).NotTo(ContainSubstring("notes")) + }) + + It("returns the decompressed bytes unchanged when a gzip layer is not a tar archive", func() { + raw := gzipBytes([]byte("apiVersion: v1\nkind: ConfigMap\n")) + + got, err := extractYAMLFromLayer(raw, rawURL, logger) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal([]byte("apiVersion: v1\nkind: ConfigMap\n"))) + }) + + It("returns raw bytes unchanged when the gzip header is present but the stream is invalid", func() { + raw := append([]byte{0x1f, 0x8b}, []byte("not actually gzip")...) + + got, err := extractYAMLFromLayer(raw, rawURL, logger) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(raw)) + }) + + It("skips macOS AppleDouble sidecar entries", func() { + ns := "apiVersion: v1\nkind: Namespace\nmetadata:\n name: sidecar-test\n" + raw := buildTar(map[string]string{ + "ns.yaml": ns, + "._ns.yaml": "\x00\x05\x16\x07\x00\x02\x00\x00AppleDouble binary junk", + }) + + got, err := extractYAMLFromLayer(raw, rawURL, logger) + Expect(err).ToNot(HaveOccurred()) + s := string(got) + Expect(s).To(ContainSubstring(ns)) + Expect(s).NotTo(ContainSubstring("AppleDouble")) + }) + + It("skips PAX header entries", func() { + crb := "apiVersion: v1\nkind: ClusterRoleBinding\nmetadata:\n name: pax-test\n" + raw := buildTar(map[string]string{ + "crb.yaml": crb, + "PaxHeader/crb2.yaml": "not a real manifest", + }) + + got, err := extractYAMLFromLayer(raw, rawURL, logger) + Expect(err).ToNot(HaveOccurred()) + s := string(got) + Expect(s).To(ContainSubstring(crb)) + Expect(s).NotTo(ContainSubstring("not a real manifest")) + }) +}) + +var _ = Describe("fetchContent (http/https)", func() { + var logger logr.Logger + + BeforeEach(func() { + logger = logr.Discard() + }) + + // serve starts an httptest server returning body for every request, and + // returns its URL alongside a cleanup func. + serve := func(body []byte) (string, func()) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, err := w.Write(body) + Expect(err).ToNot(HaveOccurred()) + })) + return server.URL, server.Close + } + + It("returns a raw YAML document unchanged", func() { + raw := []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: http-raw\n") + url, cleanup := serve(raw) + defer cleanup() + + got, err := fetchContent(context.TODO(), url, nil, "", "", libsveltosv1beta1.ClusterTypeCapi, logger) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(raw)) + }) + + It("decompresses a gzip-compressed YAML document", func() { + raw := []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: http-gzip\n") + url, cleanup := serve(gzipBytes(raw)) + defer cleanup() + + got, err := fetchContent(context.TODO(), url, nil, "", "", libsveltosv1beta1.ClusterTypeCapi, logger) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(Equal(raw)) + }) + + It("extracts .yaml files from a gzip-compressed tar archive (tar.gz)", func() { + dep := "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: http-targz\n" + archive := gzipBytes(buildTar(map[string]string{ + "deploy.yaml": dep, + "CHANGELOG": "irrelevant changelog entry", + })) + url, cleanup := serve(archive) + defer cleanup() + + got, err := fetchContent(context.TODO(), url, nil, "", "", libsveltosv1beta1.ClusterTypeCapi, logger) + Expect(err).ToNot(HaveOccurred()) + s := string(got) + Expect(s).To(ContainSubstring(dep)) + Expect(s).NotTo(ContainSubstring("changelog")) + }) + + It("extracts .yaml files from an uncompressed tar archive", func() { + svc := "apiVersion: v1\nkind: Service\nmetadata:\n name: http-tar\n" + archive := buildTar(map[string]string{"endpoints.yaml": svc}) + url, cleanup := serve(archive) + defer cleanup() + + got, err := fetchContent(context.TODO(), url, nil, "", "", libsveltosv1beta1.ClusterTypeCapi, logger) + Expect(err).ToNot(HaveOccurred()) + Expect(string(got)).To(ContainSubstring(svc)) + }) }) diff --git a/lib/crd/clusterprofiles.go b/lib/crd/clusterprofiles.go index 8d3aa977..9e585baa 100644 --- a/lib/crd/clusterprofiles.go +++ b/lib/crd/clusterprofiles.go @@ -1242,8 +1242,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: diff --git a/lib/crd/clusterpromotions.go b/lib/crd/clusterpromotions.go index 1dd87715..dff1a41a 100644 --- a/lib/crd/clusterpromotions.go +++ b/lib/crd/clusterpromotions.go @@ -1144,8 +1144,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: @@ -2590,8 +2595,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: @@ -3008,8 +3018,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: diff --git a/lib/crd/clustersummaries.go b/lib/crd/clustersummaries.go index 8575321b..e8ffb670 100644 --- a/lib/crd/clustersummaries.go +++ b/lib/crd/clustersummaries.go @@ -1281,8 +1281,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: diff --git a/lib/crd/profiles.go b/lib/crd/profiles.go index c349ca18..e9226bfc 100644 --- a/lib/crd/profiles.go +++ b/lib/crd/profiles.go @@ -1242,8 +1242,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: diff --git a/manifest/manifest.yaml b/manifest/manifest.yaml index 382e40a6..55eaa04d 100644 --- a/manifest/manifest.yaml +++ b/manifest/manifest.yaml @@ -1532,8 +1532,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: @@ -3970,8 +3975,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: @@ -5416,8 +5426,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: @@ -5834,8 +5849,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: @@ -7541,8 +7561,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: @@ -10055,8 +10080,13 @@ spec: Sveltos fetches the content on every reconciliation and redeploys if the content hash has changed. Supported schemes: - "http://" or "https://" — HTTP/HTTPS endpoint returning raw YAML/JSON - "oci://" — OCI registry artifact containing YAML manifests + "http://" or "https://" — HTTP/HTTPS endpoint returning a raw YAML/JSON + document, a gzip-compressed document, an + uncompressed tar, or a gzip-compressed tar of + .yaml/.yml/.json files + "oci://" — OCI registry artifact whose layers are accepted + in the same shapes: raw YAML/JSON, gzip-compressed + YAML/JSON, uncompressed tar, or gzip-compressed tar pattern: ^(https?|oci):// type: string required: