From debb90c6259fb91d7feb4fff34882c765732bb2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 00:11:05 -0700 Subject: [PATCH 01/29] ObjectStorage: context, create-only writes, ranged and streaming reads NewObjectStorage returns an ObjectStorage for S3 (and S3-compatible endpoints) and local storage. Every call takes a context; PutObject and UploadObject take PutOptions (ContentType, IfAbsent); GetObjectRange, DownloadObject and ObjectExists read; DeleteObjects reports per-key failures inside an accepted batch. Missing objects and refused writes come back as ErrObjectNotFound and ErrObjectExists, still wrapping the status code. Storage is unchanged. --- local_object.go | 129 +++++++++++++++++++++++++++++ object.go | 69 ++++++++++++++++ object_test.go | 131 ++++++++++++++++++++++++++++++ s3_object.go | 211 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 540 insertions(+) create mode 100644 local_object.go create mode 100644 object.go create mode 100644 object_test.go create mode 100644 s3_object.go diff --git a/local_object.go b/local_object.go new file mode 100644 index 0000000..a69256f --- /dev/null +++ b/local_object.go @@ -0,0 +1,129 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "time" +) + +type localObjects struct { + u *localUploader +} + +func NewLocalObjects(conf *LocalConfig) (ObjectStorage, error) { + u, err := NewLocal(conf) + if err != nil { + return nil, err + } + return &localObjects{u: u.(*localUploader)}, nil +} + +func (o *localObjects) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + return o.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) +} + +// UploadObject writes a temporary file and links or renames it into place, so +// readers never see a partial object and IfAbsent is atomic. +func (o *localObjects) UploadObject(_ context.Context, storagePath string, body io.Reader, opts PutOptions) error { + dst := o.u.location(storagePath) + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(dst), ".upload-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err = io.Copy(tmp, body); err != nil { + tmp.Close() + return err + } + if err = tmp.Close(); err != nil { + return err + } + if !opts.IfAbsent { + return os.Rename(tmp.Name(), dst) + } + if err = os.Link(tmp.Name(), dst); errors.Is(err, fs.ErrExist) { + return ErrObjectExists + } + return err +} + +func (o *localObjects) GetObject(_ context.Context, storagePath string) ([]byte, error) { + data, err := os.ReadFile(o.u.location(storagePath)) + return data, localObjectError(err) +} + +func (o *localObjects) GetObjectRange(_ context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + f, err := os.Open(o.u.location(storagePath)) + if err != nil { + return nil, localObjectError(err) + } + defer f.Close() + buf := make([]byte, length) + n, err := f.ReadAt(buf, offset) + if errors.Is(err, io.EOF) { + err = nil + } + return buf[:n], err +} + +func (o *localObjects) DownloadObject(_ context.Context, storagePath string, w io.Writer) (int64, error) { + f, err := os.Open(o.u.location(storagePath)) + if err != nil { + return 0, localObjectError(err) + } + defer f.Close() + return io.Copy(w, f) +} + +func (o *localObjects) ObjectExists(_ context.Context, storagePath string) (bool, error) { + _, err := os.Stat(o.u.location(storagePath)) + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return err == nil, err +} + +func (o *localObjects) ListObjects(_ context.Context, prefix string) ([]string, error) { + return o.u.ListObjects(prefix) +} + +func (o *localObjects) DeleteObjects(_ context.Context, storagePaths []string) error { + return o.u.DeleteObjects(storagePaths) +} + +func (o *localObjects) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { + return o.u.GeneratePresignedUrl(storagePath, expiration) +} + +func localObjectError(err error) error { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("%w: %w", ErrObjectNotFound, err) + } + return err +} diff --git a/object.go b/object.go new file mode 100644 index 0000000..7874a30 --- /dev/null +++ b/object.go @@ -0,0 +1,69 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "errors" + "fmt" + "io" + "time" +) + +// ObjectStorage is Storage for services that keep their own objects: every +// call takes a context, writes can refuse to replace an object, and reads can +// take a range or stream to a writer. +type ObjectStorage interface { + PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error + // UploadObject streams body in parts, holding about one part in memory. + UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error + + GetObject(ctx context.Context, storagePath string) ([]byte, error) + // GetObjectRange reads length bytes from offset; past the end it returns + // what the object holds. + GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) + DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) + ObjectExists(ctx context.Context, storagePath string) (bool, error) + + ListObjects(ctx context.Context, prefix string) ([]string, error) + DeleteObjects(ctx context.Context, storagePaths []string) error + + GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) +} + +type PutOptions struct { + ContentType string + // IfAbsent fails with ErrObjectExists when the path holds an object. + // PutObject checks it atomically. UploadObject checks it before uploading, + // so two concurrent uploads to one path can both succeed. + IfAbsent bool +} + +var ( + ErrObjectNotFound = errors.New("storage: object not found") + ErrObjectExists = errors.New("storage: object exists") +) + +// NewObjectStorage supports S3 (and S3-compatible endpoints) and local storage. +func NewObjectStorage(conf Config) (ObjectStorage, error) { + switch c := conf.(type) { + case *S3Config: + return NewS3Objects(c) + case *LocalConfig: + return NewLocalObjects(c) + default: + return nil, fmt.Errorf("object storage is not supported for %T", conf) + } +} diff --git a/object_test.go b/object_test.go new file mode 100644 index 0000000..5c16724 --- /dev/null +++ b/object_test.go @@ -0,0 +1,131 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage_test + +import ( + "bytes" + "context" + "fmt" + "os" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/livekit/storage" +) + +func TestLocalObjects(t *testing.T) { + s, err := storage.NewObjectStorage(&storage.LocalConfig{StorageDir: t.TempDir()}) + require.NoError(t, err) + testObjectStorage(t, s) +} + +// TestS3Objects reaches AWS, or an S3-compatible store at S3_ENDPOINT. +func TestS3Objects(t *testing.T) { + key := os.Getenv("AWS_ACCESS_KEY") + secret := os.Getenv("AWS_SECRET") + bucket := os.Getenv("S3_BUCKET") + if key == "" || secret == "" || bucket == "" { + t.Skip("Missing env vars") + } + endpoint := os.Getenv("S3_ENDPOINT") + s, err := storage.NewObjectStorage(&storage.S3Config{ + AccessKey: key, + Secret: secret, + SessionToken: os.Getenv("AWS_SESSION_TOKEN"), + Region: os.Getenv("AWS_REGION"), + Endpoint: endpoint, + ForcePathStyle: endpoint != "", + Bucket: bucket, + }) + require.NoError(t, err) + testObjectStorage(t, s) +} + +func testObjectStorage(t *testing.T, s storage.ObjectStorage) { + ctx := context.Background() + prefix := fmt.Sprintf("objects-test-%d/", time.Now().UnixNano()) + small := prefix + "small" + large := prefix + "large" + missing := prefix + "missing" + payload := []byte("hello from PutObject") + // Larger than one upload part, so UploadObject takes the multipart path. + big := bytes.Repeat([]byte("0123456789abcdef"), (17<<20)/16) + + t.Cleanup(func() { + _ = s.DeleteObjects(ctx, []string{small, large}) + }) + + t.Run("PutObject IfAbsent", func(t *testing.T) { + require.NoError(t, s.PutObject(ctx, small, payload, storage.PutOptions{IfAbsent: true})) + err := s.PutObject(ctx, small, []byte("other"), storage.PutOptions{IfAbsent: true}) + require.ErrorIs(t, err, storage.ErrObjectExists) + got, err := s.GetObject(ctx, small) + require.NoError(t, err) + require.Equal(t, payload, got) + }) + + t.Run("UploadObject", func(t *testing.T) { + require.NoError(t, s.UploadObject(ctx, large, bytes.NewReader(big), storage.PutOptions{IfAbsent: true})) + err := s.UploadObject(ctx, large, bytes.NewReader(big), storage.PutOptions{IfAbsent: true}) + require.ErrorIs(t, err, storage.ErrObjectExists) + var buf bytes.Buffer + n, err := s.DownloadObject(ctx, large, &buf) + require.NoError(t, err) + require.Equal(t, int64(len(big)), n) + require.Equal(t, big, buf.Bytes()) + }) + + t.Run("GetObjectRange", func(t *testing.T) { + got, err := s.GetObjectRange(ctx, small, 6, 4) + require.NoError(t, err) + require.Equal(t, payload[6:10], got) + got, err = s.GetObjectRange(ctx, small, 6, 1000) + require.NoError(t, err) + require.Equal(t, payload[6:], got) + }) + + t.Run("not found", func(t *testing.T) { + _, err := s.GetObject(ctx, missing) + require.ErrorIs(t, err, storage.ErrObjectNotFound) + _, err = s.GetObjectRange(ctx, missing, 0, 1) + require.ErrorIs(t, err, storage.ErrObjectNotFound) + exists, err := s.ObjectExists(ctx, missing) + require.NoError(t, err) + require.False(t, exists) + exists, err = s.ObjectExists(ctx, small) + require.NoError(t, err) + require.True(t, exists) + }) + + t.Run("ListObjects and DeleteObjects", func(t *testing.T) { + keys, err := s.ListObjects(ctx, prefix) + require.NoError(t, err) + sort.Strings(keys) + require.Equal(t, []string{large, small}, keys) + require.NoError(t, s.DeleteObjects(ctx, []string{small, large})) + keys, err = s.ListObjects(ctx, prefix) + require.NoError(t, err) + require.Empty(t, keys) + }) + + t.Run("GeneratePresignedUrl", func(t *testing.T) { + url, err := s.GeneratePresignedUrl(ctx, small, time.Minute) + require.NoError(t, err) + require.NotEmpty(t, url) + }) +} diff --git a/s3_object.go b/s3_object.go new file mode 100644 index 0000000..7d0e991 --- /dev/null +++ b/s3_object.go @@ -0,0 +1,211 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" +) + +const s3UploadPartSize = 16 << 20 + +type s3Objects struct { + s *s3Storage +} + +func NewS3Objects(conf *S3Config) (ObjectStorage, error) { + s, err := NewS3(conf) + if err != nil { + return nil, err + } + return &s3Objects{s: s.(*s3Storage)}, nil +} + +func (o *s3Objects) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + input := o.putInput(storagePath, bytes.NewReader(data), opts) + if opts.IfAbsent { + input.IfNoneMatch = aws.String("*") + } + _, err := o.s.getClient(nil).PutObject(ctx, input) + return wrapS3ObjectError(err) +} + +func (o *s3Objects) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + if opts.IfAbsent { + exists, err := o.ObjectExists(ctx, storagePath) + if err != nil { + return err + } + if exists { + return ErrObjectExists + } + } + uploader := manager.NewUploader(o.s.getClient(nil), func(u *manager.Uploader) { + u.PartSize = s3UploadPartSize + }) + _, err := uploader.Upload(ctx, o.putInput(storagePath, body, opts)) + return wrapS3ObjectError(err) +} + +func (o *s3Objects) putInput(storagePath string, body io.Reader, opts PutOptions) *s3.PutObjectInput { + input := &s3.PutObjectInput{ + Bucket: aws.String(o.s.conf.Bucket), + Key: aws.String(storagePath), + Body: body, + } + if opts.ContentType != "" { + input.ContentType = aws.String(opts.ContentType) + } + return input +} + +func (o *s3Objects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { + var buf bytes.Buffer + if _, err := o.DownloadObject(ctx, storagePath, &buf); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (o *s3Objects) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + out, err := o.s.getClient(nil).GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(o.s.conf.Bucket), + Key: aws.String(storagePath), + Range: aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)), + }) + if err != nil { + return nil, wrapS3ObjectError(err) + } + defer out.Body.Close() + return io.ReadAll(io.LimitReader(out.Body, length)) +} + +func (o *s3Objects) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { + out, err := o.s.getClient(nil).GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(o.s.conf.Bucket), + Key: aws.String(storagePath), + }) + if err != nil { + return 0, wrapS3ObjectError(err) + } + defer out.Body.Close() + return io.Copy(w, out.Body) +} + +func (o *s3Objects) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + _, err := o.s.getClient(nil).HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(o.s.conf.Bucket), + Key: aws.String(storagePath), + }) + err = wrapS3ObjectError(err) + if errors.Is(err, ErrObjectNotFound) { + return false, nil + } + return err == nil, err +} + +func (o *s3Objects) ListObjects(ctx context.Context, prefix string) ([]string, error) { + var keys []string + paginator := s3.NewListObjectsV2Paginator(o.s.getClient(nil), &s3.ListObjectsV2Input{ + Bucket: aws.String(o.s.conf.Bucket), + Prefix: aws.String(prefix), + }) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return nil, wrapS3Error(err) + } + for _, obj := range page.Contents { + keys = append(keys, aws.ToString(obj.Key)) + } + } + return keys, nil +} + +// DeleteObjects fails when any key fails, even within a batch S3 accepted. +func (o *s3Objects) DeleteObjects(ctx context.Context, storagePaths []string) error { + client := o.s.getClient(nil) + for i := 0; i < len(storagePaths); i += 1000 { + batch := storagePaths[i:min(i+1000, len(storagePaths))] + objects := make([]types.ObjectIdentifier, 0, len(batch)) + for _, p := range batch { + objects = append(objects, types.ObjectIdentifier{Key: aws.String(p)}) + } + input := &s3.DeleteObjectsInput{ + Bucket: aws.String(o.s.conf.Bucket), + Delete: &types.Delete{Objects: objects, Quiet: aws.Bool(true)}, + } + if o.s.conf.Endpoint != "" { + input.ChecksumAlgorithm = types.ChecksumAlgorithmSha256 + } + out, err := client.DeleteObjects(ctx, input) + if err != nil { + return wrapS3Error(err) + } + if len(out.Errors) > 0 { + e := out.Errors[0] + return fmt.Errorf("storage: %d of %d deletes failed, first %s: %s %s", + len(out.Errors), len(batch), aws.ToString(e.Key), aws.ToString(e.Code), aws.ToString(e.Message)) + } + } + return nil +} + +func (o *s3Objects) GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { + res, err := s3.NewPresignClient(o.s.getClient(nil)).PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(o.s.conf.Bucket), + Key: aws.String(storagePath), + }, s3.WithPresignExpires(expiration)) + if err != nil { + return "", wrapS3Error(err) + } + return res.URL, nil +} + +// wrapS3ObjectError is wrapS3Error with ErrObjectNotFound and ErrObjectExists +// for the codes S3-compatible stores answer them with. +func wrapS3ObjectError(err error) error { + if err == nil { + return nil + } + var noSuchKey *types.NoSuchKey + var notFound *types.NotFound + if errors.As(err, &noSuchKey) || errors.As(err, ¬Found) { + return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapS3Error(err)) + } + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch apiErr.ErrorCode() { + case "NoSuchKey", "NotFound", "404": + return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapS3Error(err)) + case "PreconditionFailed": + return fmt.Errorf("%w: %w", ErrObjectExists, wrapS3Error(err)) + } + } + return wrapS3Error(err) +} From 7e75fb0ea2c59fa30ede2ed6bda41d414d8e2011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 00:12:11 -0700 Subject: [PATCH 02/29] objects: the ObjectStorage interface without provider SDKs, as config does for configs --- object.go | 44 ++++++++-------------------------- objects/objects.go | 59 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 34 deletions(-) create mode 100644 objects/objects.go diff --git a/object.go b/object.go index 7874a30..52261b8 100644 --- a/object.go +++ b/object.go @@ -15,45 +15,21 @@ package storage import ( - "context" - "errors" "fmt" - "io" - "time" -) - -// ObjectStorage is Storage for services that keep their own objects: every -// call takes a context, writes can refuse to replace an object, and reads can -// take a range or stream to a writer. -type ObjectStorage interface { - PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error - // UploadObject streams body in parts, holding about one part in memory. - UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error - - GetObject(ctx context.Context, storagePath string) ([]byte, error) - // GetObjectRange reads length bytes from offset; past the end it returns - // what the object holds. - GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) - DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) - ObjectExists(ctx context.Context, storagePath string) (bool, error) - - ListObjects(ctx context.Context, prefix string) ([]string, error) - DeleteObjects(ctx context.Context, storagePaths []string) error - GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) -} + "github.com/livekit/storage/objects" +) -type PutOptions struct { - ContentType string - // IfAbsent fails with ErrObjectExists when the path holds an object. - // PutObject checks it atomically. UploadObject checks it before uploading, - // so two concurrent uploads to one path can both succeed. - IfAbsent bool -} +// The interface lives in storage/objects so that code implementing or +// consuming it without creating one doesn't pull in every provider SDK. +type ( + ObjectStorage = objects.Storage + PutOptions = objects.PutOptions +) var ( - ErrObjectNotFound = errors.New("storage: object not found") - ErrObjectExists = errors.New("storage: object exists") + ErrObjectNotFound = objects.ErrNotFound + ErrObjectExists = objects.ErrExists ) // NewObjectStorage supports S3 (and S3-compatible endpoints) and local storage. diff --git a/objects/objects.go b/objects/objects.go new file mode 100644 index 0000000..11dc22e --- /dev/null +++ b/objects/objects.go @@ -0,0 +1,59 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package objects is the ObjectStorage interface without any provider SDK, for +// code that implements or consumes it without creating one, the way config +// holds the config structs. +package objects + +import ( + "context" + "errors" + "io" + "time" +) + +// Storage is storage for services that keep their own objects: every call +// takes a context, writes can refuse to replace an object, and reads can take +// a range or stream to a writer. +type Storage interface { + PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error + // UploadObject streams body in parts, holding about one part in memory. + UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error + + GetObject(ctx context.Context, storagePath string) ([]byte, error) + // GetObjectRange reads length bytes from offset; past the end it returns + // what the object holds. + GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) + DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) + ObjectExists(ctx context.Context, storagePath string) (bool, error) + + ListObjects(ctx context.Context, prefix string) ([]string, error) + DeleteObjects(ctx context.Context, storagePaths []string) error + + GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) +} + +type PutOptions struct { + ContentType string + // IfAbsent fails with ErrExists when the path holds an object. PutObject + // checks it atomically. UploadObject checks it before uploading, so two + // concurrent uploads to one path can both succeed. + IfAbsent bool +} + +var ( + ErrNotFound = errors.New("storage: object not found") + ErrExists = errors.New("storage: object exists") +) From cae07d21274181bb0c2758561130e8b1086ab2b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 00:24:58 -0700 Subject: [PATCH 03/29] ObjectStorage: GCS, Azure, OCI and AliOSS backends NewObjectStorage now covers every Config. DeleteObjects succeeds for paths that hold nothing, and a range starting at or past the end returns no bytes, on every backend. The S3 uploader's checksum workaround for S3-compatible stores is shared with UploadObject. --- alioss_object.go | 173 ++++++++++++++++++++++++++++ azure_object.go | 173 ++++++++++++++++++++++++++++ gcp_object.go | 170 ++++++++++++++++++++++++++++ local_object.go | 7 +- object.go | 13 ++- object_test.go | 81 +++++++++++++- objects/objects.go | 8 +- oci_object.go | 272 +++++++++++++++++++++++++++++++++++++++++++++ s3.go | 20 ++-- s3_object.go | 6 +- 10 files changed, 905 insertions(+), 18 deletions(-) create mode 100644 alioss_object.go create mode 100644 azure_object.go create mode 100644 gcp_object.go create mode 100644 oci_object.go diff --git a/alioss_object.go b/alioss_object.go new file mode 100644 index 0000000..4770efe --- /dev/null +++ b/alioss_object.go @@ -0,0 +1,173 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "time" + + "github.com/aliyun/aliyun-oss-go-sdk/oss" +) + +type aliOSSObjects struct { + s *aliOSSStorage +} + +func NewAliOSSObjects(conf *AliOSSConfig) (ObjectStorage, error) { + s, err := NewAliOSS(conf) + if err != nil { + return nil, err + } + return &aliOSSObjects{s: s.(*aliOSSStorage)}, nil +} + +func (o *aliOSSObjects) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + return o.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) +} + +// UploadObject streams body in one PutObject, chunked when its length is +// unknown, so it holds no part in memory and caps an object at 5 GB. IfAbsent +// sets x-oss-forbid-overwrite, atomic here, but OSS ignores it on a bucket with +// versioning enabled or suspended. +func (o *aliOSSObjects) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + options := []oss.Option{oss.WithContext(ctx)} + if opts.ContentType != "" { + options = append(options, oss.ContentType(opts.ContentType)) + } + if opts.IfAbsent { + options = append(options, oss.ForbidOverWrite(true)) + } + return wrapAliOSSObjectError(o.s.bucket.PutObject(storagePath, body, options...)) +} + +func (o *aliOSSObjects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { + var buf bytes.Buffer + if _, err := o.DownloadObject(ctx, storagePath, &buf); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// GetObjectRange asks for standard range behavior: by default OSS answers an +// unsatisfiable range with the whole object. +func (o *aliOSSObjects) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + var buf bytes.Buffer + _, err := o.download(ctx, storagePath, &buf, + oss.Range(offset, offset+length-1), oss.RangeBehavior("standard")) + var svcErr oss.ServiceError + if errors.As(err, &svcErr) && svcErr.StatusCode == http.StatusRequestedRangeNotSatisfiable { + return []byte{}, nil + } + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (o *aliOSSObjects) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { + return o.download(ctx, storagePath, w) +} + +func (o *aliOSSObjects) download(ctx context.Context, storagePath string, w io.Writer, options ...oss.Option) (int64, error) { + r, err := o.s.bucket.GetObject(storagePath, append(options, oss.WithContext(ctx))...) + if err != nil { + return 0, wrapAliOSSObjectError(err) + } + defer r.Close() + n, err := io.Copy(w, r) + return n, wrapAliOSSObjectError(err) +} + +func (o *aliOSSObjects) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + exists, err := o.s.bucket.IsObjectExist(storagePath, oss.WithContext(ctx)) + return exists, wrapAliOSSObjectError(err) +} + +func (o *aliOSSObjects) ListObjects(ctx context.Context, prefix string) ([]string, error) { + var keys []string + options := []oss.Option{oss.Prefix(prefix), oss.MaxKeys(1000), oss.WithContext(ctx)} + for { + res, err := o.s.bucket.ListObjectsV2(options...) + if err != nil { + return nil, wrapAliOSSObjectError(err) + } + for _, obj := range res.Objects { + keys = append(keys, obj.Key) + } + if !res.IsTruncated || res.NextContinuationToken == "" { + return keys, nil + } + options = []oss.Option{oss.Prefix(prefix), oss.MaxKeys(1000), oss.ContinuationToken(res.NextContinuationToken), oss.WithContext(ctx)} + } +} + +// DeleteObjects sends batches of 1000, the most one request takes. OSS reports +// no per-key errors, only the keys it deleted, so a key it leaves out fails the +// call unless it holds nothing. +func (o *aliOSSObjects) DeleteObjects(ctx context.Context, storagePaths []string) error { + for i := 0; i < len(storagePaths); i += 1000 { + batch := storagePaths[i:min(i+1000, len(storagePaths))] + res, err := o.s.bucket.DeleteObjects(batch, oss.WithContext(ctx)) + if err != nil { + return wrapAliOSSObjectError(err) + } + deleted := make(map[string]bool, len(res.DeletedObjects)) + for _, k := range res.DeletedObjects { + deleted[k] = true + } + for _, k := range batch { + if deleted[k] { + continue + } + exists, err := o.ObjectExists(ctx, k) + if err != nil { + return err + } + if exists { + return fmt.Errorf("storage: delete of %s not confirmed", k) + } + } + } + return nil +} + +func (o *aliOSSObjects) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { + return o.s.GeneratePresignedUrl(storagePath, expiration) +} + +func wrapAliOSSObjectError(err error) error { + if err == nil { + return nil + } + var svcErr oss.ServiceError + if errors.As(err, &svcErr) { + switch { + // A HEAD answers a missing key with a bare 404. + case svcErr.Code == "NoSuchKey" || (svcErr.StatusCode == http.StatusNotFound && svcErr.Code == ""): + return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapAliOSSError(err)) + case svcErr.Code == "FileAlreadyExists": + return fmt.Errorf("%w: %w", ErrObjectExists, wrapAliOSSError(err)) + } + } + return wrapAliOSSError(err) +} diff --git a/azure_object.go b/azure_object.go new file mode 100644 index 0000000..e3c8c99 --- /dev/null +++ b/azure_object.go @@ -0,0 +1,173 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "bytes" + "context" + "fmt" + "io" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" +) + +type azureObjects struct { + s *azureBLOBStorage +} + +func NewAzureObjects(conf *AzureConfig) (ObjectStorage, error) { + s, err := NewAzure(conf) + if err != nil { + return nil, err + } + return &azureObjects{s: s.(*azureBLOBStorage)}, nil +} + +func (o *azureObjects) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + return o.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) +} + +// UploadObject stages blocks and commits them; IfAbsent is a condition on the +// commit, so it is atomic here. The check before staging only spares a refused +// upload its traffic. +func (o *azureObjects) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + if opts.IfAbsent { + exists, err := o.ObjectExists(ctx, storagePath) + if err != nil { + return err + } + if exists { + return ErrObjectExists + } + } + uploadOpts := &azblob.UploadStreamOptions{BlockSize: azureBlockSize, Concurrency: azureParallelism} + if opts.ContentType != "" { + uploadOpts.HTTPHeaders = &blob.HTTPHeaders{BlobContentType: to.Ptr(opts.ContentType)} + } + if opts.IfAbsent { + uploadOpts.AccessConditions = &blob.AccessConditions{ + ModifiedAccessConditions: &blob.ModifiedAccessConditions{IfNoneMatch: to.Ptr(azcore.ETagAny)}, + } + } + _, err := o.s.client.UploadStream(ctx, o.s.conf.ContainerName, storagePath, body, uploadOpts) + // A racing upload's commit discards this one's staged blocks. + if opts.IfAbsent && bloberror.HasCode(err, bloberror.InvalidBlockList) { + return fmt.Errorf("%w: %w", ErrObjectExists, wrapAzureError(err)) + } + return wrapAzureObjectError(err) +} + +func (o *azureObjects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { + var buf bytes.Buffer + if _, err := o.DownloadObject(ctx, storagePath, &buf); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (o *azureObjects) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + var buf bytes.Buffer + _, err := o.download(ctx, storagePath, &buf, blob.HTTPRange{Offset: offset, Count: length}) + if bloberror.HasCode(err, bloberror.InvalidRange) { + return []byte{}, nil + } + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (o *azureObjects) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { + return o.download(ctx, storagePath, w, blob.HTTPRange{}) +} + +func (o *azureObjects) download(ctx context.Context, storagePath string, w io.Writer, r blob.HTTPRange) (int64, error) { + resp, err := o.s.client.DownloadStream(ctx, o.s.conf.ContainerName, storagePath, &azblob.DownloadStreamOptions{Range: r}) + if err != nil { + return 0, wrapAzureObjectError(err) + } + body := resp.NewRetryReader(ctx, &blob.RetryReaderOptions{MaxRetries: 3}) + defer body.Close() + n, err := io.Copy(w, body) + return n, wrapAzureObjectError(err) +} + +func (o *azureObjects) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + blobClient := o.s.client.ServiceClient().NewContainerClient(o.s.conf.ContainerName).NewBlobClient(storagePath) + _, err := blobClient.GetProperties(ctx, nil) + if bloberror.HasCode(err, bloberror.BlobNotFound) { + return false, nil + } + return err == nil, wrapAzureObjectError(err) +} + +func (o *azureObjects) ListObjects(ctx context.Context, prefix string) ([]string, error) { + pager := o.s.client.NewListBlobsFlatPager(o.s.conf.ContainerName, &azblob.ListBlobsFlatOptions{Prefix: &prefix}) + var keys []string + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, wrapAzureObjectError(err) + } + for _, item := range page.Segment.BlobItems { + keys = append(keys, *item.Name) + } + } + return keys, nil +} + +func (o *azureObjects) DeleteObjects(ctx context.Context, storagePaths []string) error { + for _, p := range storagePaths { + _, err := o.s.client.DeleteBlob(ctx, o.s.conf.ContainerName, p, &azblob.DeleteBlobOptions{ + DeleteSnapshots: to.Ptr(azblob.DeleteSnapshotsOptionTypeInclude), + }) + if err != nil && !bloberror.HasCode(err, bloberror.BlobNotFound) { + return wrapAzureObjectError(err) + } + } + return nil +} + +// GeneratePresignedUrl signs with the account key unless a token credential is +// set, in which case it signs with a user delegation key as Storage does. +func (o *azureObjects) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { + if o.s.conf.TokenCredential != nil { + return o.s.GeneratePresignedUrl(storagePath, expiration) + } + b := o.s.client.ServiceClient().NewContainerClient(o.s.conf.ContainerName).NewBlobClient(storagePath) + u, err := b.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(expiration), nil) + return u, wrapAzureError(err) +} + +func wrapAzureObjectError(err error) error { + switch { + case err == nil: + return nil + case bloberror.HasCode(err, bloberror.BlobNotFound): + return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapAzureError(err)) + case bloberror.HasCode(err, bloberror.BlobAlreadyExists, bloberror.ConditionNotMet): + return fmt.Errorf("%w: %w", ErrObjectExists, wrapAzureError(err)) + } + return wrapAzureError(err) +} diff --git a/gcp_object.go b/gcp_object.go new file mode 100644 index 0000000..182ac8f --- /dev/null +++ b/gcp_object.go @@ -0,0 +1,170 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "time" + + "cloud.google.com/go/storage" + "google.golang.org/api/googleapi" + "google.golang.org/api/iterator" +) + +type gcpObjects struct { + s *gcpStorage +} + +func NewGCPObjects(conf *GCPConfig) (ObjectStorage, error) { + s, err := NewGCP(conf) + if err != nil { + return nil, err + } + return &gcpObjects{s: s.(*gcpStorage)}, nil +} + +func (o *gcpObjects) object(storagePath string) *storage.ObjectHandle { + return o.s.client.Bucket(o.s.conf.Bucket).Object(storagePath) +} + +// PutObject sizes the upload buffer to data, so a small put sends one request +// without a full chunk's allocation. +func (o *gcpObjects) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + return o.write(ctx, storagePath, bytes.NewReader(data), opts, min(len(data)+1, googleapi.DefaultUploadChunkSize)) +} + +// UploadObject sends body in resumable chunks; IfAbsent is a precondition on +// the whole upload, so it is atomic here. +func (o *gcpObjects) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + return o.write(ctx, storagePath, body, opts, googleapi.DefaultUploadChunkSize) +} + +func (o *gcpObjects) write(ctx context.Context, storagePath string, body io.Reader, opts PutOptions, chunkSize int) error { + obj := o.object(storagePath) + if opts.IfAbsent { + // Conditional, so the SDK retries it by default. + obj = obj.If(storage.Conditions{DoesNotExist: true}) + } else { + obj = obj.Retryer(storage.WithPolicy(storage.RetryAlways)) + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + w := obj.NewWriter(ctx) + w.ContentType = opts.ContentType + w.ChunkSize = chunkSize + if _, err := io.Copy(w, body); err != nil { + // Cancelling aborts the upload; Close would store what was sent. + cancel() + return o.writeError(err, opts.IfAbsent) + } + return o.writeError(w.Close(), opts.IfAbsent) +} + +// writeError maps a failed precondition to ErrObjectExists only where IfAbsent +// set one: GCS answers other refusals with 412 too. +func (o *gcpObjects) writeError(err error, ifAbsent bool) error { + var apiErr *googleapi.Error + if ifAbsent && errors.As(err, &apiErr) && apiErr.Code == http.StatusPreconditionFailed { + return fmt.Errorf("%w: %w", ErrObjectExists, wrapGCPError(err)) + } + return wrapGCPObjectError(err) +} + +func (o *gcpObjects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { + var buf bytes.Buffer + if _, err := o.DownloadObject(ctx, storagePath, &buf); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (o *gcpObjects) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + r, err := o.object(storagePath).NewRangeReader(ctx, offset, length) + var apiErr *googleapi.Error + if errors.As(err, &apiErr) && apiErr.Code == http.StatusRequestedRangeNotSatisfiable { + return []byte{}, nil + } + if err != nil { + return nil, wrapGCPObjectError(err) + } + defer r.Close() + data, err := io.ReadAll(r) + return data, wrapGCPObjectError(err) +} + +func (o *gcpObjects) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { + r, err := o.object(storagePath).NewReader(ctx) + if err != nil { + return 0, wrapGCPObjectError(err) + } + defer r.Close() + n, err := io.Copy(w, r) + return n, wrapGCPObjectError(err) +} + +func (o *gcpObjects) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + _, err := o.object(storagePath).Attrs(ctx) + if errors.Is(err, storage.ErrObjectNotExist) { + return false, nil + } + return err == nil, wrapGCPObjectError(err) +} + +func (o *gcpObjects) ListObjects(ctx context.Context, prefix string) ([]string, error) { + it := o.s.client.Bucket(o.s.conf.Bucket).Objects(ctx, &storage.Query{Prefix: prefix}) + var keys []string + for { + attrs, err := it.Next() + if errors.Is(err, iterator.Done) { + return keys, nil + } + if err != nil { + return nil, wrapGCPObjectError(err) + } + keys = append(keys, attrs.Name) + } +} + +func (o *gcpObjects) DeleteObjects(ctx context.Context, storagePaths []string) error { + for _, p := range storagePaths { + if err := o.object(p).Delete(ctx); err != nil && !errors.Is(err, storage.ErrObjectNotExist) { + return wrapGCPObjectError(err) + } + } + return nil +} + +func (o *gcpObjects) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { + return o.s.GeneratePresignedUrl(storagePath, expiration) +} + +func wrapGCPObjectError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, storage.ErrObjectNotExist) { + return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapGCPError(err)) + } + return wrapGCPError(err) +} diff --git a/local_object.go b/local_object.go index a69256f..f9c8cff 100644 --- a/local_object.go +++ b/local_object.go @@ -114,7 +114,12 @@ func (o *localObjects) ListObjects(_ context.Context, prefix string) ([]string, } func (o *localObjects) DeleteObjects(_ context.Context, storagePaths []string) error { - return o.u.DeleteObjects(storagePaths) + for _, p := range storagePaths { + if err := o.u.DeleteObject(p); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + } + return nil } func (o *localObjects) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { diff --git a/object.go b/object.go index 52261b8..dd76b9e 100644 --- a/object.go +++ b/object.go @@ -32,13 +32,20 @@ var ( ErrObjectExists = objects.ErrExists ) -// NewObjectStorage supports S3 (and S3-compatible endpoints) and local storage. func NewObjectStorage(conf Config) (ObjectStorage, error) { switch c := conf.(type) { - case *S3Config: - return NewS3Objects(c) + case *AliOSSConfig: + return NewAliOSSObjects(c) + case *AzureConfig: + return NewAzureObjects(c) + case *GCPConfig: + return NewGCPObjects(c) case *LocalConfig: return NewLocalObjects(c) + case *OCIConfig: + return NewOCIObjects(c) + case *S3Config: + return NewS3Objects(c) default: return nil, fmt.Errorf("object storage is not supported for %T", conf) } diff --git a/object_test.go b/object_test.go index 5c16724..d70b3f9 100644 --- a/object_test.go +++ b/object_test.go @@ -97,6 +97,9 @@ func testObjectStorage(t *testing.T, s storage.ObjectStorage) { got, err = s.GetObjectRange(ctx, small, 6, 1000) require.NoError(t, err) require.Equal(t, payload[6:], got) + got, err = s.GetObjectRange(ctx, small, int64(len(payload)), 4) + require.NoError(t, err) + require.Empty(t, got) }) t.Run("not found", func(t *testing.T) { @@ -117,7 +120,7 @@ func testObjectStorage(t *testing.T, s storage.ObjectStorage) { require.NoError(t, err) sort.Strings(keys) require.Equal(t, []string{large, small}, keys) - require.NoError(t, s.DeleteObjects(ctx, []string{small, large})) + require.NoError(t, s.DeleteObjects(ctx, []string{small, large, missing})) keys, err = s.ListObjects(ctx, prefix) require.NoError(t, err) require.Empty(t, keys) @@ -129,3 +132,79 @@ func testObjectStorage(t *testing.T, s storage.ObjectStorage) { require.NotEmpty(t, url) }) } + +// TestOCIObjects reaches OCI through its S3 compatibility API, as TestOCI does. +func TestOCIObjects(t *testing.T) { + key := os.Getenv("OCI_ACCESS_KEY") + secret := os.Getenv("OCI_SECRET") + region := os.Getenv("OCI_REGION") + endpoint := os.Getenv("OCI_ENDPOINT") + bucket := os.Getenv("OCI_BUCKET") + if key == "" || secret == "" || region == "" || endpoint == "" || bucket == "" { + t.Skip("Missing env vars") + } + s, err := storage.NewObjectStorage(&storage.S3Config{ + AccessKey: key, + Secret: secret, + Region: region, + Endpoint: endpoint, + Bucket: bucket, + ForcePathStyle: true, + }) + require.NoError(t, err) + testObjectStorage(t, s) +} + +// TestOCINativeObjects uses the OCI SDK with an API key. +func TestOCINativeObjects(t *testing.T) { + conf := &storage.OCIConfig{ + TenancyOCID: os.Getenv("OCI_TENANCY_OCID"), + UserOCID: os.Getenv("OCI_USER_OCID"), + Fingerprint: os.Getenv("OCI_FINGERPRINT"), + PrivateKey: os.Getenv("OCI_PRIVATE_KEY"), + Region: os.Getenv("OCI_REGION"), + Bucket: os.Getenv("OCI_BUCKET"), + } + if conf.TenancyOCID == "" || conf.UserOCID == "" || conf.Fingerprint == "" || conf.PrivateKey == "" || conf.Region == "" || conf.Bucket == "" { + t.Skip("Missing env vars") + } + s, err := storage.NewObjectStorage(conf) + require.NoError(t, err) + testObjectStorage(t, s) +} + +func TestGCPObjects(t *testing.T) { + creds := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + bucket := os.Getenv("GCP_BUCKET") + if creds == "" || bucket == "" { + t.Skip("Missing env vars") + } + s, err := storage.NewObjectStorage(&storage.GCPConfig{CredentialsJSON: creds, Bucket: bucket}) + require.NoError(t, err) + testObjectStorage(t, s) +} + +func TestAzureObjects(t *testing.T) { + name := os.Getenv("AZURE_ACCOUNT_NAME") + key := os.Getenv("AZURE_ACCOUNT_KEY") + container := os.Getenv("AZURE_CONTAINER_NAME") + if name == "" || key == "" || container == "" { + t.Skip("Missing env vars") + } + s, err := storage.NewObjectStorage(&storage.AzureConfig{AccountName: name, AccountKey: key, ContainerName: container}) + require.NoError(t, err) + testObjectStorage(t, s) +} + +func TestAliOSSObjects(t *testing.T) { + key := os.Getenv("ALI_ACCESS_KEY") + secret := os.Getenv("ALI_SECRET") + endpoint := os.Getenv("ALI_ENDPOINT") + bucket := os.Getenv("ALI_BUCKET") + if key == "" || secret == "" || endpoint == "" || bucket == "" { + t.Skip("Missing env vars") + } + s, err := storage.NewObjectStorage(&storage.AliOSSConfig{AccessKey: key, Secret: secret, Endpoint: endpoint, Bucket: bucket}) + require.NoError(t, err) + testObjectStorage(t, s) +} diff --git a/objects/objects.go b/objects/objects.go index 11dc22e..e60dea7 100644 --- a/objects/objects.go +++ b/objects/objects.go @@ -29,17 +29,19 @@ import ( // a range or stream to a writer. type Storage interface { PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error - // UploadObject streams body in parts, holding about one part in memory. + // UploadObject streams body in parts: memory is bounded by the part size + // times the upload concurrency, whatever the object's size. UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error GetObject(ctx context.Context, storagePath string) ([]byte, error) - // GetObjectRange reads length bytes from offset; past the end it returns - // what the object holds. + // GetObjectRange reads length bytes from offset. A range running past the + // end returns what the object holds, one starting at or past it no bytes. GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) ObjectExists(ctx context.Context, storagePath string) (bool, error) ListObjects(ctx context.Context, prefix string) ([]string, error) + // DeleteObjects succeeds for paths that hold nothing. DeleteObjects(ctx context.Context, storagePaths []string) error GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) diff --git a/oci_object.go b/oci_object.go new file mode 100644 index 0000000..99170c0 --- /dev/null +++ b/oci_object.go @@ -0,0 +1,272 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "time" + + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/objectstorage" +) + +const ociUploadPartSize = 16 << 20 + +type ociObjects struct { + s *ociStorage + client objectstorage.ObjectStorageClient +} + +// NewOCIObjects gives its calls a client of their own without the SDK's 60s +// request timeout, which covers reading a body and would cut a long download: +// the caller's context bounds each call instead. +func NewOCIObjects(conf *OCIConfig) (ObjectStorage, error) { + s, err := NewOCI(conf) + if err != nil { + return nil, err + } + o := &ociObjects{s: s.(*ociStorage)} + o.client = o.s.client + if hc, ok := o.client.HTTPClient.(*http.Client); ok { + o.client.HTTPClient = &http.Client{Transport: hc.Transport, CheckRedirect: hc.CheckRedirect, Jar: hc.Jar} + } + return o, nil +} + +func (o *ociObjects) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + size := int64(len(data)) + req := objectstorage.PutObjectRequest{ + NamespaceName: &o.s.namespace, + BucketName: &o.s.conf.Bucket, + ObjectName: &storagePath, + // Seekable, so the SDK can re-read it when it retries. + PutObjectBody: common.NewOCIReadSeekCloser(io.NopCloser(bytes.NewReader(data))), + ContentLength: &size, + } + if opts.ContentType != "" { + req.ContentType = &opts.ContentType + } + if opts.IfAbsent { + req.IfNoneMatch = common.String("*") + } + _, err := o.client.PutObject(ctx, req) + return wrapOCIObjectError(err) +} + +// UploadObject sends a body that fits one part with PutObject, otherwise a +// multipart upload of sequential parts through one buffer. IfAbsent is checked +// at create and again at commit, so it is atomic here. +func (o *ociObjects) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + part := make([]byte, ociUploadPartSize) + n, err := io.ReadFull(body, part) + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return o.PutObject(ctx, storagePath, part[:n], opts) + } + if err != nil { + return err + } + + var ifNoneMatch, contentType *string + if opts.IfAbsent { + ifNoneMatch = common.String("*") + } + if opts.ContentType != "" { + contentType = &opts.ContentType + } + created, err := o.client.CreateMultipartUpload(ctx, objectstorage.CreateMultipartUploadRequest{ + NamespaceName: &o.s.namespace, + BucketName: &o.s.conf.Bucket, + IfNoneMatch: ifNoneMatch, + CreateMultipartUploadDetails: objectstorage.CreateMultipartUploadDetails{ + Object: &storagePath, + ContentType: contentType, + }, + }) + if err != nil { + return wrapOCIObjectError(err) + } + + if err = o.uploadParts(ctx, storagePath, created.UploadId, body, part, n, ifNoneMatch); err != nil { + _, _ = o.client.AbortMultipartUpload(context.WithoutCancel(ctx), objectstorage.AbortMultipartUploadRequest{ + NamespaceName: &o.s.namespace, + BucketName: &o.s.conf.Bucket, + ObjectName: &storagePath, + UploadId: created.UploadId, + }) + } + return err +} + +// uploadParts sends part[:n], then the rest of body a part at a time, and +// commits. The condition goes on the commit only: on a part, a retry after a +// part that landed would read as the object existing. +func (o *ociObjects) uploadParts(ctx context.Context, storagePath string, uploadID *string, body io.Reader, part []byte, n int, ifNoneMatch *string) error { + var parts []objectstorage.CommitMultipartUploadPartDetails + for num := 1; n > 0; num++ { + size := int64(n) + resp, err := o.client.UploadPart(ctx, objectstorage.UploadPartRequest{ + NamespaceName: &o.s.namespace, + BucketName: &o.s.conf.Bucket, + ObjectName: &storagePath, + UploadId: uploadID, + UploadPartNum: common.Int(num), + ContentLength: &size, + UploadPartBody: io.NopCloser(bytes.NewReader(part[:n])), + }) + if err != nil { + return wrapOCIObjectError(err) + } + parts = append(parts, objectstorage.CommitMultipartUploadPartDetails{PartNum: common.Int(num), Etag: resp.ETag}) + + n, err = io.ReadFull(body, part) + if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { + return err + } + } + + _, err := o.client.CommitMultipartUpload(ctx, objectstorage.CommitMultipartUploadRequest{ + NamespaceName: &o.s.namespace, + BucketName: &o.s.conf.Bucket, + ObjectName: &storagePath, + UploadId: uploadID, + IfNoneMatch: ifNoneMatch, + CommitMultipartUploadDetails: objectstorage.CommitMultipartUploadDetails{PartsToCommit: parts}, + }) + return wrapOCIObjectError(err) +} + +func (o *ociObjects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { + var buf bytes.Buffer + if _, err := o.DownloadObject(ctx, storagePath, &buf); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (o *ociObjects) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + var buf bytes.Buffer + rng := fmt.Sprintf("bytes=%d-%d", offset, offset+length-1) + _, err := o.download(ctx, storagePath, &buf, &rng) + var se common.ServiceError + if errors.As(err, &se) && se.GetHTTPStatusCode() == http.StatusRequestedRangeNotSatisfiable { + return []byte{}, nil + } + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (o *ociObjects) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { + return o.download(ctx, storagePath, w, nil) +} + +func (o *ociObjects) download(ctx context.Context, storagePath string, w io.Writer, rng *string) (int64, error) { + resp, err := o.client.GetObject(ctx, objectstorage.GetObjectRequest{ + NamespaceName: &o.s.namespace, + BucketName: &o.s.conf.Bucket, + ObjectName: &storagePath, + Range: rng, + }) + if err != nil { + return 0, wrapOCIObjectError(err) + } + defer resp.Content.Close() + n, err := io.Copy(w, resp.Content) + return n, wrapOCIObjectError(err) +} + +func (o *ociObjects) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + _, err := o.client.HeadObject(ctx, objectstorage.HeadObjectRequest{ + NamespaceName: &o.s.namespace, + BucketName: &o.s.conf.Bucket, + ObjectName: &storagePath, + }) + err = wrapOCIObjectError(err) + if errors.Is(err, ErrObjectNotFound) { + return false, nil + } + return err == nil, err +} + +func (o *ociObjects) ListObjects(ctx context.Context, prefix string) ([]string, error) { + var keys []string + var start *string + for { + resp, err := o.client.ListObjects(ctx, objectstorage.ListObjectsRequest{ + NamespaceName: &o.s.namespace, + BucketName: &o.s.conf.Bucket, + Prefix: &prefix, + Start: start, + }) + if err != nil { + return nil, wrapOCIObjectError(err) + } + for _, obj := range resp.Objects { + keys = append(keys, *obj.Name) + } + if resp.NextStartWith == nil { + return keys, nil + } + start = resp.NextStartWith + } +} + +func (o *ociObjects) DeleteObjects(ctx context.Context, storagePaths []string) error { + for _, p := range storagePaths { + _, err := o.client.DeleteObject(ctx, objectstorage.DeleteObjectRequest{ + NamespaceName: &o.s.namespace, + BucketName: &o.s.conf.Bucket, + ObjectName: &p, + }) + if err = wrapOCIObjectError(err); err != nil && !errors.Is(err, ErrObjectNotFound) { + return err + } + } + return nil +} + +func (o *ociObjects) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { + return o.s.GeneratePresignedUrl(storagePath, expiration) +} + +func wrapOCIObjectError(err error) error { + if err == nil { + return nil + } + var se common.ServiceError + if errors.As(err, &se) { + switch se.GetHTTPStatusCode() { + case http.StatusNotFound: + // A HEAD has no body, so its code is the SDK's BadErrorResponse. + // NotAuthorizedOrNotFound stays an error. + switch se.GetCode() { + case "ObjectNotFound", "BadErrorResponse": + return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapOCIError(err)) + } + case http.StatusPreconditionFailed: + return fmt.Errorf("%w: %w", ErrObjectExists, wrapOCIError(err)) + } + } + return wrapOCIError(err) +} diff --git a/s3.go b/s3.go index ecba17d..9532e73 100644 --- a/s3.go +++ b/s3.go @@ -243,11 +243,20 @@ func (s *s3Storage) upload(reader io.Reader, storagePath, contentType string) (s input.ContentDisposition = &contentDisposition } + if _, err := s.uploader(client).Upload(context.Background(), input); err != nil { + l.WriteLogs() + return "", wrapS3Error(err) + } + + return s.location(storagePath), nil +} + +func (s *s3Storage) uploader(client *s3.Client, opts ...func(*manager.Uploader)) *manager.Uploader { // Uploader forces a CRC32 ChecksumAlgorithm on multipart uploads, // which makes UploadPart use a trailing checksum encoded as Content-Encoding: // aws-chunked. Non-AWS S3-compatible providers (e.g. OCI) don't support // this, so clear the algorithm before any checksum middleware sees it. - uploader := manager.NewUploader(client, func(u *manager.Uploader) { + return manager.NewUploader(client, append(opts, func(u *manager.Uploader) { if s.conf.Endpoint != "" { u.ClientOptions = append(u.ClientOptions, func(o *s3.Options) { o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { @@ -268,14 +277,7 @@ func (s *s3Storage) upload(reader io.Reader, storagePath, contentType string) (s }) }) } - }) - - if _, err := uploader.Upload(context.Background(), input); err != nil { - l.WriteLogs() - return "", wrapS3Error(err) - } - - return s.location(storagePath), nil + })...) } func (s *s3Storage) location(storagePath string) string { diff --git a/s3_object.go b/s3_object.go index 7d0e991..536ec87 100644 --- a/s3_object.go +++ b/s3_object.go @@ -62,7 +62,7 @@ func (o *s3Objects) UploadObject(ctx context.Context, storagePath string, body i return ErrObjectExists } } - uploader := manager.NewUploader(o.s.getClient(nil), func(u *manager.Uploader) { + uploader := o.s.uploader(o.s.getClient(nil), func(u *manager.Uploader) { u.PartSize = s3UploadPartSize }) _, err := uploader.Upload(ctx, o.putInput(storagePath, body, opts)) @@ -98,6 +98,10 @@ func (o *s3Objects) GetObjectRange(ctx context.Context, storagePath string, offs Key: aws.String(storagePath), Range: aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)), }) + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "InvalidRange" { + return []byte{}, nil + } if err != nil { return nil, wrapS3ObjectError(err) } From 14e4be19585a4faf4a05f6cd50418febf3facd01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 00:27:44 -0700 Subject: [PATCH 04/29] ObjectStorage: IfAbsent atomic on S3 uploads, one constructor, local temp files unlisted UploadObject sends If-None-Match on CompleteMultipartUpload, which the uploader drops; MinIO and OCI's S3 API both refuse with it. The per-backend constructors are unexported behind NewObjectStorage, GetObject is shared over DownloadObject, and local ListObjects leaves out UploadObject's temporary files. --- alioss_object.go | 8 ++------ azure_object.go | 8 ++------ gcp_object.go | 8 ++------ local_object.go | 15 ++++++++++++--- object.go | 23 +++++++++++++++++------ objects/objects.go | 5 ++--- oci_object.go | 10 +++------- s3_object.go | 35 ++++++++++++++++++++++++++++------- 8 files changed, 68 insertions(+), 44 deletions(-) diff --git a/alioss_object.go b/alioss_object.go index 4770efe..78c6a0e 100644 --- a/alioss_object.go +++ b/alioss_object.go @@ -30,7 +30,7 @@ type aliOSSObjects struct { s *aliOSSStorage } -func NewAliOSSObjects(conf *AliOSSConfig) (ObjectStorage, error) { +func newAliOSSObjects(conf *AliOSSConfig) (ObjectStorage, error) { s, err := NewAliOSS(conf) if err != nil { return nil, err @@ -58,11 +58,7 @@ func (o *aliOSSObjects) UploadObject(ctx context.Context, storagePath string, bo } func (o *aliOSSObjects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { - var buf bytes.Buffer - if _, err := o.DownloadObject(ctx, storagePath, &buf); err != nil { - return nil, err - } - return buf.Bytes(), nil + return getObject(ctx, o, storagePath) } // GetObjectRange asks for standard range behavior: by default OSS answers an diff --git a/azure_object.go b/azure_object.go index e3c8c99..0582f38 100644 --- a/azure_object.go +++ b/azure_object.go @@ -33,7 +33,7 @@ type azureObjects struct { s *azureBLOBStorage } -func NewAzureObjects(conf *AzureConfig) (ObjectStorage, error) { +func newAzureObjects(conf *AzureConfig) (ObjectStorage, error) { s, err := NewAzure(conf) if err != nil { return nil, err @@ -76,11 +76,7 @@ func (o *azureObjects) UploadObject(ctx context.Context, storagePath string, bod } func (o *azureObjects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { - var buf bytes.Buffer - if _, err := o.DownloadObject(ctx, storagePath, &buf); err != nil { - return nil, err - } - return buf.Bytes(), nil + return getObject(ctx, o, storagePath) } func (o *azureObjects) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { diff --git a/gcp_object.go b/gcp_object.go index 182ac8f..f4a8bdf 100644 --- a/gcp_object.go +++ b/gcp_object.go @@ -32,7 +32,7 @@ type gcpObjects struct { s *gcpStorage } -func NewGCPObjects(conf *GCPConfig) (ObjectStorage, error) { +func newGCPObjects(conf *GCPConfig) (ObjectStorage, error) { s, err := NewGCP(conf) if err != nil { return nil, err @@ -89,11 +89,7 @@ func (o *gcpObjects) writeError(err error, ifAbsent bool) error { } func (o *gcpObjects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { - var buf bytes.Buffer - if _, err := o.DownloadObject(ctx, storagePath, &buf); err != nil { - return nil, err - } - return buf.Bytes(), nil + return getObject(ctx, o, storagePath) } func (o *gcpObjects) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { diff --git a/local_object.go b/local_object.go index f9c8cff..8597177 100644 --- a/local_object.go +++ b/local_object.go @@ -23,14 +23,18 @@ import ( "io/fs" "os" "path/filepath" + "slices" + "strings" "time" ) +const localUploadPrefix = ".upload-" + type localObjects struct { u *localUploader } -func NewLocalObjects(conf *LocalConfig) (ObjectStorage, error) { +func newLocalObjects(conf *LocalConfig) (ObjectStorage, error) { u, err := NewLocal(conf) if err != nil { return nil, err @@ -49,7 +53,7 @@ func (o *localObjects) UploadObject(_ context.Context, storagePath string, body if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { return err } - tmp, err := os.CreateTemp(filepath.Dir(dst), ".upload-*") + tmp, err := os.CreateTemp(filepath.Dir(dst), localUploadPrefix+"*") if err != nil { return err } @@ -109,8 +113,13 @@ func (o *localObjects) ObjectExists(_ context.Context, storagePath string) (bool return err == nil, err } +// ListObjects leaves out UploadObject's temporary files, in progress or left by +// a crash. func (o *localObjects) ListObjects(_ context.Context, prefix string) ([]string, error) { - return o.u.ListObjects(prefix) + keys, err := o.u.ListObjects(prefix) + return slices.DeleteFunc(keys, func(k string) bool { + return strings.HasPrefix(filepath.Base(k), localUploadPrefix) + }), err } func (o *localObjects) DeleteObjects(_ context.Context, storagePaths []string) error { diff --git a/object.go b/object.go index dd76b9e..895caea 100644 --- a/object.go +++ b/object.go @@ -15,6 +15,8 @@ package storage import ( + "bytes" + "context" "fmt" "github.com/livekit/storage/objects" @@ -35,18 +37,27 @@ var ( func NewObjectStorage(conf Config) (ObjectStorage, error) { switch c := conf.(type) { case *AliOSSConfig: - return NewAliOSSObjects(c) + return newAliOSSObjects(c) case *AzureConfig: - return NewAzureObjects(c) + return newAzureObjects(c) case *GCPConfig: - return NewGCPObjects(c) + return newGCPObjects(c) case *LocalConfig: - return NewLocalObjects(c) + return newLocalObjects(c) case *OCIConfig: - return NewOCIObjects(c) + return newOCIObjects(c) case *S3Config: - return NewS3Objects(c) + return newS3Objects(c) default: return nil, fmt.Errorf("object storage is not supported for %T", conf) } } + +// getObject is GetObject for a backend whose reads are streams. +func getObject(ctx context.Context, s ObjectStorage, storagePath string) ([]byte, error) { + var buf bytes.Buffer + if _, err := s.DownloadObject(ctx, storagePath, &buf); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/objects/objects.go b/objects/objects.go index e60dea7..8a1a4c2 100644 --- a/objects/objects.go +++ b/objects/objects.go @@ -49,9 +49,8 @@ type Storage interface { type PutOptions struct { ContentType string - // IfAbsent fails with ErrExists when the path holds an object. PutObject - // checks it atomically. UploadObject checks it before uploading, so two - // concurrent uploads to one path can both succeed. + // IfAbsent fails with ErrExists when the path holds an object, atomically: + // of concurrent writes to one path, one succeeds. IfAbsent bool } diff --git a/oci_object.go b/oci_object.go index 99170c0..6a0a2c2 100644 --- a/oci_object.go +++ b/oci_object.go @@ -34,10 +34,10 @@ type ociObjects struct { client objectstorage.ObjectStorageClient } -// NewOCIObjects gives its calls a client of their own without the SDK's 60s +// newOCIObjects gives its calls a client of their own without the SDK's 60s // request timeout, which covers reading a body and would cut a long download: // the caller's context bounds each call instead. -func NewOCIObjects(conf *OCIConfig) (ObjectStorage, error) { +func newOCIObjects(conf *OCIConfig) (ObjectStorage, error) { s, err := NewOCI(conf) if err != nil { return nil, err @@ -153,11 +153,7 @@ func (o *ociObjects) uploadParts(ctx context.Context, storagePath string, upload } func (o *ociObjects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { - var buf bytes.Buffer - if _, err := o.DownloadObject(ctx, storagePath, &buf); err != nil { - return nil, err - } - return buf.Bytes(), nil + return getObject(ctx, o, storagePath) } func (o *ociObjects) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { diff --git a/s3_object.go b/s3_object.go index 536ec87..73db50d 100644 --- a/s3_object.go +++ b/s3_object.go @@ -27,6 +27,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" + "github.com/aws/smithy-go/middleware" ) const s3UploadPartSize = 16 << 20 @@ -35,7 +36,7 @@ type s3Objects struct { s *s3Storage } -func NewS3Objects(conf *S3Config) (ObjectStorage, error) { +func newS3Objects(conf *S3Config) (ObjectStorage, error) { s, err := NewS3(conf) if err != nil { return nil, err @@ -52,6 +53,8 @@ func (o *s3Objects) PutObject(ctx context.Context, storagePath string, data []by return wrapS3ObjectError(err) } +// UploadObject checks IfAbsent before uploading, which spares a refused upload +// its traffic, and again when it completes. func (o *s3Objects) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { if opts.IfAbsent { exists, err := o.ObjectExists(ctx, storagePath) @@ -62,13 +65,35 @@ func (o *s3Objects) UploadObject(ctx context.Context, storagePath string, body i return ErrObjectExists } } + input := o.putInput(storagePath, body, opts) uploader := o.s.uploader(o.s.getClient(nil), func(u *manager.Uploader) { u.PartSize = s3UploadPartSize + if opts.IfAbsent { + // A body that fits one part goes up as this PutObject; a larger one + // is completed with the same condition, which the uploader drops. + input.IfNoneMatch = aws.String("*") + u.ClientOptions = append(u.ClientOptions, completeIfAbsent) + } }) - _, err := uploader.Upload(ctx, o.putInput(storagePath, body, opts)) + _, err := uploader.Upload(ctx, input) return wrapS3ObjectError(err) } +func completeIfAbsent(o *s3.Options) { + o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { + return stack.Initialize.Add(middleware.InitializeMiddlewareFunc("CompleteIfAbsent", + func(ctx context.Context, in middleware.InitializeInput, next middleware.InitializeHandler) ( + middleware.InitializeOutput, middleware.Metadata, error, + ) { + if v, ok := in.Parameters.(*s3.CompleteMultipartUploadInput); ok { + v.IfNoneMatch = aws.String("*") + } + return next.HandleInitialize(ctx, in) + }, + ), middleware.Before) + }) +} + func (o *s3Objects) putInput(storagePath string, body io.Reader, opts PutOptions) *s3.PutObjectInput { input := &s3.PutObjectInput{ Bucket: aws.String(o.s.conf.Bucket), @@ -82,11 +107,7 @@ func (o *s3Objects) putInput(storagePath string, body io.Reader, opts PutOptions } func (o *s3Objects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { - var buf bytes.Buffer - if _, err := o.DownloadObject(ctx, storagePath, &buf); err != nil { - return nil, err - } - return buf.Bytes(), nil + return getObject(ctx, o, storagePath) } func (o *s3Objects) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { From 41d0d1608659081e3fc813d41303653db36f0fa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 00:30:57 -0700 Subject: [PATCH 05/29] README, objects: when to use ObjectStorage rather than Storage --- README.md | 13 +++++++++++++ objects/objects.go | 8 +++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ed031d9..3407d30 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,16 @@ # Storage A GORM-style storage wrapper with unified API for different cloud storage backends + +Two interfaces over the same backends (S3 and S3-compatible, GCS, Azure, OCI, AliOSS, local): + +| | `Storage` (`storage.New`) | `ObjectStorage` (`storage.NewObjectStorage`) | +|---|---|---| +| for | uploading files and handing out their location (egress, SIP) | services that keep and manage their own objects | +| context | none | every call | +| writes | replace | can refuse to replace (`PutOptions.IfAbsent`, atomic) | +| reads | whole object, to memory or a file | whole, a range, or streamed to any `io.Writer` | +| missing or existing object | `ErrorWithStatusCode` | `ErrObjectNotFound`, `ErrObjectExists` (still wrapping the status code) | +| a write returns | location URL and size | only an error | + +`ObjectStorage` is also in `storage/objects`, which imports no provider SDK, for fakes and libraries that implement or accept it without creating one. diff --git a/objects/objects.go b/objects/objects.go index 8a1a4c2..ee94f16 100644 --- a/objects/objects.go +++ b/objects/objects.go @@ -24,9 +24,11 @@ import ( "time" ) -// Storage is storage for services that keep their own objects: every call -// takes a context, writes can refuse to replace an object, and reads can take -// a range or stream to a writer. +// Storage is for services that keep and manage their own objects, where +// storage.Storage is for uploading files and handing out their location. Every +// call takes a context, a write can refuse to replace an object, a read can take +// a range or stream to any writer, and a missing or existing object comes back +// as ErrNotFound or ErrExists. storage.Storage stays as it is for its callers. type Storage interface { PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error // UploadObject streams body in parts: memory is bounded by the part size From a2c860a5c0e30095b2169c0f5b251f0e5403b719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 00:35:59 -0700 Subject: [PATCH 06/29] Storage: one interface; ListObjects, DeleteObjects and GeneratePresignedUrl take a context ObjectStorage folds into Storage: every backend type implements the new calls, and ListObjects, DeleteObjects and GeneratePresignedUrl take a context with the new semantics (DeleteObjects succeeds for missing paths and fails on any key it could not delete). The interface lives in storage/objects; storage.Storage aliases it. Each provider test runs both suites; TestS3 takes an optional S3_ENDPOINT. Breaking: callers of the three calls pass a context, and implementations of Storage add the new methods. --- README.md | 14 +-- alioss.go | 156 +++++++++++++++++++++----- alioss_object.go | 169 ---------------------------- azure.go | 150 ++++++++++++++++++++----- azure_object.go | 169 ---------------------------- gcp.go | 145 ++++++++++++++++++++---- gcp_object.go | 166 ---------------------------- local.go | 107 +++++++++++++++++- local_object.go | 143 ------------------------ object.go | 63 ----------- object_test.go | 107 +----------------- objects/objects.go | 26 +++-- oci.go | 270 ++++++++++++++++++++++++++++++++++++++------- oci_object.go | 268 -------------------------------------------- s3.go | 228 +++++++++++++++++++++++++++++--------- s3_object.go | 236 --------------------------------------- storage.go | 37 ++++--- storage_test.go | 35 ++++-- 18 files changed, 956 insertions(+), 1533 deletions(-) delete mode 100644 alioss_object.go delete mode 100644 azure_object.go delete mode 100644 gcp_object.go delete mode 100644 local_object.go delete mode 100644 object.go delete mode 100644 oci_object.go delete mode 100644 s3_object.go diff --git a/README.md b/README.md index 3407d30..875cae3 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,9 @@ A GORM-style storage wrapper with unified API for different cloud storage backends -Two interfaces over the same backends (S3 and S3-compatible, GCS, Azure, OCI, AliOSS, local): +`storage.New(conf)` returns a `Storage` for S3 and S3-compatible stores, GCS, Azure, OCI, AliOSS or a local directory. -| | `Storage` (`storage.New`) | `ObjectStorage` (`storage.NewObjectStorage`) | -|---|---|---| -| for | uploading files and handing out their location (egress, SIP) | services that keep and manage their own objects | -| context | none | every call | -| writes | replace | can refuse to replace (`PutOptions.IfAbsent`, atomic) | -| reads | whole object, to memory or a file | whole, a range, or streamed to any `io.Writer` | -| missing or existing object | `ErrorWithStatusCode` | `ErrObjectNotFound`, `ErrObjectExists` (still wrapping the status code) | -| a write returns | location URL and size | only an error | +- `UploadData`, `UploadFile`, `DownloadData`, `DownloadFile` and `DeleteObject` upload and fetch whole files and return their location. +- Every other call takes a context and reports a missing or existing object as `ErrObjectNotFound` or `ErrObjectExists` (still wrapping the status code): `PutObject` and `UploadObject` can refuse to replace an object (`PutOptions.IfAbsent`, atomic), `GetObjectRange` and `DownloadObject` read a range or stream to any `io.Writer`, `DeleteObjects` succeeds for paths that hold nothing and fails on any key it could not delete. -`ObjectStorage` is also in `storage/objects`, which imports no provider SDK, for fakes and libraries that implement or accept it without creating one. +The interface is also in `storage/objects`, which imports no provider SDK, for fakes and libraries that implement or accept it without creating one. diff --git a/alioss.go b/alioss.go index e420edf..71b3f02 100644 --- a/alioss.go +++ b/alioss.go @@ -16,8 +16,11 @@ package storage import ( "bytes" + "context" "errors" + "fmt" "io" + "net/http" "net/url" "os" "strings" @@ -98,28 +101,6 @@ func (s *aliOSSStorage) location(storagePath string) string { return u.String() } -func (s *aliOSSStorage) ListObjects(prefix string) ([]string, error) { - var objects []string - marker := oss.Marker("") - for { - lor, err := s.bucket.ListObjects(oss.Prefix(prefix), marker) - if err != nil { - return nil, wrapAliOSSError(err) - } - - for _, object := range lor.Objects { - objects = append(objects, object.Key) - } - - if !lor.IsTruncated { - break - } - marker = oss.Marker(lor.NextMarker) - } - - return objects, nil -} - func (s *aliOSSStorage) DownloadData(storagePath string) ([]byte, error) { reader, err := s.bucket.GetObject(storagePath) if err != nil { @@ -147,7 +128,7 @@ func (s *aliOSSStorage) DownloadFile(filepath, storagePath string) (int64, error return info.Size(), nil } -func (s *aliOSSStorage) GeneratePresignedUrl(storagePath string, expiration time.Duration) (string, error) { +func (s *aliOSSStorage) presignedUrl(storagePath string, expiration time.Duration) (string, error) { u, err := s.bucket.SignURL(storagePath, oss.HTTPGet, int64(expiration.Seconds())) if err != nil { return "", wrapAliOSSError(err) @@ -159,7 +140,132 @@ func (s *aliOSSStorage) DeleteObject(storagePath string) error { return wrapAliOSSError(s.bucket.DeleteObject(storagePath)) } -func (s *aliOSSStorage) DeleteObjects(storagePaths []string) error { - _, err := s.bucket.DeleteObjects(storagePaths) +func (s *aliOSSStorage) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + return s.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) +} + +// UploadObject streams body in one PutObject, chunked when its length is +// unknown, so it holds no part in memory and caps an object at 5 GB. IfAbsent +// sets x-oss-forbid-overwrite, atomic here, but OSS ignores it on a bucket with +// versioning enabled or suspended. +func (s *aliOSSStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + options := []oss.Option{oss.WithContext(ctx)} + if opts.ContentType != "" { + options = append(options, oss.ContentType(opts.ContentType)) + } + if opts.IfAbsent { + options = append(options, oss.ForbidOverWrite(true)) + } + return wrapAliOSSObjectError(s.bucket.PutObject(storagePath, body, options...)) +} + +func (s *aliOSSStorage) GetObject(ctx context.Context, storagePath string) ([]byte, error) { + return getObject(ctx, s, storagePath) +} + +// GetObjectRange asks for standard range behavior: by default OSS answers an +// unsatisfiable range with the whole object. +func (s *aliOSSStorage) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + var buf bytes.Buffer + _, err := s.download(ctx, storagePath, &buf, + oss.Range(offset, offset+length-1), oss.RangeBehavior("standard")) + var svcErr oss.ServiceError + if errors.As(err, &svcErr) && svcErr.StatusCode == http.StatusRequestedRangeNotSatisfiable { + return []byte{}, nil + } + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (s *aliOSSStorage) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { + return s.download(ctx, storagePath, w) +} + +func (s *aliOSSStorage) download(ctx context.Context, storagePath string, w io.Writer, options ...oss.Option) (int64, error) { + r, err := s.bucket.GetObject(storagePath, append(options, oss.WithContext(ctx))...) + if err != nil { + return 0, wrapAliOSSObjectError(err) + } + defer r.Close() + n, err := io.Copy(w, r) + return n, wrapAliOSSObjectError(err) +} + +func (s *aliOSSStorage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + exists, err := s.bucket.IsObjectExist(storagePath, oss.WithContext(ctx)) + return exists, wrapAliOSSObjectError(err) +} + +func (s *aliOSSStorage) ListObjects(ctx context.Context, prefix string) ([]string, error) { + var keys []string + options := []oss.Option{oss.Prefix(prefix), oss.MaxKeys(1000), oss.WithContext(ctx)} + for { + res, err := s.bucket.ListObjectsV2(options...) + if err != nil { + return nil, wrapAliOSSObjectError(err) + } + for _, obj := range res.Objects { + keys = append(keys, obj.Key) + } + if !res.IsTruncated || res.NextContinuationToken == "" { + return keys, nil + } + options = []oss.Option{oss.Prefix(prefix), oss.MaxKeys(1000), oss.ContinuationToken(res.NextContinuationToken), oss.WithContext(ctx)} + } +} + +// DeleteObjects sends batches of 1000, the most one request takes. OSS reports +// no per-key errors, only the keys it deleted, so a key it leaves out fails the +// call unless it holds nothing. +func (s *aliOSSStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { + for i := 0; i < len(storagePaths); i += 1000 { + batch := storagePaths[i:min(i+1000, len(storagePaths))] + res, err := s.bucket.DeleteObjects(batch, oss.WithContext(ctx)) + if err != nil { + return wrapAliOSSObjectError(err) + } + deleted := make(map[string]bool, len(res.DeletedObjects)) + for _, k := range res.DeletedObjects { + deleted[k] = true + } + for _, k := range batch { + if deleted[k] { + continue + } + exists, err := s.ObjectExists(ctx, k) + if err != nil { + return err + } + if exists { + return fmt.Errorf("storage: delete of %s not confirmed", k) + } + } + } + return nil +} + +func (s *aliOSSStorage) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { + return s.presignedUrl(storagePath, expiration) +} + +func wrapAliOSSObjectError(err error) error { + if err == nil { + return nil + } + var svcErr oss.ServiceError + if errors.As(err, &svcErr) { + switch { + // A HEAD answers a missing key with a bare 404. + case svcErr.Code == "NoSuchKey" || (svcErr.StatusCode == http.StatusNotFound && svcErr.Code == ""): + return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapAliOSSError(err)) + case svcErr.Code == "FileAlreadyExists": + return fmt.Errorf("%w: %w", ErrObjectExists, wrapAliOSSError(err)) + } + } return wrapAliOSSError(err) } diff --git a/alioss_object.go b/alioss_object.go deleted file mode 100644 index 78c6a0e..0000000 --- a/alioss_object.go +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "net/http" - "time" - - "github.com/aliyun/aliyun-oss-go-sdk/oss" -) - -type aliOSSObjects struct { - s *aliOSSStorage -} - -func newAliOSSObjects(conf *AliOSSConfig) (ObjectStorage, error) { - s, err := NewAliOSS(conf) - if err != nil { - return nil, err - } - return &aliOSSObjects{s: s.(*aliOSSStorage)}, nil -} - -func (o *aliOSSObjects) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { - return o.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) -} - -// UploadObject streams body in one PutObject, chunked when its length is -// unknown, so it holds no part in memory and caps an object at 5 GB. IfAbsent -// sets x-oss-forbid-overwrite, atomic here, but OSS ignores it on a bucket with -// versioning enabled or suspended. -func (o *aliOSSObjects) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { - options := []oss.Option{oss.WithContext(ctx)} - if opts.ContentType != "" { - options = append(options, oss.ContentType(opts.ContentType)) - } - if opts.IfAbsent { - options = append(options, oss.ForbidOverWrite(true)) - } - return wrapAliOSSObjectError(o.s.bucket.PutObject(storagePath, body, options...)) -} - -func (o *aliOSSObjects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { - return getObject(ctx, o, storagePath) -} - -// GetObjectRange asks for standard range behavior: by default OSS answers an -// unsatisfiable range with the whole object. -func (o *aliOSSObjects) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { - if offset < 0 || length <= 0 { - return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) - } - var buf bytes.Buffer - _, err := o.download(ctx, storagePath, &buf, - oss.Range(offset, offset+length-1), oss.RangeBehavior("standard")) - var svcErr oss.ServiceError - if errors.As(err, &svcErr) && svcErr.StatusCode == http.StatusRequestedRangeNotSatisfiable { - return []byte{}, nil - } - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -func (o *aliOSSObjects) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { - return o.download(ctx, storagePath, w) -} - -func (o *aliOSSObjects) download(ctx context.Context, storagePath string, w io.Writer, options ...oss.Option) (int64, error) { - r, err := o.s.bucket.GetObject(storagePath, append(options, oss.WithContext(ctx))...) - if err != nil { - return 0, wrapAliOSSObjectError(err) - } - defer r.Close() - n, err := io.Copy(w, r) - return n, wrapAliOSSObjectError(err) -} - -func (o *aliOSSObjects) ObjectExists(ctx context.Context, storagePath string) (bool, error) { - exists, err := o.s.bucket.IsObjectExist(storagePath, oss.WithContext(ctx)) - return exists, wrapAliOSSObjectError(err) -} - -func (o *aliOSSObjects) ListObjects(ctx context.Context, prefix string) ([]string, error) { - var keys []string - options := []oss.Option{oss.Prefix(prefix), oss.MaxKeys(1000), oss.WithContext(ctx)} - for { - res, err := o.s.bucket.ListObjectsV2(options...) - if err != nil { - return nil, wrapAliOSSObjectError(err) - } - for _, obj := range res.Objects { - keys = append(keys, obj.Key) - } - if !res.IsTruncated || res.NextContinuationToken == "" { - return keys, nil - } - options = []oss.Option{oss.Prefix(prefix), oss.MaxKeys(1000), oss.ContinuationToken(res.NextContinuationToken), oss.WithContext(ctx)} - } -} - -// DeleteObjects sends batches of 1000, the most one request takes. OSS reports -// no per-key errors, only the keys it deleted, so a key it leaves out fails the -// call unless it holds nothing. -func (o *aliOSSObjects) DeleteObjects(ctx context.Context, storagePaths []string) error { - for i := 0; i < len(storagePaths); i += 1000 { - batch := storagePaths[i:min(i+1000, len(storagePaths))] - res, err := o.s.bucket.DeleteObjects(batch, oss.WithContext(ctx)) - if err != nil { - return wrapAliOSSObjectError(err) - } - deleted := make(map[string]bool, len(res.DeletedObjects)) - for _, k := range res.DeletedObjects { - deleted[k] = true - } - for _, k := range batch { - if deleted[k] { - continue - } - exists, err := o.ObjectExists(ctx, k) - if err != nil { - return err - } - if exists { - return fmt.Errorf("storage: delete of %s not confirmed", k) - } - } - } - return nil -} - -func (o *aliOSSObjects) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { - return o.s.GeneratePresignedUrl(storagePath, expiration) -} - -func wrapAliOSSObjectError(err error) error { - if err == nil { - return nil - } - var svcErr oss.ServiceError - if errors.As(err, &svcErr) { - switch { - // A HEAD answers a missing key with a bare 404. - case svcErr.Code == "NoSuchKey" || (svcErr.StatusCode == http.StatusNotFound && svcErr.Code == ""): - return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapAliOSSError(err)) - case svcErr.Code == "FileAlreadyExists": - return fmt.Errorf("%w: %w", ErrObjectExists, wrapAliOSSError(err)) - } - } - return wrapAliOSSError(err) -} diff --git a/azure.go b/azure.go index 375ec8e..543c5c6 100644 --- a/azure.go +++ b/azure.go @@ -15,9 +15,11 @@ package storage import ( + "bytes" "context" "errors" "fmt" + "io" "net/url" "os" "path" @@ -27,6 +29,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" ) @@ -119,27 +122,6 @@ func (s *azureBLOBStorage) UploadFile(filepath, storagePath, contentType string) return s.location(storagePath), stat.Size(), nil } -func (s *azureBLOBStorage) ListObjects(prefix string) ([]string, error) { - var objects []string - - pager := s.client.NewListBlobsFlatPager(s.conf.ContainerName, &azblob.ListBlobsFlatOptions{ - Prefix: &prefix, - }) - for pager.More() { - page, err := pager.NextPage(context.Background()) - if err != nil { - return nil, wrapAzureError(err) - } - for _, item := range page.Segment.BlobItems { - if item.Name != nil { - objects = append(objects, *item.Name) - } - } - } - - return objects, nil -} - func (s *azureBLOBStorage) DownloadData(storagePath string) ([]byte, error) { ctx := context.Background() blobClient := s.client.ServiceClient().NewContainerClient(s.conf.ContainerName).NewBlobClient(storagePath) @@ -185,7 +167,7 @@ func (s *azureBLOBStorage) DownloadFile(filepath, storagePath string) (int64, er return stat.Size(), nil } -func (s *azureBLOBStorage) GeneratePresignedUrl(storagePath string, expiration time.Duration) (string, error) { +func (s *azureBLOBStorage) presignedUrlWithToken(storagePath string, expiration time.Duration) (string, error) { if s.conf.TokenCredential == nil { return "", errors.New("OAuth required") } @@ -232,11 +214,129 @@ func (s *azureBLOBStorage) DeleteObject(storagePath string) error { return wrapAzureError(err) } -func (s *azureBLOBStorage) DeleteObjects(storagePaths []string) error { - for _, p := range storagePaths { - if err := s.DeleteObject(p); err != nil { +func (s *azureBLOBStorage) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + return s.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) +} + +// UploadObject stages blocks and commits them; IfAbsent is a condition on the +// commit, so it is atomic here. The check before staging only spares a refused +// upload its traffic. +func (s *azureBLOBStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + if opts.IfAbsent { + exists, err := s.ObjectExists(ctx, storagePath) + if err != nil { return err } + if exists { + return ErrObjectExists + } + } + uploadOpts := &azblob.UploadStreamOptions{BlockSize: azureBlockSize, Concurrency: azureParallelism} + if opts.ContentType != "" { + uploadOpts.HTTPHeaders = &blob.HTTPHeaders{BlobContentType: to.Ptr(opts.ContentType)} + } + if opts.IfAbsent { + uploadOpts.AccessConditions = &blob.AccessConditions{ + ModifiedAccessConditions: &blob.ModifiedAccessConditions{IfNoneMatch: to.Ptr(azcore.ETagAny)}, + } + } + _, err := s.client.UploadStream(ctx, s.conf.ContainerName, storagePath, body, uploadOpts) + // A racing upload's commit discards this one's staged blocks. + if opts.IfAbsent && bloberror.HasCode(err, bloberror.InvalidBlockList) { + return fmt.Errorf("%w: %w", ErrObjectExists, wrapAzureError(err)) + } + return wrapAzureObjectError(err) +} + +func (s *azureBLOBStorage) GetObject(ctx context.Context, storagePath string) ([]byte, error) { + return getObject(ctx, s, storagePath) +} + +func (s *azureBLOBStorage) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + var buf bytes.Buffer + _, err := s.download(ctx, storagePath, &buf, blob.HTTPRange{Offset: offset, Count: length}) + if bloberror.HasCode(err, bloberror.InvalidRange) { + return []byte{}, nil + } + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (s *azureBLOBStorage) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { + return s.download(ctx, storagePath, w, blob.HTTPRange{}) +} + +func (s *azureBLOBStorage) download(ctx context.Context, storagePath string, w io.Writer, r blob.HTTPRange) (int64, error) { + resp, err := s.client.DownloadStream(ctx, s.conf.ContainerName, storagePath, &azblob.DownloadStreamOptions{Range: r}) + if err != nil { + return 0, wrapAzureObjectError(err) + } + body := resp.NewRetryReader(ctx, &blob.RetryReaderOptions{MaxRetries: 3}) + defer body.Close() + n, err := io.Copy(w, body) + return n, wrapAzureObjectError(err) +} + +func (s *azureBLOBStorage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + blobClient := s.client.ServiceClient().NewContainerClient(s.conf.ContainerName).NewBlobClient(storagePath) + _, err := blobClient.GetProperties(ctx, nil) + if bloberror.HasCode(err, bloberror.BlobNotFound) { + return false, nil + } + return err == nil, wrapAzureObjectError(err) +} + +func (s *azureBLOBStorage) ListObjects(ctx context.Context, prefix string) ([]string, error) { + pager := s.client.NewListBlobsFlatPager(s.conf.ContainerName, &azblob.ListBlobsFlatOptions{Prefix: &prefix}) + var keys []string + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, wrapAzureObjectError(err) + } + for _, item := range page.Segment.BlobItems { + keys = append(keys, *item.Name) + } + } + return keys, nil +} + +func (s *azureBLOBStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { + for _, p := range storagePaths { + _, err := s.client.DeleteBlob(ctx, s.conf.ContainerName, p, &azblob.DeleteBlobOptions{ + DeleteSnapshots: to.Ptr(azblob.DeleteSnapshotsOptionTypeInclude), + }) + if err != nil && !bloberror.HasCode(err, bloberror.BlobNotFound) { + return wrapAzureObjectError(err) + } } return nil } + +// GeneratePresignedUrl signs with the account key unless a token credential is +// set, in which case it signs with a user delegation key as Storage does. +func (s *azureBLOBStorage) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { + if s.conf.TokenCredential != nil { + return s.presignedUrlWithToken(storagePath, expiration) + } + b := s.client.ServiceClient().NewContainerClient(s.conf.ContainerName).NewBlobClient(storagePath) + u, err := b.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(expiration), nil) + return u, wrapAzureError(err) +} + +func wrapAzureObjectError(err error) error { + switch { + case err == nil: + return nil + case bloberror.HasCode(err, bloberror.BlobNotFound): + return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapAzureError(err)) + case bloberror.HasCode(err, bloberror.BlobAlreadyExists, bloberror.ConditionNotMet): + return fmt.Errorf("%w: %w", ErrObjectExists, wrapAzureError(err)) + } + return wrapAzureError(err) +} diff --git a/azure_object.go b/azure_object.go deleted file mode 100644 index 0582f38..0000000 --- a/azure_object.go +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "bytes" - "context" - "fmt" - "io" - "time" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" -) - -type azureObjects struct { - s *azureBLOBStorage -} - -func newAzureObjects(conf *AzureConfig) (ObjectStorage, error) { - s, err := NewAzure(conf) - if err != nil { - return nil, err - } - return &azureObjects{s: s.(*azureBLOBStorage)}, nil -} - -func (o *azureObjects) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { - return o.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) -} - -// UploadObject stages blocks and commits them; IfAbsent is a condition on the -// commit, so it is atomic here. The check before staging only spares a refused -// upload its traffic. -func (o *azureObjects) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { - if opts.IfAbsent { - exists, err := o.ObjectExists(ctx, storagePath) - if err != nil { - return err - } - if exists { - return ErrObjectExists - } - } - uploadOpts := &azblob.UploadStreamOptions{BlockSize: azureBlockSize, Concurrency: azureParallelism} - if opts.ContentType != "" { - uploadOpts.HTTPHeaders = &blob.HTTPHeaders{BlobContentType: to.Ptr(opts.ContentType)} - } - if opts.IfAbsent { - uploadOpts.AccessConditions = &blob.AccessConditions{ - ModifiedAccessConditions: &blob.ModifiedAccessConditions{IfNoneMatch: to.Ptr(azcore.ETagAny)}, - } - } - _, err := o.s.client.UploadStream(ctx, o.s.conf.ContainerName, storagePath, body, uploadOpts) - // A racing upload's commit discards this one's staged blocks. - if opts.IfAbsent && bloberror.HasCode(err, bloberror.InvalidBlockList) { - return fmt.Errorf("%w: %w", ErrObjectExists, wrapAzureError(err)) - } - return wrapAzureObjectError(err) -} - -func (o *azureObjects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { - return getObject(ctx, o, storagePath) -} - -func (o *azureObjects) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { - if offset < 0 || length <= 0 { - return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) - } - var buf bytes.Buffer - _, err := o.download(ctx, storagePath, &buf, blob.HTTPRange{Offset: offset, Count: length}) - if bloberror.HasCode(err, bloberror.InvalidRange) { - return []byte{}, nil - } - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -func (o *azureObjects) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { - return o.download(ctx, storagePath, w, blob.HTTPRange{}) -} - -func (o *azureObjects) download(ctx context.Context, storagePath string, w io.Writer, r blob.HTTPRange) (int64, error) { - resp, err := o.s.client.DownloadStream(ctx, o.s.conf.ContainerName, storagePath, &azblob.DownloadStreamOptions{Range: r}) - if err != nil { - return 0, wrapAzureObjectError(err) - } - body := resp.NewRetryReader(ctx, &blob.RetryReaderOptions{MaxRetries: 3}) - defer body.Close() - n, err := io.Copy(w, body) - return n, wrapAzureObjectError(err) -} - -func (o *azureObjects) ObjectExists(ctx context.Context, storagePath string) (bool, error) { - blobClient := o.s.client.ServiceClient().NewContainerClient(o.s.conf.ContainerName).NewBlobClient(storagePath) - _, err := blobClient.GetProperties(ctx, nil) - if bloberror.HasCode(err, bloberror.BlobNotFound) { - return false, nil - } - return err == nil, wrapAzureObjectError(err) -} - -func (o *azureObjects) ListObjects(ctx context.Context, prefix string) ([]string, error) { - pager := o.s.client.NewListBlobsFlatPager(o.s.conf.ContainerName, &azblob.ListBlobsFlatOptions{Prefix: &prefix}) - var keys []string - for pager.More() { - page, err := pager.NextPage(ctx) - if err != nil { - return nil, wrapAzureObjectError(err) - } - for _, item := range page.Segment.BlobItems { - keys = append(keys, *item.Name) - } - } - return keys, nil -} - -func (o *azureObjects) DeleteObjects(ctx context.Context, storagePaths []string) error { - for _, p := range storagePaths { - _, err := o.s.client.DeleteBlob(ctx, o.s.conf.ContainerName, p, &azblob.DeleteBlobOptions{ - DeleteSnapshots: to.Ptr(azblob.DeleteSnapshotsOptionTypeInclude), - }) - if err != nil && !bloberror.HasCode(err, bloberror.BlobNotFound) { - return wrapAzureObjectError(err) - } - } - return nil -} - -// GeneratePresignedUrl signs with the account key unless a token credential is -// set, in which case it signs with a user delegation key as Storage does. -func (o *azureObjects) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { - if o.s.conf.TokenCredential != nil { - return o.s.GeneratePresignedUrl(storagePath, expiration) - } - b := o.s.client.ServiceClient().NewContainerClient(o.s.conf.ContainerName).NewBlobClient(storagePath) - u, err := b.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(expiration), nil) - return u, wrapAzureError(err) -} - -func wrapAzureObjectError(err error) error { - switch { - case err == nil: - return nil - case bloberror.HasCode(err, bloberror.BlobNotFound): - return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapAzureError(err)) - case bloberror.HasCode(err, bloberror.BlobAlreadyExists, bloberror.ConditionNotMet): - return fmt.Errorf("%w: %w", ErrObjectExists, wrapAzureError(err)) - } - return wrapAzureError(err) -} diff --git a/gcp.go b/gcp.go index c68b32a..4b1bf34 100644 --- a/gcp.go +++ b/gcp.go @@ -150,24 +150,6 @@ func (s *gcpStorage) location(storagePath string) string { }).String() } -func (s *gcpStorage) ListObjects(prefix string) ([]string, error) { - it := s.client.Bucket(s.conf.Bucket).Objects(context.Background(), &storage.Query{ - Prefix: prefix, - }) - - var objects []string - for { - attr, err := it.Next() - if err != nil { - if errors.Is(err, iterator.Done) { - return objects, nil - } - return nil, wrapGCPError(err) - } - objects = append(objects, attr.Name) - } -} - func (s *gcpStorage) DownloadData(storagePath string) ([]byte, error) { rc, err := s.download(storagePath) if err != nil { @@ -219,7 +201,7 @@ func (s *gcpStorage) download(storagePath string) (*storage.Reader, error) { return r, nil } -func (s *gcpStorage) GeneratePresignedUrl(storagePath string, expiration time.Duration) (string, error) { +func (s *gcpStorage) presignedUrl(storagePath string, expiration time.Duration) (string, error) { u, err := s.client.Bucket(s.conf.Bucket).SignedURL(storagePath, &storage.SignedURLOptions{ Method: "GET", Expires: time.Now().Add(expiration), @@ -234,12 +216,127 @@ func (s *gcpStorage) DeleteObject(storagePath string) error { return wrapGCPError(s.client.Bucket(s.conf.Bucket).Object(storagePath).Delete(context.Background())) } -func (s *gcpStorage) DeleteObjects(storagePaths []string) error { - bucket := s.client.Bucket(s.conf.Bucket) - for _, path := range storagePaths { - if err := bucket.Object(path).Delete(context.Background()); err != nil { - return wrapGCPError(err) +func (s *gcpStorage) object(storagePath string) *storage.ObjectHandle { + return s.client.Bucket(s.conf.Bucket).Object(storagePath) +} + +// PutObject sizes the upload buffer to data, so a small put sends one request +// without a full chunk's allocation. +func (s *gcpStorage) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + return s.write(ctx, storagePath, bytes.NewReader(data), opts, min(len(data)+1, googleapi.DefaultUploadChunkSize)) +} + +// UploadObject sends body in resumable chunks; IfAbsent is a precondition on +// the whole upload, so it is atomic here. +func (s *gcpStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + return s.write(ctx, storagePath, body, opts, googleapi.DefaultUploadChunkSize) +} + +func (s *gcpStorage) write(ctx context.Context, storagePath string, body io.Reader, opts PutOptions, chunkSize int) error { + obj := s.object(storagePath) + if opts.IfAbsent { + // Conditional, so the SDK retries it by default. + obj = obj.If(storage.Conditions{DoesNotExist: true}) + } else { + obj = obj.Retryer(storage.WithPolicy(storage.RetryAlways)) + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + w := obj.NewWriter(ctx) + w.ContentType = opts.ContentType + w.ChunkSize = chunkSize + if _, err := io.Copy(w, body); err != nil { + // Cancelling aborts the upload; Close would store what was sent. + cancel() + return s.writeError(err, opts.IfAbsent) + } + return s.writeError(w.Close(), opts.IfAbsent) +} + +// writeError maps a failed precondition to ErrObjectExists only where IfAbsent +// set one: GCS answers other refusals with 412 too. +func (s *gcpStorage) writeError(err error, ifAbsent bool) error { + var apiErr *googleapi.Error + if ifAbsent && errors.As(err, &apiErr) && apiErr.Code == http.StatusPreconditionFailed { + return fmt.Errorf("%w: %w", ErrObjectExists, wrapGCPError(err)) + } + return wrapGCPObjectError(err) +} + +func (s *gcpStorage) GetObject(ctx context.Context, storagePath string) ([]byte, error) { + return getObject(ctx, s, storagePath) +} + +func (s *gcpStorage) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + r, err := s.object(storagePath).NewRangeReader(ctx, offset, length) + var apiErr *googleapi.Error + if errors.As(err, &apiErr) && apiErr.Code == http.StatusRequestedRangeNotSatisfiable { + return []byte{}, nil + } + if err != nil { + return nil, wrapGCPObjectError(err) + } + defer r.Close() + data, err := io.ReadAll(r) + return data, wrapGCPObjectError(err) +} + +func (s *gcpStorage) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { + r, err := s.object(storagePath).NewReader(ctx) + if err != nil { + return 0, wrapGCPObjectError(err) + } + defer r.Close() + n, err := io.Copy(w, r) + return n, wrapGCPObjectError(err) +} + +func (s *gcpStorage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + _, err := s.object(storagePath).Attrs(ctx) + if errors.Is(err, storage.ErrObjectNotExist) { + return false, nil + } + return err == nil, wrapGCPObjectError(err) +} + +func (s *gcpStorage) ListObjects(ctx context.Context, prefix string) ([]string, error) { + it := s.client.Bucket(s.conf.Bucket).Objects(ctx, &storage.Query{Prefix: prefix}) + var keys []string + for { + attrs, err := it.Next() + if errors.Is(err, iterator.Done) { + return keys, nil + } + if err != nil { + return nil, wrapGCPObjectError(err) + } + keys = append(keys, attrs.Name) + } +} + +func (s *gcpStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { + for _, p := range storagePaths { + if err := s.object(p).Delete(ctx); err != nil && !errors.Is(err, storage.ErrObjectNotExist) { + return wrapGCPObjectError(err) } } return nil } + +func (s *gcpStorage) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { + return s.presignedUrl(storagePath, expiration) +} + +func wrapGCPObjectError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, storage.ErrObjectNotExist) { + return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapGCPError(err)) + } + return wrapGCPError(err) +} diff --git a/gcp_object.go b/gcp_object.go deleted file mode 100644 index f4a8bdf..0000000 --- a/gcp_object.go +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "net/http" - "time" - - "cloud.google.com/go/storage" - "google.golang.org/api/googleapi" - "google.golang.org/api/iterator" -) - -type gcpObjects struct { - s *gcpStorage -} - -func newGCPObjects(conf *GCPConfig) (ObjectStorage, error) { - s, err := NewGCP(conf) - if err != nil { - return nil, err - } - return &gcpObjects{s: s.(*gcpStorage)}, nil -} - -func (o *gcpObjects) object(storagePath string) *storage.ObjectHandle { - return o.s.client.Bucket(o.s.conf.Bucket).Object(storagePath) -} - -// PutObject sizes the upload buffer to data, so a small put sends one request -// without a full chunk's allocation. -func (o *gcpObjects) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { - return o.write(ctx, storagePath, bytes.NewReader(data), opts, min(len(data)+1, googleapi.DefaultUploadChunkSize)) -} - -// UploadObject sends body in resumable chunks; IfAbsent is a precondition on -// the whole upload, so it is atomic here. -func (o *gcpObjects) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { - return o.write(ctx, storagePath, body, opts, googleapi.DefaultUploadChunkSize) -} - -func (o *gcpObjects) write(ctx context.Context, storagePath string, body io.Reader, opts PutOptions, chunkSize int) error { - obj := o.object(storagePath) - if opts.IfAbsent { - // Conditional, so the SDK retries it by default. - obj = obj.If(storage.Conditions{DoesNotExist: true}) - } else { - obj = obj.Retryer(storage.WithPolicy(storage.RetryAlways)) - } - - ctx, cancel := context.WithCancel(ctx) - defer cancel() - w := obj.NewWriter(ctx) - w.ContentType = opts.ContentType - w.ChunkSize = chunkSize - if _, err := io.Copy(w, body); err != nil { - // Cancelling aborts the upload; Close would store what was sent. - cancel() - return o.writeError(err, opts.IfAbsent) - } - return o.writeError(w.Close(), opts.IfAbsent) -} - -// writeError maps a failed precondition to ErrObjectExists only where IfAbsent -// set one: GCS answers other refusals with 412 too. -func (o *gcpObjects) writeError(err error, ifAbsent bool) error { - var apiErr *googleapi.Error - if ifAbsent && errors.As(err, &apiErr) && apiErr.Code == http.StatusPreconditionFailed { - return fmt.Errorf("%w: %w", ErrObjectExists, wrapGCPError(err)) - } - return wrapGCPObjectError(err) -} - -func (o *gcpObjects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { - return getObject(ctx, o, storagePath) -} - -func (o *gcpObjects) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { - if offset < 0 || length <= 0 { - return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) - } - r, err := o.object(storagePath).NewRangeReader(ctx, offset, length) - var apiErr *googleapi.Error - if errors.As(err, &apiErr) && apiErr.Code == http.StatusRequestedRangeNotSatisfiable { - return []byte{}, nil - } - if err != nil { - return nil, wrapGCPObjectError(err) - } - defer r.Close() - data, err := io.ReadAll(r) - return data, wrapGCPObjectError(err) -} - -func (o *gcpObjects) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { - r, err := o.object(storagePath).NewReader(ctx) - if err != nil { - return 0, wrapGCPObjectError(err) - } - defer r.Close() - n, err := io.Copy(w, r) - return n, wrapGCPObjectError(err) -} - -func (o *gcpObjects) ObjectExists(ctx context.Context, storagePath string) (bool, error) { - _, err := o.object(storagePath).Attrs(ctx) - if errors.Is(err, storage.ErrObjectNotExist) { - return false, nil - } - return err == nil, wrapGCPObjectError(err) -} - -func (o *gcpObjects) ListObjects(ctx context.Context, prefix string) ([]string, error) { - it := o.s.client.Bucket(o.s.conf.Bucket).Objects(ctx, &storage.Query{Prefix: prefix}) - var keys []string - for { - attrs, err := it.Next() - if errors.Is(err, iterator.Done) { - return keys, nil - } - if err != nil { - return nil, wrapGCPObjectError(err) - } - keys = append(keys, attrs.Name) - } -} - -func (o *gcpObjects) DeleteObjects(ctx context.Context, storagePaths []string) error { - for _, p := range storagePaths { - if err := o.object(p).Delete(ctx); err != nil && !errors.Is(err, storage.ErrObjectNotExist) { - return wrapGCPObjectError(err) - } - } - return nil -} - -func (o *gcpObjects) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { - return o.s.GeneratePresignedUrl(storagePath, expiration) -} - -func wrapGCPObjectError(err error) error { - if err == nil { - return nil - } - if errors.Is(err, storage.ErrObjectNotExist) { - return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapGCPError(err)) - } - return wrapGCPError(err) -} diff --git a/local.go b/local.go index 03677fa..f975265 100644 --- a/local.go +++ b/local.go @@ -15,11 +15,17 @@ package storage import ( + "bytes" + "context" + "errors" + "fmt" "io" + "io/fs" "net/url" "os" "path" "path/filepath" + "slices" "strings" "time" ) @@ -95,7 +101,7 @@ func (u *localUploader) location(storagePath string) string { return path.Join(u.StorageDir, storagePath) } -func (u *localUploader) ListObjects(prefix string) ([]string, error) { +func (u *localUploader) listObjects(prefix string) ([]string, error) { absPrefix := path.Join(u.StorageDir, prefix) dir, filenamePrefix := path.Split(absPrefix) @@ -175,7 +181,7 @@ func (u *localUploader) DownloadFile(localPath, storagePath string) (int64, erro return size, nil } -func (u *localUploader) GeneratePresignedUrl(storagePath string, _ time.Duration) (string, error) { +func (u *localUploader) presignedUrl(storagePath string, _ time.Duration) (string, error) { abs := filepath.Join(u.StorageDir, storagePath) return (&url.URL{Scheme: "file", Path: filepath.ToSlash(abs)}).String(), nil } @@ -201,11 +207,104 @@ func (u *localUploader) DeleteObject(storagePath string) error { } } -func (u *localUploader) DeleteObjects(storagePaths []string) error { +const localUploadPrefix = ".upload-" + +func (u *localUploader) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + return u.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) +} + +// UploadObject writes a temporary file and links or renames it into place, so +// readers never see a partial object and IfAbsent is atomic. +func (u *localUploader) UploadObject(_ context.Context, storagePath string, body io.Reader, opts PutOptions) error { + dst := u.location(storagePath) + if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(dst), localUploadPrefix+"*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err = io.Copy(tmp, body); err != nil { + tmp.Close() + return err + } + if err = tmp.Close(); err != nil { + return err + } + if !opts.IfAbsent { + return os.Rename(tmp.Name(), dst) + } + if err = os.Link(tmp.Name(), dst); errors.Is(err, fs.ErrExist) { + return ErrObjectExists + } + return err +} + +func (u *localUploader) GetObject(_ context.Context, storagePath string) ([]byte, error) { + data, err := os.ReadFile(u.location(storagePath)) + return data, localObjectError(err) +} + +func (u *localUploader) GetObjectRange(_ context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + f, err := os.Open(u.location(storagePath)) + if err != nil { + return nil, localObjectError(err) + } + defer f.Close() + buf := make([]byte, length) + n, err := f.ReadAt(buf, offset) + if errors.Is(err, io.EOF) { + err = nil + } + return buf[:n], err +} + +func (u *localUploader) DownloadObject(_ context.Context, storagePath string, w io.Writer) (int64, error) { + f, err := os.Open(u.location(storagePath)) + if err != nil { + return 0, localObjectError(err) + } + defer f.Close() + return io.Copy(w, f) +} + +func (u *localUploader) ObjectExists(_ context.Context, storagePath string) (bool, error) { + _, err := os.Stat(u.location(storagePath)) + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return err == nil, err +} + +// ListObjects leaves out UploadObject's temporary files, in progress or left by +// a crash. +func (u *localUploader) ListObjects(_ context.Context, prefix string) ([]string, error) { + keys, err := u.listObjects(prefix) + return slices.DeleteFunc(keys, func(k string) bool { + return strings.HasPrefix(filepath.Base(k), localUploadPrefix) + }), err +} + +func (u *localUploader) DeleteObjects(_ context.Context, storagePaths []string) error { for _, p := range storagePaths { - if err := u.DeleteObject(p); err != nil { + if err := u.DeleteObject(p); err != nil && !errors.Is(err, fs.ErrNotExist) { return err } } return nil } + +func (u *localUploader) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { + return u.presignedUrl(storagePath, expiration) +} + +func localObjectError(err error) error { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("%w: %w", ErrObjectNotFound, err) + } + return err +} diff --git a/local_object.go b/local_object.go deleted file mode 100644 index 8597177..0000000 --- a/local_object.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "slices" - "strings" - "time" -) - -const localUploadPrefix = ".upload-" - -type localObjects struct { - u *localUploader -} - -func newLocalObjects(conf *LocalConfig) (ObjectStorage, error) { - u, err := NewLocal(conf) - if err != nil { - return nil, err - } - return &localObjects{u: u.(*localUploader)}, nil -} - -func (o *localObjects) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { - return o.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) -} - -// UploadObject writes a temporary file and links or renames it into place, so -// readers never see a partial object and IfAbsent is atomic. -func (o *localObjects) UploadObject(_ context.Context, storagePath string, body io.Reader, opts PutOptions) error { - dst := o.u.location(storagePath) - if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { - return err - } - tmp, err := os.CreateTemp(filepath.Dir(dst), localUploadPrefix+"*") - if err != nil { - return err - } - defer os.Remove(tmp.Name()) - if _, err = io.Copy(tmp, body); err != nil { - tmp.Close() - return err - } - if err = tmp.Close(); err != nil { - return err - } - if !opts.IfAbsent { - return os.Rename(tmp.Name(), dst) - } - if err = os.Link(tmp.Name(), dst); errors.Is(err, fs.ErrExist) { - return ErrObjectExists - } - return err -} - -func (o *localObjects) GetObject(_ context.Context, storagePath string) ([]byte, error) { - data, err := os.ReadFile(o.u.location(storagePath)) - return data, localObjectError(err) -} - -func (o *localObjects) GetObjectRange(_ context.Context, storagePath string, offset, length int64) ([]byte, error) { - if offset < 0 || length <= 0 { - return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) - } - f, err := os.Open(o.u.location(storagePath)) - if err != nil { - return nil, localObjectError(err) - } - defer f.Close() - buf := make([]byte, length) - n, err := f.ReadAt(buf, offset) - if errors.Is(err, io.EOF) { - err = nil - } - return buf[:n], err -} - -func (o *localObjects) DownloadObject(_ context.Context, storagePath string, w io.Writer) (int64, error) { - f, err := os.Open(o.u.location(storagePath)) - if err != nil { - return 0, localObjectError(err) - } - defer f.Close() - return io.Copy(w, f) -} - -func (o *localObjects) ObjectExists(_ context.Context, storagePath string) (bool, error) { - _, err := os.Stat(o.u.location(storagePath)) - if errors.Is(err, fs.ErrNotExist) { - return false, nil - } - return err == nil, err -} - -// ListObjects leaves out UploadObject's temporary files, in progress or left by -// a crash. -func (o *localObjects) ListObjects(_ context.Context, prefix string) ([]string, error) { - keys, err := o.u.ListObjects(prefix) - return slices.DeleteFunc(keys, func(k string) bool { - return strings.HasPrefix(filepath.Base(k), localUploadPrefix) - }), err -} - -func (o *localObjects) DeleteObjects(_ context.Context, storagePaths []string) error { - for _, p := range storagePaths { - if err := o.u.DeleteObject(p); err != nil && !errors.Is(err, fs.ErrNotExist) { - return err - } - } - return nil -} - -func (o *localObjects) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { - return o.u.GeneratePresignedUrl(storagePath, expiration) -} - -func localObjectError(err error) error { - if errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("%w: %w", ErrObjectNotFound, err) - } - return err -} diff --git a/object.go b/object.go deleted file mode 100644 index 895caea..0000000 --- a/object.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "bytes" - "context" - "fmt" - - "github.com/livekit/storage/objects" -) - -// The interface lives in storage/objects so that code implementing or -// consuming it without creating one doesn't pull in every provider SDK. -type ( - ObjectStorage = objects.Storage - PutOptions = objects.PutOptions -) - -var ( - ErrObjectNotFound = objects.ErrNotFound - ErrObjectExists = objects.ErrExists -) - -func NewObjectStorage(conf Config) (ObjectStorage, error) { - switch c := conf.(type) { - case *AliOSSConfig: - return newAliOSSObjects(c) - case *AzureConfig: - return newAzureObjects(c) - case *GCPConfig: - return newGCPObjects(c) - case *LocalConfig: - return newLocalObjects(c) - case *OCIConfig: - return newOCIObjects(c) - case *S3Config: - return newS3Objects(c) - default: - return nil, fmt.Errorf("object storage is not supported for %T", conf) - } -} - -// getObject is GetObject for a backend whose reads are streams. -func getObject(ctx context.Context, s ObjectStorage, storagePath string) ([]byte, error) { - var buf bytes.Buffer - if _, err := s.DownloadObject(ctx, storagePath, &buf); err != nil { - return nil, err - } - return buf.Bytes(), nil -} diff --git a/object_test.go b/object_test.go index d70b3f9..f401bf3 100644 --- a/object_test.go +++ b/object_test.go @@ -18,7 +18,6 @@ import ( "bytes" "context" "fmt" - "os" "sort" "testing" "time" @@ -28,35 +27,7 @@ import ( "github.com/livekit/storage" ) -func TestLocalObjects(t *testing.T) { - s, err := storage.NewObjectStorage(&storage.LocalConfig{StorageDir: t.TempDir()}) - require.NoError(t, err) - testObjectStorage(t, s) -} - -// TestS3Objects reaches AWS, or an S3-compatible store at S3_ENDPOINT. -func TestS3Objects(t *testing.T) { - key := os.Getenv("AWS_ACCESS_KEY") - secret := os.Getenv("AWS_SECRET") - bucket := os.Getenv("S3_BUCKET") - if key == "" || secret == "" || bucket == "" { - t.Skip("Missing env vars") - } - endpoint := os.Getenv("S3_ENDPOINT") - s, err := storage.NewObjectStorage(&storage.S3Config{ - AccessKey: key, - Secret: secret, - SessionToken: os.Getenv("AWS_SESSION_TOKEN"), - Region: os.Getenv("AWS_REGION"), - Endpoint: endpoint, - ForcePathStyle: endpoint != "", - Bucket: bucket, - }) - require.NoError(t, err) - testObjectStorage(t, s) -} - -func testObjectStorage(t *testing.T, s storage.ObjectStorage) { +func testObjects(t *testing.T, s storage.Storage) { ctx := context.Background() prefix := fmt.Sprintf("objects-test-%d/", time.Now().UnixNano()) small := prefix + "small" @@ -132,79 +103,3 @@ func testObjectStorage(t *testing.T, s storage.ObjectStorage) { require.NotEmpty(t, url) }) } - -// TestOCIObjects reaches OCI through its S3 compatibility API, as TestOCI does. -func TestOCIObjects(t *testing.T) { - key := os.Getenv("OCI_ACCESS_KEY") - secret := os.Getenv("OCI_SECRET") - region := os.Getenv("OCI_REGION") - endpoint := os.Getenv("OCI_ENDPOINT") - bucket := os.Getenv("OCI_BUCKET") - if key == "" || secret == "" || region == "" || endpoint == "" || bucket == "" { - t.Skip("Missing env vars") - } - s, err := storage.NewObjectStorage(&storage.S3Config{ - AccessKey: key, - Secret: secret, - Region: region, - Endpoint: endpoint, - Bucket: bucket, - ForcePathStyle: true, - }) - require.NoError(t, err) - testObjectStorage(t, s) -} - -// TestOCINativeObjects uses the OCI SDK with an API key. -func TestOCINativeObjects(t *testing.T) { - conf := &storage.OCIConfig{ - TenancyOCID: os.Getenv("OCI_TENANCY_OCID"), - UserOCID: os.Getenv("OCI_USER_OCID"), - Fingerprint: os.Getenv("OCI_FINGERPRINT"), - PrivateKey: os.Getenv("OCI_PRIVATE_KEY"), - Region: os.Getenv("OCI_REGION"), - Bucket: os.Getenv("OCI_BUCKET"), - } - if conf.TenancyOCID == "" || conf.UserOCID == "" || conf.Fingerprint == "" || conf.PrivateKey == "" || conf.Region == "" || conf.Bucket == "" { - t.Skip("Missing env vars") - } - s, err := storage.NewObjectStorage(conf) - require.NoError(t, err) - testObjectStorage(t, s) -} - -func TestGCPObjects(t *testing.T) { - creds := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") - bucket := os.Getenv("GCP_BUCKET") - if creds == "" || bucket == "" { - t.Skip("Missing env vars") - } - s, err := storage.NewObjectStorage(&storage.GCPConfig{CredentialsJSON: creds, Bucket: bucket}) - require.NoError(t, err) - testObjectStorage(t, s) -} - -func TestAzureObjects(t *testing.T) { - name := os.Getenv("AZURE_ACCOUNT_NAME") - key := os.Getenv("AZURE_ACCOUNT_KEY") - container := os.Getenv("AZURE_CONTAINER_NAME") - if name == "" || key == "" || container == "" { - t.Skip("Missing env vars") - } - s, err := storage.NewObjectStorage(&storage.AzureConfig{AccountName: name, AccountKey: key, ContainerName: container}) - require.NoError(t, err) - testObjectStorage(t, s) -} - -func TestAliOSSObjects(t *testing.T) { - key := os.Getenv("ALI_ACCESS_KEY") - secret := os.Getenv("ALI_SECRET") - endpoint := os.Getenv("ALI_ENDPOINT") - bucket := os.Getenv("ALI_BUCKET") - if key == "" || secret == "" || endpoint == "" || bucket == "" { - t.Skip("Missing env vars") - } - s, err := storage.NewObjectStorage(&storage.AliOSSConfig{AccessKey: key, Secret: secret, Endpoint: endpoint, Bucket: bucket}) - require.NoError(t, err) - testObjectStorage(t, s) -} diff --git a/objects/objects.go b/objects/objects.go index ee94f16..db20142 100644 --- a/objects/objects.go +++ b/objects/objects.go @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package objects is the ObjectStorage interface without any provider SDK, for -// code that implements or consumes it without creating one, the way config -// holds the config structs. +// Package objects is the Storage interface without any provider SDK, for code +// that implements or accepts it without creating one, the way config holds the +// config structs. package objects import ( @@ -24,12 +24,19 @@ import ( "time" ) -// Storage is for services that keep and manage their own objects, where -// storage.Storage is for uploading files and handing out their location. Every -// call takes a context, a write can refuse to replace an object, a read can take -// a range or stream to any writer, and a missing or existing object comes back -// as ErrNotFound or ErrExists. storage.Storage stays as it is for its callers. type Storage interface { + // UploadData and UploadFile replace any object at storagePath and return + // its location. + UploadData(data []byte, storagePath, contentType string) (location string, size int64, err error) + UploadFile(filepath, storagePath, contentType string) (location string, size int64, err error) + + DownloadData(storagePath string) (data []byte, err error) + DownloadFile(filepath, storagePath string) (size int64, err error) + DeleteObject(storagePath string) error + + // The calls below take a context, and report a missing or existing object as + // ErrNotFound or ErrExists. + PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error // UploadObject streams body in parts: memory is bounded by the part size // times the upload concurrency, whatever the object's size. @@ -43,7 +50,8 @@ type Storage interface { ObjectExists(ctx context.Context, storagePath string) (bool, error) ListObjects(ctx context.Context, prefix string) ([]string, error) - // DeleteObjects succeeds for paths that hold nothing. + // DeleteObjects succeeds for paths that hold nothing and fails when a path + // is not deleted. DeleteObjects(ctx context.Context, storagePaths []string) error GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) diff --git a/oci.go b/oci.go index 1b7f5b5..f1994f0 100644 --- a/oci.go +++ b/oci.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "io" + "net/http" "net/url" "os" "path" @@ -32,8 +33,12 @@ import ( ) type ociStorage struct { - conf *OCIConfig - client objectstorage.ObjectStorageClient + conf *OCIConfig + client objectstorage.ObjectStorageClient + // ctxClient is client without the SDK's 60s request timeout, which covers + // reading a body and would cut a long download; the calls that take a + // context use it, and the context bounds them instead. + ctxClient objectstorage.ObjectStorageClient namespace string host string } @@ -68,6 +73,11 @@ func NewOCI(conf *OCIConfig) (Storage, error) { host: host, } + s.ctxClient = client + if hc, ok := client.HTTPClient.(*http.Client); ok { + s.ctxClient.HTTPClient = &http.Client{Transport: hc.Transport, CheckRedirect: hc.CheckRedirect, Jar: hc.Jar} + } + if s.namespace == "" { req := objectstorage.GetNamespaceRequest{} if conf.CompartmentID != "" { @@ -165,32 +175,6 @@ func (s *ociStorage) location(storagePath string) string { }).String() } -func (s *ociStorage) ListObjects(prefix string) ([]string, error) { - var objects []string - var start *string - - for { - resp, err := s.client.ListObjects(context.Background(), objectstorage.ListObjectsRequest{ - NamespaceName: &s.namespace, - BucketName: &s.conf.Bucket, - Prefix: &prefix, - Start: start, - }) - if err != nil { - return nil, wrapOCIError(err) - } - - for _, obj := range resp.Objects { - objects = append(objects, *obj.Name) - } - - if resp.NextStartWith == nil { - return objects, nil - } - start = resp.NextStartWith - } -} - func (s *ociStorage) DownloadData(storagePath string) ([]byte, error) { resp, err := s.download(storagePath) if err != nil { @@ -242,7 +226,7 @@ func (s *ociStorage) download(storagePath string) (objectstorage.GetObjectRespon // presigned URL, a PAR is a durable object: it is listable and revocable, it // costs an API call to mint, and it needs PAR_MANAGE on the bucket. Its access // URI is returned only here and cannot be read back afterwards. -func (s *ociStorage) GeneratePresignedUrl(storagePath string, expiration time.Duration) (string, error) { +func (s *ociStorage) presignedUrl(storagePath string, expiration time.Duration) (string, error) { name := fmt.Sprintf("livekit-storage-%s-%d", storagePath, time.Now().UnixNano()) expires := common.SDKTime{Time: time.Now().Add(expiration)} @@ -276,27 +260,235 @@ func (s *ociStorage) DeleteObject(storagePath string) error { return wrapOCIError(err) } -// DeleteObjects deletes one at a time — OCI has no bulk equivalent — and stops -// at the first failure, leaving earlier deletions in place, as gcp.go does. -func (s *ociStorage) DeleteObjects(storagePaths []string) error { - for _, storagePath := range storagePaths { - if err := s.DeleteObject(storagePath); err != nil { +func wrapOCIError(err error) error { + if err == nil { + return nil + } + var se common.ServiceError + if errors.As(err, &se) { + return &ErrorWithStatusCode{ + Err: err, + StatusCode: se.GetHTTPStatusCode(), + } + } + return err +} + +const ociUploadPartSize = 16 << 20 + +func (s *ociStorage) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + size := int64(len(data)) + req := objectstorage.PutObjectRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &storagePath, + // Seekable, so the SDK can re-read it when it retries. + PutObjectBody: common.NewOCIReadSeekCloser(io.NopCloser(bytes.NewReader(data))), + ContentLength: &size, + } + if opts.ContentType != "" { + req.ContentType = &opts.ContentType + } + if opts.IfAbsent { + req.IfNoneMatch = common.String("*") + } + _, err := s.ctxClient.PutObject(ctx, req) + return wrapOCIObjectError(err) +} + +// UploadObject sends a body that fits one part with PutObject, otherwise a +// multipart upload of sequential parts through one buffer. IfAbsent is checked +// at create and again at commit, so it is atomic here. +func (s *ociStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + part := make([]byte, ociUploadPartSize) + n, err := io.ReadFull(body, part) + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return s.PutObject(ctx, storagePath, part[:n], opts) + } + if err != nil { + return err + } + + var ifNoneMatch, contentType *string + if opts.IfAbsent { + ifNoneMatch = common.String("*") + } + if opts.ContentType != "" { + contentType = &opts.ContentType + } + created, err := s.ctxClient.CreateMultipartUpload(ctx, objectstorage.CreateMultipartUploadRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + IfNoneMatch: ifNoneMatch, + CreateMultipartUploadDetails: objectstorage.CreateMultipartUploadDetails{ + Object: &storagePath, + ContentType: contentType, + }, + }) + if err != nil { + return wrapOCIObjectError(err) + } + + if err = s.uploadParts(ctx, storagePath, created.UploadId, body, part, n, ifNoneMatch); err != nil { + _, _ = s.ctxClient.AbortMultipartUpload(context.WithoutCancel(ctx), objectstorage.AbortMultipartUploadRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &storagePath, + UploadId: created.UploadId, + }) + } + return err +} + +// uploadParts sends part[:n], then the rest of body a part at a time, and +// commits. The condition goes on the commit only: on a part, a retry after a +// part that landed would read as the object existing. +func (s *ociStorage) uploadParts(ctx context.Context, storagePath string, uploadID *string, body io.Reader, part []byte, n int, ifNoneMatch *string) error { + var parts []objectstorage.CommitMultipartUploadPartDetails + for num := 1; n > 0; num++ { + size := int64(n) + resp, err := s.ctxClient.UploadPart(ctx, objectstorage.UploadPartRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &storagePath, + UploadId: uploadID, + UploadPartNum: common.Int(num), + ContentLength: &size, + UploadPartBody: io.NopCloser(bytes.NewReader(part[:n])), + }) + if err != nil { + return wrapOCIObjectError(err) + } + parts = append(parts, objectstorage.CommitMultipartUploadPartDetails{PartNum: common.Int(num), Etag: resp.ETag}) + + n, err = io.ReadFull(body, part) + if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { + return err + } + } + + _, err := s.ctxClient.CommitMultipartUpload(ctx, objectstorage.CommitMultipartUploadRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &storagePath, + UploadId: uploadID, + IfNoneMatch: ifNoneMatch, + CommitMultipartUploadDetails: objectstorage.CommitMultipartUploadDetails{PartsToCommit: parts}, + }) + return wrapOCIObjectError(err) +} + +func (s *ociStorage) GetObject(ctx context.Context, storagePath string) ([]byte, error) { + return getObject(ctx, s, storagePath) +} + +func (s *ociStorage) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + var buf bytes.Buffer + rng := fmt.Sprintf("bytes=%d-%d", offset, offset+length-1) + _, err := s.readTo(ctx, storagePath, &buf, &rng) + var se common.ServiceError + if errors.As(err, &se) && se.GetHTTPStatusCode() == http.StatusRequestedRangeNotSatisfiable { + return []byte{}, nil + } + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (s *ociStorage) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { + return s.readTo(ctx, storagePath, w, nil) +} + +func (s *ociStorage) readTo(ctx context.Context, storagePath string, w io.Writer, rng *string) (int64, error) { + resp, err := s.ctxClient.GetObject(ctx, objectstorage.GetObjectRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &storagePath, + Range: rng, + }) + if err != nil { + return 0, wrapOCIObjectError(err) + } + defer resp.Content.Close() + n, err := io.Copy(w, resp.Content) + return n, wrapOCIObjectError(err) +} + +func (s *ociStorage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + _, err := s.ctxClient.HeadObject(ctx, objectstorage.HeadObjectRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &storagePath, + }) + err = wrapOCIObjectError(err) + if errors.Is(err, ErrObjectNotFound) { + return false, nil + } + return err == nil, err +} + +func (s *ociStorage) ListObjects(ctx context.Context, prefix string) ([]string, error) { + var keys []string + var start *string + for { + resp, err := s.ctxClient.ListObjects(ctx, objectstorage.ListObjectsRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + Prefix: &prefix, + Start: start, + }) + if err != nil { + return nil, wrapOCIObjectError(err) + } + for _, obj := range resp.Objects { + keys = append(keys, *obj.Name) + } + if resp.NextStartWith == nil { + return keys, nil + } + start = resp.NextStartWith + } +} + +func (s *ociStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { + for _, p := range storagePaths { + _, err := s.ctxClient.DeleteObject(ctx, objectstorage.DeleteObjectRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &p, + }) + if err = wrapOCIObjectError(err); err != nil && !errors.Is(err, ErrObjectNotFound) { return err } } return nil } -func wrapOCIError(err error) error { +func (s *ociStorage) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { + return s.presignedUrl(storagePath, expiration) +} + +func wrapOCIObjectError(err error) error { if err == nil { return nil } var se common.ServiceError if errors.As(err, &se) { - return &ErrorWithStatusCode{ - Err: err, - StatusCode: se.GetHTTPStatusCode(), + switch se.GetHTTPStatusCode() { + case http.StatusNotFound: + // A HEAD has no body, so its code is the SDK's BadErrorResponse. + // NotAuthorizedOrNotFound stays an error. + switch se.GetCode() { + case "ObjectNotFound", "BadErrorResponse": + return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapOCIError(err)) + } + case http.StatusPreconditionFailed: + return fmt.Errorf("%w: %w", ErrObjectExists, wrapOCIError(err)) } } - return err + return wrapOCIError(err) } diff --git a/oci_object.go b/oci_object.go deleted file mode 100644 index 6a0a2c2..0000000 --- a/oci_object.go +++ /dev/null @@ -1,268 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "net/http" - "time" - - "github.com/oracle/oci-go-sdk/v65/common" - "github.com/oracle/oci-go-sdk/v65/objectstorage" -) - -const ociUploadPartSize = 16 << 20 - -type ociObjects struct { - s *ociStorage - client objectstorage.ObjectStorageClient -} - -// newOCIObjects gives its calls a client of their own without the SDK's 60s -// request timeout, which covers reading a body and would cut a long download: -// the caller's context bounds each call instead. -func newOCIObjects(conf *OCIConfig) (ObjectStorage, error) { - s, err := NewOCI(conf) - if err != nil { - return nil, err - } - o := &ociObjects{s: s.(*ociStorage)} - o.client = o.s.client - if hc, ok := o.client.HTTPClient.(*http.Client); ok { - o.client.HTTPClient = &http.Client{Transport: hc.Transport, CheckRedirect: hc.CheckRedirect, Jar: hc.Jar} - } - return o, nil -} - -func (o *ociObjects) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { - size := int64(len(data)) - req := objectstorage.PutObjectRequest{ - NamespaceName: &o.s.namespace, - BucketName: &o.s.conf.Bucket, - ObjectName: &storagePath, - // Seekable, so the SDK can re-read it when it retries. - PutObjectBody: common.NewOCIReadSeekCloser(io.NopCloser(bytes.NewReader(data))), - ContentLength: &size, - } - if opts.ContentType != "" { - req.ContentType = &opts.ContentType - } - if opts.IfAbsent { - req.IfNoneMatch = common.String("*") - } - _, err := o.client.PutObject(ctx, req) - return wrapOCIObjectError(err) -} - -// UploadObject sends a body that fits one part with PutObject, otherwise a -// multipart upload of sequential parts through one buffer. IfAbsent is checked -// at create and again at commit, so it is atomic here. -func (o *ociObjects) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { - part := make([]byte, ociUploadPartSize) - n, err := io.ReadFull(body, part) - if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { - return o.PutObject(ctx, storagePath, part[:n], opts) - } - if err != nil { - return err - } - - var ifNoneMatch, contentType *string - if opts.IfAbsent { - ifNoneMatch = common.String("*") - } - if opts.ContentType != "" { - contentType = &opts.ContentType - } - created, err := o.client.CreateMultipartUpload(ctx, objectstorage.CreateMultipartUploadRequest{ - NamespaceName: &o.s.namespace, - BucketName: &o.s.conf.Bucket, - IfNoneMatch: ifNoneMatch, - CreateMultipartUploadDetails: objectstorage.CreateMultipartUploadDetails{ - Object: &storagePath, - ContentType: contentType, - }, - }) - if err != nil { - return wrapOCIObjectError(err) - } - - if err = o.uploadParts(ctx, storagePath, created.UploadId, body, part, n, ifNoneMatch); err != nil { - _, _ = o.client.AbortMultipartUpload(context.WithoutCancel(ctx), objectstorage.AbortMultipartUploadRequest{ - NamespaceName: &o.s.namespace, - BucketName: &o.s.conf.Bucket, - ObjectName: &storagePath, - UploadId: created.UploadId, - }) - } - return err -} - -// uploadParts sends part[:n], then the rest of body a part at a time, and -// commits. The condition goes on the commit only: on a part, a retry after a -// part that landed would read as the object existing. -func (o *ociObjects) uploadParts(ctx context.Context, storagePath string, uploadID *string, body io.Reader, part []byte, n int, ifNoneMatch *string) error { - var parts []objectstorage.CommitMultipartUploadPartDetails - for num := 1; n > 0; num++ { - size := int64(n) - resp, err := o.client.UploadPart(ctx, objectstorage.UploadPartRequest{ - NamespaceName: &o.s.namespace, - BucketName: &o.s.conf.Bucket, - ObjectName: &storagePath, - UploadId: uploadID, - UploadPartNum: common.Int(num), - ContentLength: &size, - UploadPartBody: io.NopCloser(bytes.NewReader(part[:n])), - }) - if err != nil { - return wrapOCIObjectError(err) - } - parts = append(parts, objectstorage.CommitMultipartUploadPartDetails{PartNum: common.Int(num), Etag: resp.ETag}) - - n, err = io.ReadFull(body, part) - if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { - return err - } - } - - _, err := o.client.CommitMultipartUpload(ctx, objectstorage.CommitMultipartUploadRequest{ - NamespaceName: &o.s.namespace, - BucketName: &o.s.conf.Bucket, - ObjectName: &storagePath, - UploadId: uploadID, - IfNoneMatch: ifNoneMatch, - CommitMultipartUploadDetails: objectstorage.CommitMultipartUploadDetails{PartsToCommit: parts}, - }) - return wrapOCIObjectError(err) -} - -func (o *ociObjects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { - return getObject(ctx, o, storagePath) -} - -func (o *ociObjects) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { - if offset < 0 || length <= 0 { - return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) - } - var buf bytes.Buffer - rng := fmt.Sprintf("bytes=%d-%d", offset, offset+length-1) - _, err := o.download(ctx, storagePath, &buf, &rng) - var se common.ServiceError - if errors.As(err, &se) && se.GetHTTPStatusCode() == http.StatusRequestedRangeNotSatisfiable { - return []byte{}, nil - } - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -func (o *ociObjects) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { - return o.download(ctx, storagePath, w, nil) -} - -func (o *ociObjects) download(ctx context.Context, storagePath string, w io.Writer, rng *string) (int64, error) { - resp, err := o.client.GetObject(ctx, objectstorage.GetObjectRequest{ - NamespaceName: &o.s.namespace, - BucketName: &o.s.conf.Bucket, - ObjectName: &storagePath, - Range: rng, - }) - if err != nil { - return 0, wrapOCIObjectError(err) - } - defer resp.Content.Close() - n, err := io.Copy(w, resp.Content) - return n, wrapOCIObjectError(err) -} - -func (o *ociObjects) ObjectExists(ctx context.Context, storagePath string) (bool, error) { - _, err := o.client.HeadObject(ctx, objectstorage.HeadObjectRequest{ - NamespaceName: &o.s.namespace, - BucketName: &o.s.conf.Bucket, - ObjectName: &storagePath, - }) - err = wrapOCIObjectError(err) - if errors.Is(err, ErrObjectNotFound) { - return false, nil - } - return err == nil, err -} - -func (o *ociObjects) ListObjects(ctx context.Context, prefix string) ([]string, error) { - var keys []string - var start *string - for { - resp, err := o.client.ListObjects(ctx, objectstorage.ListObjectsRequest{ - NamespaceName: &o.s.namespace, - BucketName: &o.s.conf.Bucket, - Prefix: &prefix, - Start: start, - }) - if err != nil { - return nil, wrapOCIObjectError(err) - } - for _, obj := range resp.Objects { - keys = append(keys, *obj.Name) - } - if resp.NextStartWith == nil { - return keys, nil - } - start = resp.NextStartWith - } -} - -func (o *ociObjects) DeleteObjects(ctx context.Context, storagePaths []string) error { - for _, p := range storagePaths { - _, err := o.client.DeleteObject(ctx, objectstorage.DeleteObjectRequest{ - NamespaceName: &o.s.namespace, - BucketName: &o.s.conf.Bucket, - ObjectName: &p, - }) - if err = wrapOCIObjectError(err); err != nil && !errors.Is(err, ErrObjectNotFound) { - return err - } - } - return nil -} - -func (o *ociObjects) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { - return o.s.GeneratePresignedUrl(storagePath, expiration) -} - -func wrapOCIObjectError(err error) error { - if err == nil { - return nil - } - var se common.ServiceError - if errors.As(err, &se) { - switch se.GetHTTPStatusCode() { - case http.StatusNotFound: - // A HEAD has no body, so its code is the SDK's BadErrorResponse. - // NotAuthorizedOrNotFound stays an error. - switch se.GetCode() { - case "ObjectNotFound", "BadErrorResponse": - return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapOCIError(err)) - } - case http.StatusPreconditionFailed: - return fmt.Errorf("%w: %w", ErrObjectExists, wrapOCIError(err)) - } - } - return wrapOCIError(err) -} diff --git a/s3.go b/s3.go index 9532e73..5e74b9b 100644 --- a/s3.go +++ b/s3.go @@ -37,6 +37,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/aws/smithy-go" "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" ) @@ -299,29 +300,6 @@ func (s *s3Storage) location(storagePath string) string { return loc.String() } -func (s *s3Storage) ListObjects(prefix string) ([]string, error) { - client := s.getClient(nil) - - var objects []string - paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{ - Bucket: aws.String(s.conf.Bucket), - Prefix: aws.String(prefix), - }) - - for paginator.HasMorePages() { - page, err := paginator.NextPage(context.Background()) - if err != nil { - return nil, wrapS3Error(err) - } - - for _, obj := range page.Contents { - objects = append(objects, *obj.Key) - } - } - - return objects, nil -} - func (s *s3Storage) DownloadData(storagePath string) ([]byte, error) { w := &manager.WriteAtBuffer{} _, err := s.download(w, storagePath) @@ -356,61 +334,205 @@ func (s *s3Storage) download(w io.WriterAt, storagePath string) (int64, error) { return n, wrapS3Error(err) } -func (s *s3Storage) GeneratePresignedUrl(storagePath string, expiration time.Duration) (string, error) { +func (s *s3Storage) DeleteObject(storagePath string) error { client := s.getClient(nil) - res, err := s3.NewPresignClient(client).PresignGetObject(context.Background(), &s3.GetObjectInput{ + _, err := client.DeleteObject(context.Background(), &s3.DeleteObjectInput{ Bucket: aws.String(s.conf.Bucket), Key: aws.String(storagePath), - }, s3.WithPresignExpires(expiration)) - if err != nil { - return "", wrapS3Error(err) + }) + return wrapS3Error(err) +} + +const s3UploadPartSize = 16 << 20 + +func (s *s3Storage) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + input := s.putInput(storagePath, bytes.NewReader(data), opts) + if opts.IfAbsent { + input.IfNoneMatch = aws.String("*") } + _, err := s.getClient(nil).PutObject(ctx, input) + return wrapS3ObjectError(err) +} - return res.URL, nil +// UploadObject checks IfAbsent before uploading, which spares a refused upload +// its traffic, and again when it completes. +func (s *s3Storage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + if opts.IfAbsent { + exists, err := s.ObjectExists(ctx, storagePath) + if err != nil { + return err + } + if exists { + return ErrObjectExists + } + } + input := s.putInput(storagePath, body, opts) + uploader := s.uploader(s.getClient(nil), func(u *manager.Uploader) { + u.PartSize = s3UploadPartSize + if opts.IfAbsent { + // A body that fits one part goes up as this PutObject; a larger one + // is completed with the same condition, which the uploader drops. + input.IfNoneMatch = aws.String("*") + u.ClientOptions = append(u.ClientOptions, completeIfAbsent) + } + }) + _, err := uploader.Upload(ctx, input) + return wrapS3ObjectError(err) } -func (s *s3Storage) DeleteObject(storagePath string) error { - client := s.getClient(nil) +func completeIfAbsent(o *s3.Options) { + o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { + return stack.Initialize.Add(middleware.InitializeMiddlewareFunc("CompleteIfAbsent", + func(ctx context.Context, in middleware.InitializeInput, next middleware.InitializeHandler) ( + middleware.InitializeOutput, middleware.Metadata, error, + ) { + if v, ok := in.Parameters.(*s3.CompleteMultipartUploadInput); ok { + v.IfNoneMatch = aws.String("*") + } + return next.HandleInitialize(ctx, in) + }, + ), middleware.Before) + }) +} - _, err := client.DeleteObject(context.Background(), &s3.DeleteObjectInput{ +func (s *s3Storage) putInput(storagePath string, body io.Reader, opts PutOptions) *s3.PutObjectInput { + input := &s3.PutObjectInput{ Bucket: aws.String(s.conf.Bucket), Key: aws.String(storagePath), + Body: body, + } + if opts.ContentType != "" { + input.ContentType = aws.String(opts.ContentType) + } + return input +} + +func (s *s3Storage) GetObject(ctx context.Context, storagePath string) ([]byte, error) { + return getObject(ctx, s, storagePath) +} + +func (s *s3Storage) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + out, err := s.getClient(nil).GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.conf.Bucket), + Key: aws.String(storagePath), + Range: aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)), }) - return wrapS3Error(err) + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "InvalidRange" { + return []byte{}, nil + } + if err != nil { + return nil, wrapS3ObjectError(err) + } + defer out.Body.Close() + return io.ReadAll(io.LimitReader(out.Body, length)) } -func (s *s3Storage) DeleteObjects(storagePaths []string) error { - client := s.getClient(nil) +func (s *s3Storage) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { + out, err := s.getClient(nil).GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.conf.Bucket), + Key: aws.String(storagePath), + }) + if err != nil { + return 0, wrapS3ObjectError(err) + } + defer out.Body.Close() + return io.Copy(w, out.Body) +} - for i := 0; i < len(storagePaths); i += 1000 { - end := i + 1000 - if end > len(storagePaths) { - end = len(storagePaths) - } +func (s *s3Storage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + _, err := s.getClient(nil).HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(s.conf.Bucket), + Key: aws.String(storagePath), + }) + err = wrapS3ObjectError(err) + if errors.Is(err, ErrObjectNotFound) { + return false, nil + } + return err == nil, err +} - objects := make([]types.ObjectIdentifier, 0, end-i) - for _, path := range storagePaths[i:end] { - objects = append(objects, types.ObjectIdentifier{Key: aws.String(path)}) +func (s *s3Storage) ListObjects(ctx context.Context, prefix string) ([]string, error) { + var keys []string + paginator := s3.NewListObjectsV2Paginator(s.getClient(nil), &s3.ListObjectsV2Input{ + Bucket: aws.String(s.conf.Bucket), + Prefix: aws.String(prefix), + }) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return nil, wrapS3Error(err) } + for _, obj := range page.Contents { + keys = append(keys, aws.ToString(obj.Key)) + } + } + return keys, nil +} - deleteInput := &s3.DeleteObjectsInput{ +// DeleteObjects fails when any key fails, even within a batch S3 accepted. +func (s *s3Storage) DeleteObjects(ctx context.Context, storagePaths []string) error { + client := s.getClient(nil) + for i := 0; i < len(storagePaths); i += 1000 { + batch := storagePaths[i:min(i+1000, len(storagePaths))] + objects := make([]types.ObjectIdentifier, 0, len(batch)) + for _, p := range batch { + objects = append(objects, types.ObjectIdentifier{Key: aws.String(p)}) + } + input := &s3.DeleteObjectsInput{ Bucket: aws.String(s.conf.Bucket), - Delete: &types.Delete{ - Objects: objects, - Quiet: aws.Bool(true), - }, + Delete: &types.Delete{Objects: objects, Quiet: aws.Bool(true)}, } if s.conf.Endpoint != "" { - // SHA256 is supported by most S3-compatible backends - deleteInput.ChecksumAlgorithm = types.ChecksumAlgorithmSha256 + input.ChecksumAlgorithm = types.ChecksumAlgorithmSha256 } - - _, err := client.DeleteObjects(context.Background(), deleteInput) + out, err := client.DeleteObjects(ctx, input) if err != nil { return wrapS3Error(err) } + if len(out.Errors) > 0 { + e := out.Errors[0] + return fmt.Errorf("storage: %d of %d deletes failed, first %s: %s %s", + len(out.Errors), len(batch), aws.ToString(e.Key), aws.ToString(e.Code), aws.ToString(e.Message)) + } } - return nil } + +func (s *s3Storage) GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { + res, err := s3.NewPresignClient(s.getClient(nil)).PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.conf.Bucket), + Key: aws.String(storagePath), + }, s3.WithPresignExpires(expiration)) + if err != nil { + return "", wrapS3Error(err) + } + return res.URL, nil +} + +// wrapS3ObjectError is wrapS3Error with ErrObjectNotFound and ErrObjectExists +// for the codes S3-compatible stores answer them with. +func wrapS3ObjectError(err error) error { + if err == nil { + return nil + } + var noSuchKey *types.NoSuchKey + var notFound *types.NotFound + if errors.As(err, &noSuchKey) || errors.As(err, ¬Found) { + return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapS3Error(err)) + } + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch apiErr.ErrorCode() { + case "NoSuchKey", "NotFound", "404": + return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapS3Error(err)) + case "PreconditionFailed": + return fmt.Errorf("%w: %w", ErrObjectExists, wrapS3Error(err)) + } + } + return wrapS3Error(err) +} diff --git a/s3_object.go b/s3_object.go deleted file mode 100644 index 73db50d..0000000 --- a/s3_object.go +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/feature/s3/manager" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/smithy-go" - "github.com/aws/smithy-go/middleware" -) - -const s3UploadPartSize = 16 << 20 - -type s3Objects struct { - s *s3Storage -} - -func newS3Objects(conf *S3Config) (ObjectStorage, error) { - s, err := NewS3(conf) - if err != nil { - return nil, err - } - return &s3Objects{s: s.(*s3Storage)}, nil -} - -func (o *s3Objects) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { - input := o.putInput(storagePath, bytes.NewReader(data), opts) - if opts.IfAbsent { - input.IfNoneMatch = aws.String("*") - } - _, err := o.s.getClient(nil).PutObject(ctx, input) - return wrapS3ObjectError(err) -} - -// UploadObject checks IfAbsent before uploading, which spares a refused upload -// its traffic, and again when it completes. -func (o *s3Objects) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { - if opts.IfAbsent { - exists, err := o.ObjectExists(ctx, storagePath) - if err != nil { - return err - } - if exists { - return ErrObjectExists - } - } - input := o.putInput(storagePath, body, opts) - uploader := o.s.uploader(o.s.getClient(nil), func(u *manager.Uploader) { - u.PartSize = s3UploadPartSize - if opts.IfAbsent { - // A body that fits one part goes up as this PutObject; a larger one - // is completed with the same condition, which the uploader drops. - input.IfNoneMatch = aws.String("*") - u.ClientOptions = append(u.ClientOptions, completeIfAbsent) - } - }) - _, err := uploader.Upload(ctx, input) - return wrapS3ObjectError(err) -} - -func completeIfAbsent(o *s3.Options) { - o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { - return stack.Initialize.Add(middleware.InitializeMiddlewareFunc("CompleteIfAbsent", - func(ctx context.Context, in middleware.InitializeInput, next middleware.InitializeHandler) ( - middleware.InitializeOutput, middleware.Metadata, error, - ) { - if v, ok := in.Parameters.(*s3.CompleteMultipartUploadInput); ok { - v.IfNoneMatch = aws.String("*") - } - return next.HandleInitialize(ctx, in) - }, - ), middleware.Before) - }) -} - -func (o *s3Objects) putInput(storagePath string, body io.Reader, opts PutOptions) *s3.PutObjectInput { - input := &s3.PutObjectInput{ - Bucket: aws.String(o.s.conf.Bucket), - Key: aws.String(storagePath), - Body: body, - } - if opts.ContentType != "" { - input.ContentType = aws.String(opts.ContentType) - } - return input -} - -func (o *s3Objects) GetObject(ctx context.Context, storagePath string) ([]byte, error) { - return getObject(ctx, o, storagePath) -} - -func (o *s3Objects) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { - if offset < 0 || length <= 0 { - return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) - } - out, err := o.s.getClient(nil).GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(o.s.conf.Bucket), - Key: aws.String(storagePath), - Range: aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)), - }) - var apiErr smithy.APIError - if errors.As(err, &apiErr) && apiErr.ErrorCode() == "InvalidRange" { - return []byte{}, nil - } - if err != nil { - return nil, wrapS3ObjectError(err) - } - defer out.Body.Close() - return io.ReadAll(io.LimitReader(out.Body, length)) -} - -func (o *s3Objects) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { - out, err := o.s.getClient(nil).GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(o.s.conf.Bucket), - Key: aws.String(storagePath), - }) - if err != nil { - return 0, wrapS3ObjectError(err) - } - defer out.Body.Close() - return io.Copy(w, out.Body) -} - -func (o *s3Objects) ObjectExists(ctx context.Context, storagePath string) (bool, error) { - _, err := o.s.getClient(nil).HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(o.s.conf.Bucket), - Key: aws.String(storagePath), - }) - err = wrapS3ObjectError(err) - if errors.Is(err, ErrObjectNotFound) { - return false, nil - } - return err == nil, err -} - -func (o *s3Objects) ListObjects(ctx context.Context, prefix string) ([]string, error) { - var keys []string - paginator := s3.NewListObjectsV2Paginator(o.s.getClient(nil), &s3.ListObjectsV2Input{ - Bucket: aws.String(o.s.conf.Bucket), - Prefix: aws.String(prefix), - }) - for paginator.HasMorePages() { - page, err := paginator.NextPage(ctx) - if err != nil { - return nil, wrapS3Error(err) - } - for _, obj := range page.Contents { - keys = append(keys, aws.ToString(obj.Key)) - } - } - return keys, nil -} - -// DeleteObjects fails when any key fails, even within a batch S3 accepted. -func (o *s3Objects) DeleteObjects(ctx context.Context, storagePaths []string) error { - client := o.s.getClient(nil) - for i := 0; i < len(storagePaths); i += 1000 { - batch := storagePaths[i:min(i+1000, len(storagePaths))] - objects := make([]types.ObjectIdentifier, 0, len(batch)) - for _, p := range batch { - objects = append(objects, types.ObjectIdentifier{Key: aws.String(p)}) - } - input := &s3.DeleteObjectsInput{ - Bucket: aws.String(o.s.conf.Bucket), - Delete: &types.Delete{Objects: objects, Quiet: aws.Bool(true)}, - } - if o.s.conf.Endpoint != "" { - input.ChecksumAlgorithm = types.ChecksumAlgorithmSha256 - } - out, err := client.DeleteObjects(ctx, input) - if err != nil { - return wrapS3Error(err) - } - if len(out.Errors) > 0 { - e := out.Errors[0] - return fmt.Errorf("storage: %d of %d deletes failed, first %s: %s %s", - len(out.Errors), len(batch), aws.ToString(e.Key), aws.ToString(e.Code), aws.ToString(e.Message)) - } - } - return nil -} - -func (o *s3Objects) GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { - res, err := s3.NewPresignClient(o.s.getClient(nil)).PresignGetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(o.s.conf.Bucket), - Key: aws.String(storagePath), - }, s3.WithPresignExpires(expiration)) - if err != nil { - return "", wrapS3Error(err) - } - return res.URL, nil -} - -// wrapS3ObjectError is wrapS3Error with ErrObjectNotFound and ErrObjectExists -// for the codes S3-compatible stores answer them with. -func wrapS3ObjectError(err error) error { - if err == nil { - return nil - } - var noSuchKey *types.NoSuchKey - var notFound *types.NotFound - if errors.As(err, &noSuchKey) || errors.As(err, ¬Found) { - return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapS3Error(err)) - } - var apiErr smithy.APIError - if errors.As(err, &apiErr) { - switch apiErr.ErrorCode() { - case "NoSuchKey", "NotFound", "404": - return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapS3Error(err)) - case "PreconditionFailed": - return fmt.Errorf("%w: %w", ErrObjectExists, wrapS3Error(err)) - } - } - return wrapS3Error(err) -} diff --git a/storage.go b/storage.go index 0317964..84b0bca 100644 --- a/storage.go +++ b/storage.go @@ -14,23 +14,34 @@ package storage -import "time" +import ( + "bytes" + "context" -type Storage interface { - UploadData(data []byte, storagePath, contentType string) (location string, size int64, err error) - UploadFile(filepath, storagePath, contentType string) (location string, size int64, err error) + "github.com/livekit/storage/objects" +) - ListObjects(prefix string) ([]string, error) +// The interface lives in storage/objects so that code implementing or +// accepting it without creating one doesn't pull in every provider SDK. +type ( + Storage = objects.Storage + PutOptions = objects.PutOptions +) - DownloadData(storagePath string) (data []byte, err error) - DownloadFile(filepath, storagePath string) (size int64, err error) - - GeneratePresignedUrl(storagePath string, expiration time.Duration) (url string, err error) - - DeleteObject(storagePath string) error - DeleteObjects(storagePaths []string) error -} +var ( + ErrObjectNotFound = objects.ErrNotFound + ErrObjectExists = objects.ErrExists +) func New(conf Config) (Storage, error) { return newStorage(conf) } + +// getObject is GetObject for a backend whose reads are streams. +func getObject(ctx context.Context, s Storage, storagePath string) ([]byte, error) { + var buf bytes.Buffer + if _, err := s.DownloadObject(ctx, storagePath, &buf); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/storage_test.go b/storage_test.go index 0632202..5236f69 100644 --- a/storage_test.go +++ b/storage_test.go @@ -16,6 +16,7 @@ package storage_test import ( "bytes" + "context" "fmt" "io" "net/http" @@ -50,6 +51,7 @@ func TestAliOSS(t *testing.T) { require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func TestAzure(t *testing.T) { @@ -69,6 +71,7 @@ func TestAzure(t *testing.T) { require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func TestGCP(t *testing.T) { @@ -86,6 +89,7 @@ func TestGCP(t *testing.T) { require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func TestLocal(t *testing.T) { @@ -93,6 +97,7 @@ func TestLocal(t *testing.T) { require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func TestOCI(t *testing.T) { @@ -117,6 +122,7 @@ func TestOCI(t *testing.T) { require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func TestOCINative(t *testing.T) { @@ -145,6 +151,7 @@ func TestOCINative(t *testing.T) { require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func TestSupabase(t *testing.T) { @@ -169,6 +176,7 @@ func TestSupabase(t *testing.T) { require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func TestS3(t *testing.T) { @@ -180,16 +188,21 @@ func TestS3(t *testing.T) { t.Skip("Missing env vars") } + // S3_ENDPOINT reaches an S3-compatible store instead of AWS. + endpoint := os.Getenv("S3_ENDPOINT") s, err := storage.New(&storage.S3Config{ - AccessKey: key, - Secret: secret, - SessionToken: os.Getenv("AWS_SESSION_TOKEN"), - Region: os.Getenv("AWS_REGION"), - Bucket: bucket, + AccessKey: key, + Secret: secret, + SessionToken: os.Getenv("AWS_SESSION_TOKEN"), + Region: os.Getenv("AWS_REGION"), + Endpoint: endpoint, + ForcePathStyle: endpoint != "", + Bucket: bucket, }) require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func testStorage(t *testing.T, s storage.Storage) { @@ -206,7 +219,7 @@ func testStorage(t *testing.T, s storage.Storage) { // best-effort cleanup so a mid-test failure doesn't leave orphans in the bucket t.Cleanup(func() { - _ = s.DeleteObjects([]string{pathData, pathFile, pathBulk1, pathBulk2}) + _ = s.DeleteObjects(context.Background(), []string{pathData, pathFile, pathBulk1, pathBulk2}) }) t.Run("UploadData", func(t *testing.T) { @@ -232,7 +245,7 @@ func testStorage(t *testing.T, s storage.Storage) { }) t.Run("ListObjects", func(t *testing.T) { - items, err := s.ListObjects(prefix) + items, err := s.ListObjects(context.Background(), prefix) require.NoError(t, err) require.Len(t, items, 2) @@ -262,7 +275,7 @@ func testStorage(t *testing.T, s storage.Storage) { }) t.Run("GeneratePresignedUrl", func(t *testing.T) { - rawURL, err := s.GeneratePresignedUrl(pathData, 5*time.Minute) + rawURL, err := s.GeneratePresignedUrl(context.Background(), pathData, 5*time.Minute) if err != nil { // Some configurations (e.g. Azure without OAuth) don't support presigning; // the method exists and reported a clear error, which is the contract. @@ -293,7 +306,7 @@ func testStorage(t *testing.T, s storage.Storage) { t.Run("DeleteObject", func(t *testing.T) { require.NoError(t, s.DeleteObject(pathData)) - items, err := s.ListObjects(prefix) + items, err := s.ListObjects(context.Background(), prefix) require.NoError(t, err) require.Len(t, items, 1) require.True(t, strings.HasSuffix(items[0], pathFile)) @@ -306,9 +319,9 @@ func testStorage(t *testing.T, s storage.Storage) { _, _, err = s.UploadData(bulk2Payload, pathBulk2, "text/plain") require.NoError(t, err) - require.NoError(t, s.DeleteObjects([]string{pathFile, pathBulk1, pathBulk2})) + require.NoError(t, s.DeleteObjects(context.Background(), []string{pathFile, pathBulk1, pathBulk2})) - items, err := s.ListObjects(prefix) + items, err := s.ListObjects(context.Background(), prefix) require.NoError(t, err) require.Empty(t, items) }) From 8970c754c48c643997f9f74fd006061aafdefd67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 00:46:23 -0700 Subject: [PATCH 07/29] Storage embeds Bucket; one error rule, listing order and range check objects.Bucket holds the context calls, so a fake or a consumer that only manages objects takes the small interface; Storage is Bucket plus the file calls. DownloadData and DownloadFile report ErrObjectNotFound too, ListObjects is in byte order on every backend (local sorts), and S3 GetObjectRange refuses an answer without Content-Range. The doc comments state IfAbsent's limits, UploadObject's memory per backend, the range arguments, and when DeleteObjects is safe to retry. --- alioss.go | 4 +-- azure.go | 6 ++--- gcp.go | 2 +- local.go | 11 +++++--- object_test.go | 4 +-- objects/objects.go | 62 ++++++++++++++++++++++++++++++---------------- oci.go | 2 +- s3.go | 6 ++++- storage.go | 5 ++-- 9 files changed, 64 insertions(+), 38 deletions(-) diff --git a/alioss.go b/alioss.go index 71b3f02..039e2f9 100644 --- a/alioss.go +++ b/alioss.go @@ -104,7 +104,7 @@ func (s *aliOSSStorage) location(storagePath string) string { func (s *aliOSSStorage) DownloadData(storagePath string) ([]byte, error) { reader, err := s.bucket.GetObject(storagePath) if err != nil { - return nil, wrapAliOSSError(err) + return nil, wrapAliOSSObjectError(err) } defer reader.Close() @@ -117,7 +117,7 @@ func (s *aliOSSStorage) DownloadData(storagePath string) ([]byte, error) { func (s *aliOSSStorage) DownloadFile(filepath, storagePath string) (int64, error) { if err := s.bucket.GetObjectToFile(storagePath, filepath); err != nil { - return 0, wrapAliOSSError(err) + return 0, wrapAliOSSObjectError(err) } info, err := os.Stat(filepath) diff --git a/azure.go b/azure.go index 543c5c6..592bc6e 100644 --- a/azure.go +++ b/azure.go @@ -128,7 +128,7 @@ func (s *azureBLOBStorage) DownloadData(storagePath string) ([]byte, error) { props, err := blobClient.GetProperties(ctx, nil) if err != nil { - return nil, wrapAzureError(err) + return nil, wrapAzureObjectError(err) } if props.ContentLength == nil { return nil, errors.New("azure: missing content length") @@ -140,7 +140,7 @@ func (s *azureBLOBStorage) DownloadData(storagePath string) ([]byte, error) { Concurrency: azureParallelism, }) if err != nil { - return nil, wrapAzureError(err) + return nil, wrapAzureObjectError(err) } return buf, nil } @@ -157,7 +157,7 @@ func (s *azureBLOBStorage) DownloadFile(filepath, storagePath string) (int64, er Concurrency: azureParallelism, }) if err != nil { - return 0, wrapAzureError(err) + return 0, wrapAzureObjectError(err) } stat, err := file.Stat() diff --git a/gcp.go b/gcp.go index 4b1bf34..9ff423d 100644 --- a/gcp.go +++ b/gcp.go @@ -196,7 +196,7 @@ func (s *gcpStorage) download(storagePath string) (*storage.Reader, error) { storage.WithPolicy(storage.RetryAlways), ).NewReader(context.Background()) if err != nil { - return nil, wrapGCPError(err) + return nil, wrapGCPObjectError(err) } return r, nil } diff --git a/local.go b/local.go index f975265..a147fdb 100644 --- a/local.go +++ b/local.go @@ -149,7 +149,8 @@ func (u *localUploader) listObjects(prefix string) ([]string, error) { } func (u *localUploader) DownloadData(storagePath string) ([]byte, error) { - return os.ReadFile(path.Join(u.StorageDir, storagePath)) + data, err := os.ReadFile(path.Join(u.StorageDir, storagePath)) + return data, localObjectError(err) } func (u *localUploader) DownloadFile(localPath, storagePath string) (int64, error) { @@ -157,7 +158,7 @@ func (u *localUploader) DownloadFile(localPath, storagePath string) (int64, erro storage, err := os.Open(storagePath) if err != nil { - return 0, err + return 0, localObjectError(err) } defer storage.Close() @@ -284,9 +285,11 @@ func (u *localUploader) ObjectExists(_ context.Context, storagePath string) (boo // a crash. func (u *localUploader) ListObjects(_ context.Context, prefix string) ([]string, error) { keys, err := u.listObjects(prefix) - return slices.DeleteFunc(keys, func(k string) bool { + keys = slices.DeleteFunc(keys, func(k string) bool { return strings.HasPrefix(filepath.Base(k), localUploadPrefix) - }), err + }) + slices.Sort(keys) + return keys, err } func (u *localUploader) DeleteObjects(_ context.Context, storagePaths []string) error { diff --git a/object_test.go b/object_test.go index f401bf3..7040ed5 100644 --- a/object_test.go +++ b/object_test.go @@ -18,7 +18,6 @@ import ( "bytes" "context" "fmt" - "sort" "testing" "time" @@ -78,6 +77,8 @@ func testObjects(t *testing.T, s storage.Storage) { require.ErrorIs(t, err, storage.ErrObjectNotFound) _, err = s.GetObjectRange(ctx, missing, 0, 1) require.ErrorIs(t, err, storage.ErrObjectNotFound) + _, err = s.DownloadData(missing) + require.ErrorIs(t, err, storage.ErrObjectNotFound) exists, err := s.ObjectExists(ctx, missing) require.NoError(t, err) require.False(t, exists) @@ -89,7 +90,6 @@ func testObjects(t *testing.T, s storage.Storage) { t.Run("ListObjects and DeleteObjects", func(t *testing.T) { keys, err := s.ListObjects(ctx, prefix) require.NoError(t, err) - sort.Strings(keys) require.Equal(t, []string{large, small}, keys) require.NoError(t, s.DeleteObjects(ctx, []string{small, large, missing})) keys, err = s.ListObjects(ctx, prefix) diff --git a/objects/objects.go b/objects/objects.go index db20142..9495db3 100644 --- a/objects/objects.go +++ b/objects/objects.go @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package objects is the Storage interface without any provider SDK, for code -// that implements or accepts it without creating one, the way config holds the +// Package objects is the storage interfaces without any provider SDK, for code +// that implements or accepts them without creating one, such as a fake: the +// root package imports every provider's SDK. config does the same for the // config structs. package objects @@ -24,43 +25,60 @@ import ( "time" ) -type Storage interface { - // UploadData and UploadFile replace any object at storagePath and return - // its location. - UploadData(data []byte, storagePath, contentType string) (location string, size int64, err error) - UploadFile(filepath, storagePath, contentType string) (location string, size int64, err error) - - DownloadData(storagePath string) (data []byte, err error) - DownloadFile(filepath, storagePath string) (size int64, err error) - DeleteObject(storagePath string) error - - // The calls below take a context, and report a missing or existing object as - // ErrNotFound or ErrExists. - +// Bucket is object storage for a service that keeps and manages its own +// objects. A read of a missing object reports ErrNotFound and a refused write +// ErrExists; test them with errors.Is. +type Bucket interface { + // PutObject sends data in one request, which the provider's SDK can retry. PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error - // UploadObject streams body in parts: memory is bounded by the part size - // times the upload concurrency, whatever the object's size. + // UploadObject streams body in parts and does not rewind it, so a retry + // needs a new reader. Memory is the part size times the upload concurrency: + // 16 MiB x 5 on S3, 4 MiB x 16 on Azure, one 16 MiB part on GCS and OCI, + // none on AliOSS (one chunked request, up to 5 GB) and local. UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error GetObject(ctx context.Context, storagePath string) ([]byte, error) - // GetObjectRange reads length bytes from offset. A range running past the - // end returns what the object holds, one starting at or past it no bytes. + // GetObjectRange reads length bytes from offset, with offset >= 0 and + // length > 0. A range running past the end returns what the object holds, + // one starting at or past it no bytes. GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) + // DownloadObject streams the object to w, which holds part of it when an + // error comes back. DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) + // ObjectExists reports a missing object as false, not ErrNotFound. ObjectExists(ctx context.Context, storagePath string) (bool, error) + // ListObjects returns every key under prefix, recursively, as full paths, + // in byte order. ListObjects(ctx context.Context, prefix string) ([]string, error) // DeleteObjects succeeds for paths that hold nothing and fails when a path - // is not deleted. + // is not deleted; some may be deleted by then, and a retry is safe. GCS, + // Azure and OCI send one request per path. DeleteObjects(ctx context.Context, storagePaths []string) error + // GeneratePresignedUrl is a GET URL; local returns a file:// URL. GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) } +// Storage is Bucket and the calls that upload and fetch whole files and return +// their location. +type Storage interface { + Bucket + + // UploadData and UploadFile replace any object at storagePath. + UploadData(data []byte, storagePath, contentType string) (location string, size int64, err error) + UploadFile(filepath, storagePath, contentType string) (location string, size int64, err error) + DownloadData(storagePath string) (data []byte, err error) + DownloadFile(filepath, storagePath string) (size int64, err error) + DeleteObject(storagePath string) error +} + type PutOptions struct { ContentType string - // IfAbsent fails with ErrExists when the path holds an object, atomically: - // of concurrent writes to one path, one succeeds. + // IfAbsent fails with ErrExists when the path holds an object, atomically + // where the store honors the condition: an S3-compatible store that ignores + // If-None-Match overwrites, and AliOSS ignores it on a versioned bucket. A + // retry after a lost response can see the caller's own write as existing. IfAbsent bool } diff --git a/oci.go b/oci.go index f1994f0..41d6c37 100644 --- a/oci.go +++ b/oci.go @@ -219,7 +219,7 @@ func (s *ociStorage) download(storagePath string) (objectstorage.GetObjectRespon BucketName: &s.conf.Bucket, ObjectName: &storagePath, }) - return resp, wrapOCIError(err) + return resp, wrapOCIObjectError(err) } // GeneratePresignedUrl creates a pre-authenticated request. Unlike an S3 diff --git a/s3.go b/s3.go index 5e74b9b..5362b9d 100644 --- a/s3.go +++ b/s3.go @@ -331,7 +331,7 @@ func (s *s3Storage) download(w io.WriterAt, storagePath string) (int64, error) { Key: aws.String(storagePath), }, ) - return n, wrapS3Error(err) + return n, wrapS3ObjectError(err) } func (s *s3Storage) DeleteObject(storagePath string) error { @@ -429,6 +429,10 @@ func (s *s3Storage) GetObjectRange(ctx context.Context, storagePath string, offs return nil, wrapS3ObjectError(err) } defer out.Body.Close() + // A store that ignores Range answers with the whole object. + if out.ContentRange == nil { + return nil, fmt.Errorf("storage: %s answered without Content-Range", storagePath) + } return io.ReadAll(io.LimitReader(out.Body, length)) } diff --git a/storage.go b/storage.go index 84b0bca..6339471 100644 --- a/storage.go +++ b/storage.go @@ -21,10 +21,11 @@ import ( "github.com/livekit/storage/objects" ) -// The interface lives in storage/objects so that code implementing or -// accepting it without creating one doesn't pull in every provider SDK. +// The interfaces live in storage/objects so that code implementing or +// accepting them without creating one doesn't pull in every provider SDK. type ( Storage = objects.Storage + Bucket = objects.Bucket PutOptions = objects.PutOptions ) From 0137431c1cb05855dd090b4dfd4c76298b087403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 00:58:11 -0700 Subject: [PATCH 08/29] Storage: every call takes a context; DownloadData and DeleteObject removed UploadData, UploadFile and DownloadFile take a context. DownloadData and DeleteObject duplicated GetObject and DeleteObjects and are gone. The file calls go through the object calls, so S3's config metadata, tagging and content disposition apply to every write. OCI has one client: the SDK's 60s whole-request timeout gives way to dial, TLS and response header timeouts, which S3 and GCS clients get too. --- README.md | 10 ++-- alioss.go | 38 +++--------- azure.go | 55 ++++-------------- error_test.go | 2 +- gcp.go | 106 ++++++++-------------------------- local.go | 58 +++++-------------- object_test.go | 3 +- objects/objects.go | 14 ++--- oci.go | 141 ++++++++++----------------------------------- s3.go | 103 +++++++++------------------------ storage.go | 24 +++++++- storage_test.go | 22 +++---- 12 files changed, 164 insertions(+), 412 deletions(-) diff --git a/README.md b/README.md index 875cae3..105fceb 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ A GORM-style storage wrapper with unified API for different cloud storage backends -`storage.New(conf)` returns a `Storage` for S3 and S3-compatible stores, GCS, Azure, OCI, AliOSS or a local directory. +`storage.New(conf)` returns a `Storage` for S3 and S3-compatible stores, GCS, Azure, OCI, AliOSS or a local directory. Every call takes a context. -- `UploadData`, `UploadFile`, `DownloadData`, `DownloadFile` and `DeleteObject` upload and fetch whole files and return their location. -- Every other call takes a context and reports a missing or existing object as `ErrObjectNotFound` or `ErrObjectExists` (still wrapping the status code): `PutObject` and `UploadObject` can refuse to replace an object (`PutOptions.IfAbsent`, atomic), `GetObjectRange` and `DownloadObject` read a range or stream to any `io.Writer`, `DeleteObjects` succeeds for paths that hold nothing and fails on any key it could not delete. - -The interface is also in `storage/objects`, which imports no provider SDK, for fakes and libraries that implement or accept it without creating one. +- `Bucket` (`storage/objects`): `PutObject`, `UploadObject` (can refuse to replace an object, `PutOptions.IfAbsent`), `GetObject`, `GetObjectRange`, `DownloadObject` (to any `io.Writer`), `ObjectExists`, `ListObjects`, `DeleteObjects`, `GeneratePresignedUrl`. A missing object is `ErrObjectNotFound` and a refused write `ErrObjectExists` (`errors.Is`). +- `Storage` is `Bucket` plus `UploadData`, `UploadFile` and `DownloadFile`, which publish and fetch whole files and return their location. +- `storage/objects` imports no provider SDK, so a fake or a library can implement or accept the interfaces without pulling in every SDK. +- The clients this package builds time out a connection that stalls before answering, never a transfer in progress; the context bounds the call. diff --git a/alioss.go b/alioss.go index 039e2f9..453f789 100644 --- a/alioss.go +++ b/alioss.go @@ -72,25 +72,21 @@ func NewAliOSS(conf *AliOSSConfig) (Storage, error) { }, nil } -func (s *aliOSSStorage) UploadData(data []byte, storagePath, contentType string) (string, int64, error) { - reader := bytes.NewBuffer(data) - if err := s.bucket.PutObject(storagePath, reader, oss.ContentType(contentType)); err != nil { - return "", 0, wrapAliOSSError(err) +func (s *aliOSSStorage) UploadData(ctx context.Context, data []byte, storagePath, contentType string) (string, int64, error) { + if err := s.PutObject(ctx, storagePath, data, PutOptions{ContentType: contentType}); err != nil { + return "", 0, err } - return s.location(storagePath), int64(len(data)), nil } -func (s *aliOSSStorage) UploadFile(filepath, storagePath, contentType string) (string, int64, error) { +func (s *aliOSSStorage) UploadFile(ctx context.Context, filepath, storagePath, contentType string) (string, int64, error) { info, err := os.Stat(filepath) if err != nil { return "", 0, err } - - if err = s.bucket.PutObjectFromFile(storagePath, filepath, oss.ContentType(contentType)); err != nil { + if err = s.bucket.PutObjectFromFile(storagePath, filepath, oss.ContentType(contentType), oss.WithContext(ctx)); err != nil { return "", 0, wrapAliOSSError(err) } - return s.location(storagePath), info.Size(), nil } @@ -101,30 +97,14 @@ func (s *aliOSSStorage) location(storagePath string) string { return u.String() } -func (s *aliOSSStorage) DownloadData(storagePath string) ([]byte, error) { - reader, err := s.bucket.GetObject(storagePath) - if err != nil { - return nil, wrapAliOSSObjectError(err) - } - defer reader.Close() - - data, err := io.ReadAll(reader) - if err != nil { - return nil, wrapAliOSSError(err) - } - return data, nil -} - -func (s *aliOSSStorage) DownloadFile(filepath, storagePath string) (int64, error) { - if err := s.bucket.GetObjectToFile(storagePath, filepath); err != nil { +func (s *aliOSSStorage) DownloadFile(ctx context.Context, filepath, storagePath string) (int64, error) { + if err := s.bucket.GetObjectToFile(storagePath, filepath, oss.WithContext(ctx)); err != nil { return 0, wrapAliOSSObjectError(err) } - info, err := os.Stat(filepath) if err != nil { return 0, err } - return info.Size(), nil } @@ -136,10 +116,6 @@ func (s *aliOSSStorage) presignedUrl(storagePath string, expiration time.Duratio return u, nil } -func (s *aliOSSStorage) DeleteObject(storagePath string) error { - return wrapAliOSSError(s.bucket.DeleteObject(storagePath)) -} - func (s *aliOSSStorage) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { return s.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) } diff --git a/azure.go b/azure.go index 592bc6e..caa5aed 100644 --- a/azure.go +++ b/azure.go @@ -84,19 +84,14 @@ func (s *azureBLOBStorage) location(storagePath string) string { }).String() } -func (s *azureBLOBStorage) UploadData(data []byte, storagePath, contentType string) (string, int64, error) { - _, err := s.client.UploadBuffer(context.Background(), s.conf.ContainerName, storagePath, data, &azblob.UploadBufferOptions{ - HTTPHeaders: &blob.HTTPHeaders{BlobContentType: &contentType}, - BlockSize: azureBlockSize, - Concurrency: azureParallelism, - }) - if err != nil { - return "", 0, wrapAzureError(err) +func (s *azureBLOBStorage) UploadData(ctx context.Context, data []byte, storagePath, contentType string) (string, int64, error) { + if err := s.PutObject(ctx, storagePath, data, PutOptions{ContentType: contentType}); err != nil { + return "", 0, err } return s.location(storagePath), int64(len(data)), nil } -func (s *azureBLOBStorage) UploadFile(filepath, storagePath, contentType string) (string, int64, error) { +func (s *azureBLOBStorage) UploadFile(ctx context.Context, filepath, storagePath, contentType string) (string, int64, error) { file, err := os.Open(filepath) if err != nil { return "", 0, err @@ -110,7 +105,7 @@ func (s *azureBLOBStorage) UploadFile(filepath, storagePath, contentType string) return "", 0, err } - _, err = s.client.UploadFile(context.Background(), s.conf.ContainerName, storagePath, file, &azblob.UploadFileOptions{ + _, err = s.client.UploadFile(ctx, s.conf.ContainerName, storagePath, file, &azblob.UploadFileOptions{ HTTPHeaders: &blob.HTTPHeaders{BlobContentType: &contentType}, BlockSize: azureBlockSize, Concurrency: azureParallelism, @@ -122,37 +117,14 @@ func (s *azureBLOBStorage) UploadFile(filepath, storagePath, contentType string) return s.location(storagePath), stat.Size(), nil } -func (s *azureBLOBStorage) DownloadData(storagePath string) ([]byte, error) { - ctx := context.Background() - blobClient := s.client.ServiceClient().NewContainerClient(s.conf.ContainerName).NewBlobClient(storagePath) - - props, err := blobClient.GetProperties(ctx, nil) - if err != nil { - return nil, wrapAzureObjectError(err) - } - if props.ContentLength == nil { - return nil, errors.New("azure: missing content length") - } - - buf := make([]byte, *props.ContentLength) - _, err = s.client.DownloadBuffer(ctx, s.conf.ContainerName, storagePath, buf, &azblob.DownloadBufferOptions{ - BlockSize: azureBlockSize, - Concurrency: azureParallelism, - }) - if err != nil { - return nil, wrapAzureObjectError(err) - } - return buf, nil -} - -func (s *azureBLOBStorage) DownloadFile(filepath, storagePath string) (int64, error) { +func (s *azureBLOBStorage) DownloadFile(ctx context.Context, filepath, storagePath string) (int64, error) { file, err := os.Create(filepath) if err != nil { return 0, err } defer file.Close() - _, err = s.client.DownloadFile(context.Background(), s.conf.ContainerName, storagePath, file, &azblob.DownloadFileOptions{ + _, err = s.client.DownloadFile(ctx, s.conf.ContainerName, storagePath, file, &azblob.DownloadFileOptions{ BlockSize: azureBlockSize, Concurrency: azureParallelism, }) @@ -167,7 +139,7 @@ func (s *azureBLOBStorage) DownloadFile(filepath, storagePath string) (int64, er return stat.Size(), nil } -func (s *azureBLOBStorage) presignedUrlWithToken(storagePath string, expiration time.Duration) (string, error) { +func (s *azureBLOBStorage) presignedUrlWithToken(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { if s.conf.TokenCredential == nil { return "", errors.New("OAuth required") } @@ -180,7 +152,7 @@ func (s *azureBLOBStorage) presignedUrlWithToken(storagePath string, expiration return "", err } - udc, err := svcClient.GetUserDelegationCredential(context.Background(), service.KeyInfo{ + udc, err := svcClient.GetUserDelegationCredential(ctx, service.KeyInfo{ Start: to.Ptr(now.Format(sas.TimeFormat)), Expiry: to.Ptr(exp.Format(sas.TimeFormat)), }, nil) @@ -209,11 +181,6 @@ func (s *azureBLOBStorage) presignedUrlWithToken(storagePath string, expiration return loc.String(), nil } -func (s *azureBLOBStorage) DeleteObject(storagePath string) error { - _, err := s.client.DeleteBlob(context.Background(), s.conf.ContainerName, storagePath, nil) - return wrapAzureError(err) -} - func (s *azureBLOBStorage) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { return s.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) } @@ -320,9 +287,9 @@ func (s *azureBLOBStorage) DeleteObjects(ctx context.Context, storagePaths []str // GeneratePresignedUrl signs with the account key unless a token credential is // set, in which case it signs with a user delegation key as Storage does. -func (s *azureBLOBStorage) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { +func (s *azureBLOBStorage) GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { if s.conf.TokenCredential != nil { - return s.presignedUrlWithToken(storagePath, expiration) + return s.presignedUrlWithToken(ctx, storagePath, expiration) } b := s.client.ServiceClient().NewContainerClient(s.conf.ContainerName).NewBlobClient(storagePath) u, err := b.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(expiration), nil) diff --git a/error_test.go b/error_test.go index 3f0966b..a235772 100644 --- a/error_test.go +++ b/error_test.go @@ -22,9 +22,9 @@ import ( "cloud.google.com/go/storage" "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/aliyun/aliyun-oss-go-sdk/oss" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" smithyhttp "github.com/aws/smithy-go/transport/http" - "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/stretchr/testify/require" "google.golang.org/api/googleapi" ) diff --git a/gcp.go b/gcp.go index 9ff423d..d4066d0 100644 --- a/gcp.go +++ b/gcp.go @@ -76,7 +76,7 @@ func NewGCP(conf *GCPConfig) (Storage, error) { } defaultTransport := http.DefaultTransport.(*http.Transport) - transportClone := defaultTransport.Clone() + transportClone := transportWithTimeouts(defaultTransport) if conf.ProxyConfig != nil { proxyUrl, err := url.Parse(conf.ProxyConfig.Url) @@ -103,102 +103,46 @@ func NewGCP(conf *GCPConfig) (Storage, error) { return u, nil } -func (s *gcpStorage) UploadData(data []byte, storagePath, contentType string) (string, int64, error) { - return s.upload(bytes.NewReader(data), storagePath, contentType) +func (s *gcpStorage) UploadData(ctx context.Context, data []byte, storagePath, contentType string) (string, int64, error) { + if err := s.PutObject(ctx, storagePath, data, PutOptions{ContentType: contentType}); err != nil { + return "", 0, err + } + return s.location(storagePath), int64(len(data)), nil } -func (s *gcpStorage) UploadFile(filepath, storagePath, contentType string) (string, int64, error) { +func (s *gcpStorage) UploadFile(ctx context.Context, filepath, storagePath, contentType string) (string, int64, error) { file, err := os.Open(filepath) if err != nil { return "", 0, err } defer file.Close() - return s.upload(file, storagePath, contentType) -} - -func (s *gcpStorage) upload(reader io.Reader, storagePath, contentType string) (string, int64, error) { - wc := s.client.Bucket(s.conf.Bucket).Object(storagePath).Retryer( - storage.WithBackoff(gax.Backoff{ - Initial: time.Millisecond * 100, - Max: time.Second * 5, - Multiplier: 2, - }), - storage.WithMaxAttempts(5), - storage.WithPolicy(storage.RetryAlways), - ).NewWriter(context.Background()) - wc.ContentType = contentType - wc.ChunkRetryDeadline = 0 - - n, err := io.Copy(wc, reader) + stat, err := file.Stat() if err != nil { - return "", 0, wrapGCPError(err) - } - - if err = wc.Close(); err != nil { - return "", 0, wrapGCPError(err) - } - - return s.location(storagePath), n, nil -} - -func (s *gcpStorage) location(storagePath string) string { - return (&url.URL{ - Scheme: "https", - Host: s.conf.Bucket + ".storage.googleapis.com", - Path: storagePath, - }).String() -} - -func (s *gcpStorage) DownloadData(storagePath string) ([]byte, error) { - rc, err := s.download(storagePath) - if err != nil { - return nil, err + return "", 0, err } - defer rc.Close() - - data, err := io.ReadAll(rc) - if err != nil { - return nil, wrapGCPError(err) + if err = s.UploadObject(ctx, storagePath, file, PutOptions{ContentType: contentType}); err != nil { + return "", 0, err } - return data, nil + return s.location(storagePath), stat.Size(), nil } -func (s *gcpStorage) DownloadFile(filepath, storagePath string) (int64, error) { +func (s *gcpStorage) DownloadFile(ctx context.Context, filepath, storagePath string) (int64, error) { file, err := os.Create(filepath) if err != nil { return 0, err } defer file.Close() - rc, err := s.download(storagePath) - if err != nil { - return 0, err - } - - _, err = io.Copy(file, rc) - _ = rc.Close() - if err != nil { - return 0, wrapGCPError(err) - } - - return rc.Attrs.Size, nil + return s.DownloadObject(ctx, storagePath, file) } -func (s *gcpStorage) download(storagePath string) (*storage.Reader, error) { - r, err := s.client.Bucket(s.conf.Bucket).Object(storagePath).Retryer( - storage.WithBackoff( - gax.Backoff{ - Initial: time.Millisecond * 100, - Max: time.Second * 5, - Multiplier: 2, - }), - storage.WithPolicy(storage.RetryAlways), - ).NewReader(context.Background()) - if err != nil { - return nil, wrapGCPObjectError(err) - } - return r, nil +func (s *gcpStorage) location(storagePath string) string { + return (&url.URL{ + Scheme: "https", + Host: s.conf.Bucket + ".storage.googleapis.com", + Path: storagePath, + }).String() } func (s *gcpStorage) presignedUrl(storagePath string, expiration time.Duration) (string, error) { @@ -212,10 +156,6 @@ func (s *gcpStorage) presignedUrl(storagePath string, expiration time.Duration) return u, nil } -func (s *gcpStorage) DeleteObject(storagePath string) error { - return wrapGCPError(s.client.Bucket(s.conf.Bucket).Object(storagePath).Delete(context.Background())) -} - func (s *gcpStorage) object(storagePath string) *storage.ObjectHandle { return s.client.Bucket(s.conf.Bucket).Object(storagePath) } @@ -238,7 +178,11 @@ func (s *gcpStorage) write(ctx context.Context, storagePath string, body io.Read // Conditional, so the SDK retries it by default. obj = obj.If(storage.Conditions{DoesNotExist: true}) } else { - obj = obj.Retryer(storage.WithPolicy(storage.RetryAlways)) + obj = obj.Retryer( + storage.WithBackoff(gax.Backoff{Initial: 100 * time.Millisecond, Max: 5 * time.Second, Multiplier: 2}), + storage.WithMaxAttempts(5), + storage.WithPolicy(storage.RetryAlways), + ) } ctx, cancel := context.WithCancel(ctx) diff --git a/local.go b/local.go index a147fdb..533f49a 100644 --- a/local.go +++ b/local.go @@ -45,56 +45,28 @@ func NewLocal(conf *LocalConfig) (Storage, error) { }, nil } -func (u *localUploader) UploadFile(localPath, storagePath string, _ string) (string, int64, error) { - storagePath = u.location(storagePath) - - local, err := os.Open(localPath) +func (u *localUploader) UploadFile(ctx context.Context, localPath, storagePath, _ string) (string, int64, error) { + file, err := os.Open(localPath) if err != nil { return "", 0, err } - defer local.Close() - - if dir, _ := path.Split(storagePath); dir != "" { - if err = os.MkdirAll(dir, 0755); err != nil { - return "", 0, err - } - } + defer file.Close() - storage, err := os.Create(storagePath) + stat, err := file.Stat() if err != nil { return "", 0, err } - defer storage.Close() - - size, err := io.Copy(storage, local) - if err != nil { + if err = u.UploadObject(ctx, storagePath, file, PutOptions{}); err != nil { return "", 0, err } - - return storagePath, size, nil + return u.location(storagePath), stat.Size(), nil } -func (u *localUploader) UploadData(data []byte, storagePath, _ string) (string, int64, error) { - storagePath = u.location(storagePath) - - if dir, _ := path.Split(storagePath); dir != "" { - if err := os.MkdirAll(dir, 0755); err != nil { - return "", 0, err - } - } - - storage, err := os.Create(storagePath) - if err != nil { - return "", 0, err - } - defer storage.Close() - - size, err := storage.Write(data) - if err != nil { +func (u *localUploader) UploadData(ctx context.Context, data []byte, storagePath, _ string) (string, int64, error) { + if err := u.PutObject(ctx, storagePath, data, PutOptions{}); err != nil { return "", 0, err } - - return storagePath, int64(size), nil + return u.location(storagePath), int64(len(data)), nil } func (u *localUploader) location(storagePath string) string { @@ -148,12 +120,7 @@ func (u *localUploader) listObjects(prefix string) ([]string, error) { return files, nil } -func (u *localUploader) DownloadData(storagePath string) ([]byte, error) { - data, err := os.ReadFile(path.Join(u.StorageDir, storagePath)) - return data, localObjectError(err) -} - -func (u *localUploader) DownloadFile(localPath, storagePath string) (int64, error) { +func (u *localUploader) DownloadFile(_ context.Context, localPath, storagePath string) (int64, error) { storagePath = path.Join(u.StorageDir, storagePath) storage, err := os.Open(storagePath) @@ -187,7 +154,8 @@ func (u *localUploader) presignedUrl(storagePath string, _ time.Duration) (strin return (&url.URL{Scheme: "file", Path: filepath.ToSlash(abs)}).String(), nil } -func (u *localUploader) DeleteObject(storagePath string) error { +// deleteObject removes the file, then each parent directory it leaves empty. +func (u *localUploader) deleteObject(storagePath string) error { storagePath = path.Join(u.StorageDir, storagePath) for { @@ -294,7 +262,7 @@ func (u *localUploader) ListObjects(_ context.Context, prefix string) ([]string, func (u *localUploader) DeleteObjects(_ context.Context, storagePaths []string) error { for _, p := range storagePaths { - if err := u.DeleteObject(p); err != nil && !errors.Is(err, fs.ErrNotExist) { + if err := u.deleteObject(p); err != nil && !errors.Is(err, fs.ErrNotExist) { return err } } diff --git a/object_test.go b/object_test.go index 7040ed5..3495672 100644 --- a/object_test.go +++ b/object_test.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "fmt" + "path/filepath" "testing" "time" @@ -77,7 +78,7 @@ func testObjects(t *testing.T, s storage.Storage) { require.ErrorIs(t, err, storage.ErrObjectNotFound) _, err = s.GetObjectRange(ctx, missing, 0, 1) require.ErrorIs(t, err, storage.ErrObjectNotFound) - _, err = s.DownloadData(missing) + _, err = s.DownloadFile(ctx, filepath.Join(t.TempDir(), "missing"), missing) require.ErrorIs(t, err, storage.ErrObjectNotFound) exists, err := s.ObjectExists(ctx, missing) require.NoError(t, err) diff --git a/objects/objects.go b/objects/objects.go index 9495db3..bd22a8c 100644 --- a/objects/objects.go +++ b/objects/objects.go @@ -60,17 +60,15 @@ type Bucket interface { GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) } -// Storage is Bucket and the calls that upload and fetch whole files and return -// their location. +// Storage is Bucket and the calls that publish and fetch whole files. type Storage interface { Bucket - // UploadData and UploadFile replace any object at storagePath. - UploadData(data []byte, storagePath, contentType string) (location string, size int64, err error) - UploadFile(filepath, storagePath, contentType string) (location string, size int64, err error) - DownloadData(storagePath string) (data []byte, err error) - DownloadFile(filepath, storagePath string) (size int64, err error) - DeleteObject(storagePath string) error + // UploadData and UploadFile replace any object at storagePath and return + // its location, a URL to it. + UploadData(ctx context.Context, data []byte, storagePath, contentType string) (location string, size int64, err error) + UploadFile(ctx context.Context, filepath, storagePath, contentType string) (location string, size int64, err error) + DownloadFile(ctx context.Context, filepath, storagePath string) (size int64, err error) } type PutOptions struct { diff --git a/oci.go b/oci.go index 41d6c37..c0be9fd 100644 --- a/oci.go +++ b/oci.go @@ -33,12 +33,8 @@ import ( ) type ociStorage struct { - conf *OCIConfig - client objectstorage.ObjectStorageClient - // ctxClient is client without the SDK's 60s request timeout, which covers - // reading a body and would cut a long download; the calls that take a - // context use it, and the context bounds them instead. - ctxClient objectstorage.ObjectStorageClient + conf *OCIConfig + client objectstorage.ObjectStorageClient namespace string host string } @@ -60,6 +56,11 @@ func NewOCI(conf *OCIConfig) (Storage, error) { if conf.Endpoint != "" { client.Host = conf.Endpoint } + // The SDK's 60s http.Client.Timeout covers reading the body and would cut a + // long transfer; a response header timeout catches a dead connection instead. + if hc, ok := client.HTTPClient.(*http.Client); ok { + client.HTTPClient = &http.Client{Transport: transportWithTimeouts(hc.Transport), CheckRedirect: hc.CheckRedirect, Jar: hc.Jar} + } // The SDK only prepends a scheme to client.Host on the first request, so // read it before GetNamespace below. @@ -73,11 +74,6 @@ func NewOCI(conf *OCIConfig) (Storage, error) { host: host, } - s.ctxClient = client - if hc, ok := client.HTTPClient.(*http.Client); ok { - s.ctxClient.HTTPClient = &http.Client{Transport: hc.Transport, CheckRedirect: hc.CheckRedirect, Jar: hc.Jar} - } - if s.namespace == "" { req := objectstorage.GetNamespaceRequest{} if conf.CompartmentID != "" { @@ -116,55 +112,28 @@ func newConfigProvider(conf *OCIConfig) (common.ConfigurationProvider, error) { return common.DefaultConfigProvider(), nil } -func (s *ociStorage) UploadData(data []byte, storagePath, contentType string) (string, int64, error) { - size := int64(len(data)) - body := common.NewOCIReadSeekCloser(io.NopCloser(bytes.NewReader(data))) - - location, err := s.upload(body, size, storagePath, contentType) - if err != nil { +func (s *ociStorage) UploadData(ctx context.Context, data []byte, storagePath, contentType string) (string, int64, error) { + if err := s.PutObject(ctx, storagePath, data, PutOptions{ContentType: contentType}); err != nil { return "", 0, err } - return location, size, nil + return s.location(storagePath), int64(len(data)), nil } -func (s *ociStorage) UploadFile(filepath, storagePath, contentType string) (string, int64, error) { +func (s *ociStorage) UploadFile(ctx context.Context, filepath, storagePath, contentType string) (string, int64, error) { file, err := os.Open(filepath) if err != nil { return "", 0, err } - defer func() { - _ = file.Close() - }() + defer file.Close() stat, err := file.Stat() if err != nil { return "", 0, err } - - location, err := s.upload(file, stat.Size(), storagePath, contentType) - if err != nil { + if err = s.UploadObject(ctx, storagePath, file, PutOptions{ContentType: contentType}); err != nil { return "", 0, err } - return location, stat.Size(), nil -} - -// upload takes a seekable body: the SDK re-reads it when it retries. -func (s *ociStorage) upload(body io.ReadCloser, size int64, storagePath, contentType string) (string, error) { - contentDisposition := "inline" - _, err := s.client.PutObject(context.Background(), objectstorage.PutObjectRequest{ - NamespaceName: &s.namespace, - BucketName: &s.conf.Bucket, - ObjectName: &storagePath, - PutObjectBody: body, - ContentLength: &size, - ContentType: &contentType, - ContentDisposition: &contentDisposition, - }) - if err != nil { - return "", wrapOCIError(err) - } - - return s.location(storagePath), nil + return s.location(storagePath), stat.Size(), nil } func (s *ociStorage) location(storagePath string) string { @@ -175,62 +144,25 @@ func (s *ociStorage) location(storagePath string) string { }).String() } -func (s *ociStorage) DownloadData(storagePath string) ([]byte, error) { - resp, err := s.download(storagePath) - if err != nil { - return nil, err - } - defer func() { - _ = resp.Content.Close() - }() - - data, err := io.ReadAll(resp.Content) - if err != nil { - return nil, wrapOCIError(err) - } - return data, nil -} - -func (s *ociStorage) DownloadFile(filepath, storagePath string) (int64, error) { +func (s *ociStorage) DownloadFile(ctx context.Context, filepath, storagePath string) (int64, error) { file, err := os.Create(filepath) if err != nil { return 0, err } defer file.Close() - resp, err := s.download(storagePath) - if err != nil { - return 0, err - } - defer func() { - _ = resp.Content.Close() - }() - - if _, err = io.Copy(file, resp.Content); err != nil { - return 0, wrapOCIError(err) - } - - return *resp.ContentLength, nil -} - -func (s *ociStorage) download(storagePath string) (objectstorage.GetObjectResponse, error) { - resp, err := s.client.GetObject(context.Background(), objectstorage.GetObjectRequest{ - NamespaceName: &s.namespace, - BucketName: &s.conf.Bucket, - ObjectName: &storagePath, - }) - return resp, wrapOCIObjectError(err) + return s.DownloadObject(ctx, storagePath, file) } // GeneratePresignedUrl creates a pre-authenticated request. Unlike an S3 // presigned URL, a PAR is a durable object: it is listable and revocable, it // costs an API call to mint, and it needs PAR_MANAGE on the bucket. Its access // URI is returned only here and cannot be read back afterwards. -func (s *ociStorage) presignedUrl(storagePath string, expiration time.Duration) (string, error) { +func (s *ociStorage) presignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { name := fmt.Sprintf("livekit-storage-%s-%d", storagePath, time.Now().UnixNano()) expires := common.SDKTime{Time: time.Now().Add(expiration)} - resp, err := s.client.CreatePreauthenticatedRequest(context.Background(), objectstorage.CreatePreauthenticatedRequestRequest{ + resp, err := s.client.CreatePreauthenticatedRequest(ctx, objectstorage.CreatePreauthenticatedRequestRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, CreatePreauthenticatedRequestDetails: objectstorage.CreatePreauthenticatedRequestDetails{ @@ -251,15 +183,6 @@ func (s *ociStorage) presignedUrl(storagePath string, expiration time.Duration) }).String(), nil } -func (s *ociStorage) DeleteObject(storagePath string) error { - _, err := s.client.DeleteObject(context.Background(), objectstorage.DeleteObjectRequest{ - NamespaceName: &s.namespace, - BucketName: &s.conf.Bucket, - ObjectName: &storagePath, - }) - return wrapOCIError(err) -} - func wrapOCIError(err error) error { if err == nil { return nil @@ -292,7 +215,7 @@ func (s *ociStorage) PutObject(ctx context.Context, storagePath string, data []b if opts.IfAbsent { req.IfNoneMatch = common.String("*") } - _, err := s.ctxClient.PutObject(ctx, req) + _, err := s.client.PutObject(ctx, req) return wrapOCIObjectError(err) } @@ -316,7 +239,7 @@ func (s *ociStorage) UploadObject(ctx context.Context, storagePath string, body if opts.ContentType != "" { contentType = &opts.ContentType } - created, err := s.ctxClient.CreateMultipartUpload(ctx, objectstorage.CreateMultipartUploadRequest{ + created, err := s.client.CreateMultipartUpload(ctx, objectstorage.CreateMultipartUploadRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, IfNoneMatch: ifNoneMatch, @@ -330,7 +253,7 @@ func (s *ociStorage) UploadObject(ctx context.Context, storagePath string, body } if err = s.uploadParts(ctx, storagePath, created.UploadId, body, part, n, ifNoneMatch); err != nil { - _, _ = s.ctxClient.AbortMultipartUpload(context.WithoutCancel(ctx), objectstorage.AbortMultipartUploadRequest{ + _, _ = s.client.AbortMultipartUpload(context.WithoutCancel(ctx), objectstorage.AbortMultipartUploadRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, ObjectName: &storagePath, @@ -347,7 +270,7 @@ func (s *ociStorage) uploadParts(ctx context.Context, storagePath string, upload var parts []objectstorage.CommitMultipartUploadPartDetails for num := 1; n > 0; num++ { size := int64(n) - resp, err := s.ctxClient.UploadPart(ctx, objectstorage.UploadPartRequest{ + resp, err := s.client.UploadPart(ctx, objectstorage.UploadPartRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, ObjectName: &storagePath, @@ -367,7 +290,7 @@ func (s *ociStorage) uploadParts(ctx context.Context, storagePath string, upload } } - _, err := s.ctxClient.CommitMultipartUpload(ctx, objectstorage.CommitMultipartUploadRequest{ + _, err := s.client.CommitMultipartUpload(ctx, objectstorage.CommitMultipartUploadRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, ObjectName: &storagePath, @@ -388,7 +311,7 @@ func (s *ociStorage) GetObjectRange(ctx context.Context, storagePath string, off } var buf bytes.Buffer rng := fmt.Sprintf("bytes=%d-%d", offset, offset+length-1) - _, err := s.readTo(ctx, storagePath, &buf, &rng) + _, err := s.download(ctx, storagePath, &buf, &rng) var se common.ServiceError if errors.As(err, &se) && se.GetHTTPStatusCode() == http.StatusRequestedRangeNotSatisfiable { return []byte{}, nil @@ -400,11 +323,11 @@ func (s *ociStorage) GetObjectRange(ctx context.Context, storagePath string, off } func (s *ociStorage) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { - return s.readTo(ctx, storagePath, w, nil) + return s.download(ctx, storagePath, w, nil) } -func (s *ociStorage) readTo(ctx context.Context, storagePath string, w io.Writer, rng *string) (int64, error) { - resp, err := s.ctxClient.GetObject(ctx, objectstorage.GetObjectRequest{ +func (s *ociStorage) download(ctx context.Context, storagePath string, w io.Writer, rng *string) (int64, error) { + resp, err := s.client.GetObject(ctx, objectstorage.GetObjectRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, ObjectName: &storagePath, @@ -419,7 +342,7 @@ func (s *ociStorage) readTo(ctx context.Context, storagePath string, w io.Writer } func (s *ociStorage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { - _, err := s.ctxClient.HeadObject(ctx, objectstorage.HeadObjectRequest{ + _, err := s.client.HeadObject(ctx, objectstorage.HeadObjectRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, ObjectName: &storagePath, @@ -435,7 +358,7 @@ func (s *ociStorage) ListObjects(ctx context.Context, prefix string) ([]string, var keys []string var start *string for { - resp, err := s.ctxClient.ListObjects(ctx, objectstorage.ListObjectsRequest{ + resp, err := s.client.ListObjects(ctx, objectstorage.ListObjectsRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, Prefix: &prefix, @@ -456,7 +379,7 @@ func (s *ociStorage) ListObjects(ctx context.Context, prefix string) ([]string, func (s *ociStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { for _, p := range storagePaths { - _, err := s.ctxClient.DeleteObject(ctx, objectstorage.DeleteObjectRequest{ + _, err := s.client.DeleteObject(ctx, objectstorage.DeleteObjectRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, ObjectName: &p, @@ -468,8 +391,8 @@ func (s *ociStorage) DeleteObjects(ctx context.Context, storagePaths []string) e return nil } -func (s *ociStorage) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { - return s.presignedUrl(storagePath, expiration) +func (s *ociStorage) GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { + return s.presignedUrl(ctx, storagePath, expiration) } func wrapOCIObjectError(err error) error { diff --git a/s3.go b/s3.go index 5362b9d..760483a 100644 --- a/s3.go +++ b/s3.go @@ -16,6 +16,7 @@ package storage import ( "bytes" + "cmp" "context" "encoding/base64" "errors" @@ -128,12 +129,12 @@ func getConf(conf *S3Config, cp aws.CredentialsProvider) (*aws.Config, error) { }) } + s3Transport := transportWithTimeouts(nil) if conf.ProxyConfig != nil { proxyUrl, err := url.Parse(conf.ProxyConfig.Url) if err != nil { return err } - s3Transport := http.DefaultTransport.(*http.Transport).Clone() s3Transport.Proxy = http.ProxyURL(proxyUrl) if conf.ProxyConfig.Username != "" && conf.ProxyConfig.Password != "" { auth := fmt.Sprintf("%s:%s", conf.ProxyConfig.Username, conf.ProxyConfig.Password) @@ -141,8 +142,8 @@ func getConf(conf *S3Config, cp aws.CredentialsProvider) (*aws.Config, error) { s3Transport.ProxyConnectHeader = http.Header{} s3Transport.ProxyConnectHeader.Add("Proxy-Authorization", basicAuth) } - o.HTTPClient = &http.Client{Transport: s3Transport} } + o.HTTPClient = &http.Client{Transport: s3Transport} return nil } @@ -193,63 +194,37 @@ func (s *s3Storage) getClient(l logging.Logger) *s3.Client { }) } -func (s *s3Storage) UploadData(data []byte, storagePath, contentType string) (string, int64, error) { - location, err := s.upload(bytes.NewReader(data), storagePath, contentType) - if err != nil { +func (s *s3Storage) UploadData(ctx context.Context, data []byte, storagePath, contentType string) (string, int64, error) { + if err := s.upload(ctx, bytes.NewReader(data), storagePath, contentType); err != nil { return "", 0, err } - return location, int64(len(data)), nil + return s.location(storagePath), int64(len(data)), nil } -func (s *s3Storage) UploadFile(filepath, storagePath, contentType string) (string, int64, error) { +func (s *s3Storage) UploadFile(ctx context.Context, filepath, storagePath, contentType string) (string, int64, error) { file, err := os.Open(filepath) if err != nil { return "", 0, err } - defer func() { - _ = file.Close() - }() + defer file.Close() stat, err := file.Stat() if err != nil { return "", 0, err } - - location, err := s.upload(file, storagePath, contentType) - if err != nil { + if err = s.upload(ctx, file, storagePath, contentType); err != nil { return "", 0, err } - - return location, stat.Size(), nil + return s.location(storagePath), stat.Size(), nil } -func (s *s3Storage) upload(reader io.Reader, storagePath, contentType string) (string, error) { +func (s *s3Storage) upload(ctx context.Context, body io.Reader, storagePath, contentType string) error { l := NewS3Logger() - client := s.getClient(l) - - input := &s3.PutObjectInput{ - Body: reader, - Bucket: aws.String(s.conf.Bucket), - ContentType: aws.String(contentType), - Key: aws.String(storagePath), - Metadata: s.conf.Metadata, - } - if s.conf.Tagging != "" { - input.Tagging = &s.conf.Tagging - } - if s.conf.ContentDisposition != "" { - input.ContentDisposition = &s.conf.ContentDisposition - } else { - contentDisposition := "inline" - input.ContentDisposition = &contentDisposition - } - - if _, err := s.uploader(client).Upload(context.Background(), input); err != nil { + if _, err := s.uploader(s.getClient(l)).Upload(ctx, s.putInput(storagePath, body, PutOptions{ContentType: contentType})); err != nil { l.WriteLogs() - return "", wrapS3Error(err) + return wrapS3Error(err) } - - return s.location(storagePath), nil + return nil } func (s *s3Storage) uploader(client *s3.Client, opts ...func(*manager.Uploader)) *manager.Uploader { @@ -300,48 +275,19 @@ func (s *s3Storage) location(storagePath string) string { return loc.String() } -func (s *s3Storage) DownloadData(storagePath string) ([]byte, error) { - w := &manager.WriteAtBuffer{} - _, err := s.download(w, storagePath) - if err != nil { - return nil, err - } - - return w.Bytes(), nil -} - -func (s *s3Storage) DownloadFile(filepath, storagePath string) (int64, error) { +// DownloadFile fetches ranges in parallel into the file. +func (s *s3Storage) DownloadFile(ctx context.Context, filepath, storagePath string) (int64, error) { file, err := os.Create(filepath) if err != nil { return 0, err } defer file.Close() - return s.download(file, storagePath) -} - -func (s *s3Storage) download(w io.WriterAt, storagePath string) (int64, error) { - client := s.getClient(nil) - - n, err := manager.NewDownloader(client).Download( - context.Background(), - w, - &s3.GetObjectInput{ - Bucket: aws.String(s.conf.Bucket), - Key: aws.String(storagePath), - }, - ) - return n, wrapS3ObjectError(err) -} - -func (s *s3Storage) DeleteObject(storagePath string) error { - client := s.getClient(nil) - - _, err := client.DeleteObject(context.Background(), &s3.DeleteObjectInput{ + n, err := manager.NewDownloader(s.getClient(nil)).Download(ctx, file, &s3.GetObjectInput{ Bucket: aws.String(s.conf.Bucket), Key: aws.String(storagePath), }) - return wrapS3Error(err) + return n, wrapS3ObjectError(err) } const s3UploadPartSize = 16 << 20 @@ -396,15 +342,22 @@ func completeIfAbsent(o *s3.Options) { }) } +// putInput is every write's request: the content type, and the metadata, +// tagging and content disposition (inline by default) the config sets. func (s *s3Storage) putInput(storagePath string, body io.Reader, opts PutOptions) *s3.PutObjectInput { input := &s3.PutObjectInput{ - Bucket: aws.String(s.conf.Bucket), - Key: aws.String(storagePath), - Body: body, + Bucket: aws.String(s.conf.Bucket), + Key: aws.String(storagePath), + Body: body, + Metadata: s.conf.Metadata, + ContentDisposition: aws.String(cmp.Or(s.conf.ContentDisposition, "inline")), } if opts.ContentType != "" { input.ContentType = aws.String(opts.ContentType) } + if s.conf.Tagging != "" { + input.Tagging = aws.String(s.conf.Tagging) + } return input } diff --git a/storage.go b/storage.go index 6339471..b2c0047 100644 --- a/storage.go +++ b/storage.go @@ -17,6 +17,9 @@ package storage import ( "bytes" "context" + "net" + "net/http" + "time" "github.com/livekit/storage/objects" ) @@ -39,10 +42,29 @@ func New(conf Config) (Storage, error) { } // getObject is GetObject for a backend whose reads are streams. -func getObject(ctx context.Context, s Storage, storagePath string) ([]byte, error) { +func getObject(ctx context.Context, s Bucket, storagePath string) ([]byte, error) { var buf bytes.Buffer if _, err := s.DownloadObject(ctx, storagePath, &buf); err != nil { return nil, err } return buf.Bytes(), nil } + +// responseHeaderTimeout bounds the wait for a response's headers on the clients +// this package builds. It catches a dead connection without a deadline on the +// body, so a long transfer is never cut; a call's context bounds the call. +const responseHeaderTimeout = time.Minute + +// transportWithTimeouts is t, or Go's default transport, with dial, TLS and +// response header timeouts. +func transportWithTimeouts(rt http.RoundTripper) *http.Transport { + t, ok := rt.(*http.Transport) + if !ok { + t = http.DefaultTransport.(*http.Transport) + } + t = t.Clone() + t.DialContext = (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext + t.TLSHandshakeTimeout = 10 * time.Second + t.ResponseHeaderTimeout = responseHeaderTimeout + return t +} diff --git a/storage_test.go b/storage_test.go index 5236f69..1aa58d6 100644 --- a/storage_test.go +++ b/storage_test.go @@ -223,7 +223,7 @@ func testStorage(t *testing.T, s storage.Storage) { }) t.Run("UploadData", func(t *testing.T) { - loc, size, err := s.UploadData(dataPayload, pathData, "text/plain") + loc, size, err := s.UploadData(context.Background(), dataPayload, pathData, "text/plain") require.NoError(t, err) require.Equal(t, int64(len(dataPayload)), size) assertLocation(t, loc, pathData) @@ -238,7 +238,7 @@ func testStorage(t *testing.T, s storage.Storage) { require.NoError(t, err) require.NoError(t, tmp.Close()) - loc, size, err := s.UploadFile(tmp.Name(), pathFile, "text/plain") + loc, size, err := s.UploadFile(context.Background(), tmp.Name(), pathFile, "text/plain") require.NoError(t, err) require.Equal(t, int64(len(filePayload)), size) assertLocation(t, loc, pathFile) @@ -254,7 +254,7 @@ func testStorage(t *testing.T, s storage.Storage) { }) t.Run("DownloadData", func(t *testing.T) { - downloaded, err := s.DownloadData(pathData) + downloaded, err := s.GetObject(context.Background(), pathData) require.NoError(t, err) require.Equal(t, dataPayload, downloaded) }) @@ -265,7 +265,7 @@ func testStorage(t *testing.T, s storage.Storage) { require.NoError(t, tmp.Close()) defer os.Remove(tmp.Name()) - size, err := s.DownloadFile(tmp.Name(), pathFile) + size, err := s.DownloadFile(context.Background(), tmp.Name(), pathFile) require.NoError(t, err) require.Equal(t, int64(len(filePayload)), size) @@ -303,8 +303,8 @@ func testStorage(t *testing.T, s storage.Storage) { require.Equal(t, dataPayload, body) }) - t.Run("DeleteObject", func(t *testing.T) { - require.NoError(t, s.DeleteObject(pathData)) + t.Run("DeleteObjects one path", func(t *testing.T) { + require.NoError(t, s.DeleteObjects(context.Background(), []string{pathData})) items, err := s.ListObjects(context.Background(), prefix) require.NoError(t, err) @@ -314,9 +314,9 @@ func testStorage(t *testing.T, s storage.Storage) { t.Run("DeleteObjects", func(t *testing.T) { // Upload two more objects, then bulk-delete them along with the leftover pathFile. - _, _, err := s.UploadData(bulk1Payload, pathBulk1, "text/plain") + _, _, err := s.UploadData(context.Background(), bulk1Payload, pathBulk1, "text/plain") require.NoError(t, err) - _, _, err = s.UploadData(bulk2Payload, pathBulk2, "text/plain") + _, _, err = s.UploadData(context.Background(), bulk2Payload, pathBulk2, "text/plain") require.NoError(t, err) require.NoError(t, s.DeleteObjects(context.Background(), []string{pathFile, pathBulk1, pathBulk2})) @@ -329,7 +329,7 @@ func testStorage(t *testing.T, s storage.Storage) { t.Run("Multipart", func(t *testing.T) { pathMultipart := prefix + "-multipart.bin" t.Cleanup(func() { - _ = s.DeleteObject(pathMultipart) + _ = s.DeleteObjects(context.Background(), []string{pathMultipart}) }) // 6 MiB exceeds the S3 manager's default 5 MiB part size, forcing the @@ -339,12 +339,12 @@ func testStorage(t *testing.T, s storage.Storage) { payload[i] = byte(i) } - loc, size, err := s.UploadData(payload, pathMultipart, "application/octet-stream") + loc, size, err := s.UploadData(context.Background(), payload, pathMultipart, "application/octet-stream") require.NoError(t, err) require.Equal(t, int64(len(payload)), size) assertLocation(t, loc, pathMultipart) - downloaded, err := s.DownloadData(pathMultipart) + downloaded, err := s.GetObject(context.Background(), pathMultipart) require.NoError(t, err) require.Equal(t, len(payload), len(downloaded)) require.True(t, bytes.Equal(payload, downloaded), "downloaded multipart content does not match uploaded") From 7d4eff7c6f03d98eeecaa8f42b40fe9ee5d3d7f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 01:19:53 -0700 Subject: [PATCH 09/29] Tests: every backend but AliOSS end to end against floci in Docker floci_test.go starts floci's S3, Azure Blob, GCS and OCI emulators with dockertest and runs the Storage and Bucket suites on each; without Docker they skip, except under CI. AzureConfig and GCPConfig gain an Endpoint for emulators. Found by the emulators: - GCS presign signs with CredentialsJSON's service account: passed only as a token source, the SDK had no signer off GCE. URLs are V4 (7 days at most). - OCI presign and locations keep an http:// endpoint's scheme. - OCI answers a range starting at the end with 400 on floci; the object's size decides. UploadObject with IfAbsent checks first, as S3 and Azure do, since floci-oci ignores if-none-match on multipart uploads. --- README.md | 4 + azure.go | 34 +++++--- config/config.go | 6 ++ floci_test.go | 197 +++++++++++++++++++++++++++++++++++++++++++++++ gcp.go | 34 ++++++-- go.mod | 26 +++++++ go.sum | 102 ++++++++++++++++++++++++ oci.go | 48 +++++++++--- 8 files changed, 424 insertions(+), 27 deletions(-) create mode 100644 floci_test.go diff --git a/README.md b/README.md index 105fceb..e379643 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,7 @@ A GORM-style storage wrapper with unified API for different cloud storage backen - `Storage` is `Bucket` plus `UploadData`, `UploadFile` and `DownloadFile`, which publish and fetch whole files and return their location. - `storage/objects` imports no provider SDK, so a fake or a library can implement or accept the interfaces without pulling in every SDK. - The clients this package builds time out a connection that stalls before answering, never a transfer in progress; the context bounds the call. + +## Tests + +`go test ./...` runs every backend except AliOSS end to end against [floci](https://github.com/floci-io) emulators in Docker (S3, Azure Blob, GCS, OCI Object Storage), plus the local directory. Without Docker those tests skip, except when `CI` is set. The provider tests (`TestS3`, `TestGCP`, ...) reach real accounts when their environment variables are set; AliOSS has no emulator and is covered only that way. diff --git a/azure.go b/azure.go index caa5aed..0b61d61 100644 --- a/azure.go +++ b/azure.go @@ -63,7 +63,10 @@ func NewAzure(conf *AzureConfig) (Storage, error) { return nil, err } - serviceUrl := fmt.Sprintf("https://%s.blob.core.windows.net/", conf.AccountName) + serviceUrl := azureServiceURL(conf) + if _, err := url.Parse(serviceUrl); err != nil { + return nil, fmt.Errorf("azure endpoint: %w", err) + } client, err := azblob.NewClientWithSharedKeyCredential(serviceUrl, cred, nil) if err != nil { return nil, err @@ -77,11 +80,22 @@ func NewAzure(conf *AzureConfig) (Storage, error) { } func (s *azureBLOBStorage) location(storagePath string) string { - return (&url.URL{ - Scheme: "https", - Host: fmt.Sprintf("%s.blob.core.windows.net", s.conf.AccountName), - Path: path.Join(s.conf.ContainerName, storagePath), - }).String() + return s.blobURL(storagePath).String() +} + +func azureServiceURL(conf *AzureConfig) string { + if conf.Endpoint != "" { + return conf.Endpoint + } + return fmt.Sprintf("https://%s.blob.core.windows.net/", conf.AccountName) +} + +// blobURL is storagePath under the service URL, path-style below an +// emulator's account segment. NewAzure has checked the URL parses. +func (s *azureBLOBStorage) blobURL(storagePath string) *url.URL { + u, _ := url.Parse(azureServiceURL(s.conf)) + u.Path = path.Join(u.Path, s.conf.ContainerName, storagePath) + return u } func (s *azureBLOBStorage) UploadData(ctx context.Context, data []byte, storagePath, contentType string) (string, int64, error) { @@ -172,12 +186,8 @@ func (s *azureBLOBStorage) presignedUrlWithToken(ctx context.Context, storagePat return "", err } - loc := &url.URL{ - Scheme: "https", - Host: fmt.Sprintf("%s.blob.core.windows.net", s.conf.AccountName), - Path: path.Join(s.conf.ContainerName, storagePath), - RawQuery: qp.Encode(), - } + loc := s.blobURL(storagePath) + loc.RawQuery = qp.Encode() return loc.String(), nil } diff --git a/config/config.go b/config/config.go index 7e90a57..1abe09d 100644 --- a/config/config.go +++ b/config/config.go @@ -41,6 +41,9 @@ type AzureConfig struct { AccountKey string `yaml:"account_key,omitempty"` // (env AZURE_STORAGE_KEY) ContainerName string `yaml:"container_name,omitempty"` TokenCredential azcore.TokenCredential `yaml:"-"` // required for presigned url generation + // Endpoint replaces the service URL https://.blob.core.windows.net/, + // e.g. an emulator's http://localhost:4577//. + Endpoint string `yaml:"endpoint,omitempty"` } func (*AzureConfig) storageConfig() {} @@ -49,6 +52,9 @@ type GCPConfig struct { CredentialsJSON string `yaml:"credentials_json,omitempty"` // (env GOOGLE_APPLICATION_CREDENTIALS) Bucket string `yaml:"bucket,omitempty"` ProxyConfig *ProxyConfig `yaml:"proxy_config,omitempty"` + // Endpoint points the client at an emulator, e.g. http://localhost:4588. + // Requests to it are unauthenticated; CredentialsJSON still signs URLs. + Endpoint string `yaml:"endpoint,omitempty"` } func (*GCPConfig) storageConfig() {} diff --git a/floci_test.go b/floci_test.go new file mode 100644 index 0000000..548325e --- /dev/null +++ b/floci_test.go @@ -0,0 +1,197 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage_test + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "os" + "strings" + "testing" + + gcs "cloud.google.com/go/storage" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/objectstorage" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" + "github.com/stretchr/testify/require" + "google.golang.org/api/option" + + "github.com/livekit/storage" +) + +// The emulators run with the tests on any machine with Docker, and must in CI. +// Each holds its state in memory and is removed when its test ends. + +const flociBucket = "livekit" + +func TestFlociS3(t *testing.T) { + endpoint := floci(t, "floci/floci", "2.3.0", "4566") + conf := &storage.S3Config{ + AccessKey: "test", + Secret: "test", + Region: "us-east-1", + Endpoint: endpoint, + ForcePathStyle: true, + Bucket: flociBucket, + } + admin := s3.New(s3.Options{ + Region: conf.Region, + BaseEndpoint: &conf.Endpoint, + UsePathStyle: true, + Credentials: credentials.NewStaticCredentialsProvider(conf.AccessKey, conf.Secret, ""), + }) + _, err := admin.CreateBucket(t.Context(), &s3.CreateBucketInput{Bucket: aws.String(flociBucket)}) + require.NoError(t, err) + + testProvider(t, conf) +} + +// flociAzureKey is Azurite's well-known development key; floci-az accepts any. +const flociAzureKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + +func TestFlociAzure(t *testing.T) { + // An IP host: the SDK parses an account from the path only then. + endpoint := strings.Replace(floci(t, "floci/floci-az", "0.15.0", "4577"), "localhost", "127.0.0.1", 1) + conf := &storage.AzureConfig{ + AccountName: "devstoreaccount1", + AccountKey: flociAzureKey, + ContainerName: flociBucket, + Endpoint: endpoint + "/devstoreaccount1/", + } + cred, err := azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey) + require.NoError(t, err) + admin, err := azblob.NewClientWithSharedKeyCredential(conf.Endpoint, cred, nil) + require.NoError(t, err) + _, err = admin.CreateContainer(t.Context(), flociBucket, nil) + require.NoError(t, err) + + testProvider(t, conf) +} + +func TestFlociGCP(t *testing.T) { + endpoint := floci(t, "floci/floci-gcp", "0.11.0", "4588") + conf := &storage.GCPConfig{ + // A service account of its own, to sign URLs. + CredentialsJSON: gcpServiceAccount(t), + Bucket: flociBucket, + Endpoint: endpoint, + } + admin, err := gcs.NewClient(t.Context(), option.WithEndpoint(endpoint+"/storage/v1/"), option.WithoutAuthentication()) + require.NoError(t, err) + t.Cleanup(func() { _ = admin.Close() }) + require.NoError(t, admin.Bucket(flociBucket).Create(t.Context(), "floci-local", nil)) + + testProvider(t, conf) +} + +// flociTenancy is floci-oci's tenancy, which is also the root compartment. +const flociTenancy = "ocid1.tenancy.oc1..flocilocaltenancy0000000000000000000000000000000000000000" + +func TestFlociOCI(t *testing.T) { + endpoint := floci(t, "floci/floci-oci", "0.5.0", "4599") + // floci-oci reads the tenancy and user from the request signature without + // verifying it, so any key signs. + conf := &storage.OCIConfig{ + TenancyOCID: flociTenancy, + UserOCID: "ocid1.user.oc1..livekit", + Fingerprint: "00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00", + PrivateKey: rsaKeyPEM(t), + Region: "us-ashburn-1", + Endpoint: endpoint, + Bucket: flociBucket, + } + provider := common.NewRawConfigurationProvider(conf.TenancyOCID, conf.UserOCID, conf.Region, conf.Fingerprint, conf.PrivateKey, nil) + admin, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(provider) + require.NoError(t, err) + admin.Host = endpoint + ns, err := admin.GetNamespace(t.Context(), objectstorage.GetNamespaceRequest{}) + require.NoError(t, err) + _, err = admin.CreateBucket(t.Context(), objectstorage.CreateBucketRequest{ + NamespaceName: ns.Value, + CreateBucketDetails: objectstorage.CreateBucketDetails{ + Name: common.String(flociBucket), + CompartmentId: common.String(flociTenancy), + }, + }) + require.NoError(t, err) + + testProvider(t, conf) +} + +func testProvider(t *testing.T, conf storage.Config) { + s, err := storage.New(conf) + require.NoError(t, err) + t.Run("Storage", func(t *testing.T) { testStorage(t, s) }) + t.Run("Bucket", func(t *testing.T) { testObjects(t, s) }) +} + +// floci starts repo:tag and returns its http:// endpoint once it answers. It +// skips the test where Docker is unavailable, except in CI. +func floci(t *testing.T, repo, tag, port string) string { + pool, err := dockertest.NewPool("") + if err == nil { + err = pool.Client.Ping() + } + if err != nil { + if os.Getenv("CI") != "" { + t.Fatalf("docker: %v", err) + } + t.Skipf("docker unavailable: %v", err) + } + + res, err := pool.RunWithOptions(&dockertest.RunOptions{Repository: repo, Tag: tag}, func(hc *docker.HostConfig) { + hc.AutoRemove = true + }) + require.NoError(t, err) + t.Cleanup(func() { _ = pool.Purge(res) }) + + endpoint := "http://" + res.GetHostPort(port+"/tcp") + require.NoError(t, pool.Retry(func() error { + resp, err := pool.Client.HTTPClient.Get(endpoint) + if err == nil { + resp.Body.Close() + } + return err + })) + return endpoint +} + +func rsaKeyPEM(t *testing.T) string { + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + der, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(t, err) + return string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})) +} + +func gcpServiceAccount(t *testing.T) string { + b, err := json.Marshal(map[string]string{ + "type": "service_account", + "project_id": "floci-local", + "client_email": "livekit@floci-local.iam.gserviceaccount.com", + "private_key": rsaKeyPEM(t), + "token_uri": "https://oauth2.googleapis.com/token", + }) + require.NoError(t, err) + return string(b) +} diff --git a/gcp.go b/gcp.go index d4066d0..545f504 100644 --- a/gcp.go +++ b/gcp.go @@ -24,11 +24,13 @@ import ( "net/http" "net/url" "os" + "strings" "time" "cloud.google.com/go/storage" "github.com/googleapis/gax-go/v2" "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" "google.golang.org/api/googleapi" "google.golang.org/api/iterator" "google.golang.org/api/option" @@ -59,6 +61,10 @@ func wrapGCPError(err error) error { type gcpStorage struct { conf *GCPConfig client *storage.Client + // signer is CredentialsJSON's service account, which signs URLs: with the + // credentials passed as a token source, the SDK finds no signer of its own + // off GCE. + signer *jwt.Config } func NewGCP(conf *GCPConfig) (Storage, error) { @@ -72,7 +78,16 @@ func NewGCP(conf *GCPConfig) (Storage, error) { if err != nil { return nil, err } - opts = append(opts, option.WithTokenSource(jwtConfig.TokenSource(context.Background()))) + u.signer = jwtConfig + if conf.Endpoint == "" { + opts = append(opts, option.WithTokenSource(jwtConfig.TokenSource(context.Background()))) + } + } + if conf.Endpoint != "" { + opts = append(opts, + option.WithEndpoint(strings.TrimSuffix(conf.Endpoint, "/")+"/storage/v1/"), + option.WithoutAuthentication(), + ) } defaultTransport := http.DefaultTransport.(*http.Transport) @@ -146,10 +161,19 @@ func (s *gcpStorage) location(storagePath string) string { } func (s *gcpStorage) presignedUrl(storagePath string, expiration time.Duration) (string, error) { - u, err := s.client.Bucket(s.conf.Bucket).SignedURL(storagePath, &storage.SignedURLOptions{ - Method: "GET", - Expires: time.Now().Add(expiration), - }) + // V4, which the SDK can address to an http:// emulator, signs for at most 7 + // days. + opts := &storage.SignedURLOptions{ + Scheme: storage.SigningSchemeV4, + Method: "GET", + Expires: time.Now().Add(expiration), + Insecure: strings.HasPrefix(s.conf.Endpoint, "http://"), + } + if s.signer != nil { + opts.GoogleAccessID = s.signer.Email + opts.PrivateKey = s.signer.PrivateKey + } + u, err := s.client.Bucket(s.conf.Bucket).SignedURL(storagePath, opts) if err != nil { return "", wrapGCPError(err) } diff --git a/go.mod b/go.mod index ea3dbcd..a4bf58c 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/joho/godotenv v1.5.1 github.com/livekit/protocol v1.52.1 github.com/oracle/oci-go-sdk/v65 v65.126.1 + github.com/ory/dockertest/v3 v3.12.0 github.com/stretchr/testify v1.12.1 golang.org/x/oauth2 v0.37.0 google.golang.org/api v0.300.0 @@ -31,11 +32,15 @@ require ( cloud.google.com/go/compute/metadata v0.10.0 // indirect cloud.google.com/go/iam v1.12.0 // indirect cloud.google.com/go/monitoring v1.30.0 // indirect + dario.cat/mergo v1.0.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/apache/arrow-go/v18 v18.7.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect @@ -49,27 +54,47 @@ require ( github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/containerd/continuity v0.4.5 // indirect + github.com/docker/cli v27.4.1+incompatible // indirect + github.com/docker/docker v27.1.1+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/gofrs/flock v0.10.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/google/s2a-go v0.1.10 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.22 // indirect github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/sys/user v0.3.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/opencontainers/runc v1.2.3 // indirect github.com/pierrec/lz4/v4 v4.1.28 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/puzpuzpuz/xsync/v4 v4.5.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker/v2 v2.4.0 // indirect github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect @@ -98,4 +123,5 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 // indirect google.golang.org/grpc v1.84.0 // indirect google.golang.org/protobuf v1.36.12 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index cd5cfc6..7338446 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,10 @@ cloud.google.com/go/storage v1.69.0 h1:jAAMC1411HEh78nKsU0Zns+eFj3TnhjAWIhg5Ud/X cloud.google.com/go/storage v1.69.0/go.mod h1:PELYsxTYm2peE4mwLEC1+mS1dA/kUSRUxNv56rOy44g= cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2 h1:utpeoEeZjd+A8J41zvoLsOOrqXHhX1Kx/X/tCW9dEYQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU= @@ -38,6 +42,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 h1:tQJdgQjbKKffEiMOK github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2/go.mod h1:kTkpJchXAdO7LGkKZP97a+Ou8TgaF4NT6MHwOIy5uSQ= github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 h1:X/bTwe0qpvmvPPutc+LWo7T4cKMqDP9JHP4k+Co9ZB4= github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0/go.mod h1:+kX7ahyFelQ42F3RScM2CFgHJljp0phc/FieKLYVpFI= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI= github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 h1:bN1gA3of5bXtbnLsRPrwfmbbe7A5UWFlcTHseujLnpc= @@ -48,6 +54,10 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 h1:RoO5+d7uCmDqovLrHCr2/BuViUXvdcrNxyNM1pN9dDQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= @@ -96,12 +106,28 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZj github.com/aws/aws-sdk-go-v2/service/sts v1.51.1/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM= github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= +github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dennwc/iters v1.2.2 h1:XH2/Etihiy9ZvPOVCR+icQXeYlhbvS7k0qro4x/2qQo= github.com/dennwc/iters v1.2.2/go.mod h1:M9KuuMBeyEXYTmB7EnI9SCyALFCmPWOIxn5W1L0CjGg= +github.com/docker/cli v27.4.1+incompatible h1:VzPiUlRJ/xh+otB75gva3r05isHMo5wXDfPRi5/b4hI= +github.com/docker/cli v27.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= @@ -119,10 +145,16 @@ github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/flock v0.10.0 h1:SHMXenfaB03KbroETaCMtbBg3Yn29v4w1r+tgy4ff4k= github.com/gofrs/flock v0.10.0/go.mod h1:FirDy1Ing0mI2+kB6wk+vyyAH+e6xiE+EYA0jnzV9jc= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -137,6 +169,8 @@ github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9 github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.10 h1:EMp+aOuXN6l8cE/gjF5Bt+vyZxsUuyCWe9chDWR/+uU= github.com/google/s2a-go v0.1.10/go.mod h1:pz4tyvwXvJLLbyrkh6FW1eS2zPUXMaTmyNhYtyP2tNw= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.22 h1:NU4XpII6jD+Dxcot94fqjE+AfJoE/lQP9q3faYGzC/c= @@ -145,6 +179,8 @@ github.com/googleapis/gax-go/v2 v2.26.2 h1:ydkmNXxj7bEmmeK5AihkKnWxyOyBR9TDebvp5 github.com/googleapis/gax-go/v2 v2.26.2/go.mod h1:sMKqnMesnKH+3wiRJROcttA+cJoZoGbZl1vDQ8XYtGk= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= @@ -155,36 +191,69 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5ATTo469PQPkqzdoU7be46ryiCDO3boc= github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/protocol v1.52.1 h1:w8wRjRlj/57AJBxwomEzpEMzf+lT0qOoLQLO7awdTMU= github.com/livekit/protocol v1.52.1/go.mod h1:t5CV/LCsWbI9zsSTHTJKDyLtByqvvVkioSP7jgqpIGA= github.com/livekit/psrpc v0.8.0 h1:uo3x+DL8pK2OIaCxg/s9sLUO2cIocS5HwdPhhI+m+fQ= github.com/livekit/psrpc v0.8.0/go.mod h1:6KnU8vULoizX0Er9NNfeKJDV8ug87j3fuo19M0bhj6Q= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= +github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/runc v1.2.3 h1:fxE7amCzfZflJO2lHXf4y/y8M1BoAqp+FVmG19oYB80= +github.com/opencontainers/runc v1.2.3/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM= github.com/oracle/oci-go-sdk/v65 v65.126.1 h1:WmQ2Igq7/L/cHlR2jZnJkBC2UI51Ams4DJZW3CZGVBo= github.com/oracle/oci-go-sdk/v65 v65.126.1/go.mod h1:YmvgbsnUfrsiJ410XAGQ1BRsrUYJJ7W3O5wUikmeI8w= +github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCyRCw= +github.com/ory/dockertest/v3 v3.12.0/go.mod h1:aKNDTva3cp8dwOWwb9cWuX84aH5akkxXRvO7KCwWVjE= github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio= github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 h1:S1hI5JiKP7883xBzZAr1ydcxrKNSVNm7+3+JwjxZEsg= github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25/go.mod h1:ZQntvDG8TkPgljxtA0R9frDoND4QORU1VXz015N5Ks4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg= github.com/sony/gobreaker/v2 v2.4.0/go.mod h1:pTyFJgcZ3h2tdQVLZZruK2C0eoFL1fb/G83wK1ZQl+s= github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJXP61mNV3/7iuU= github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= @@ -223,22 +292,49 @@ go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.300.0 h1:2rvPV2bqnPuHOaF4gGOBiT1IIc6JVXYyHCkZeqdzjNk= @@ -253,7 +349,13 @@ google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/oci.go b/oci.go index c0be9fd..fb576fe 100644 --- a/oci.go +++ b/oci.go @@ -37,6 +37,8 @@ type ociStorage struct { client objectstorage.ObjectStorageClient namespace string host string + // insecure is an http:// Endpoint, such as an emulator's. + insecure bool } func NewOCI(conf *OCIConfig) (Storage, error) { @@ -65,6 +67,7 @@ func NewOCI(conf *OCIConfig) (Storage, error) { // The SDK only prepends a scheme to client.Host on the first request, so // read it before GetNamespace below. host := strings.TrimPrefix(client.Host, "https://") + insecure := strings.HasPrefix(host, "http://") host = strings.TrimPrefix(host, "http://") s := &ociStorage{ @@ -72,6 +75,7 @@ func NewOCI(conf *OCIConfig) (Storage, error) { client: client, namespace: conf.Namespace, host: host, + insecure: insecure, } if s.namespace == "" { @@ -137,11 +141,15 @@ func (s *ociStorage) UploadFile(ctx context.Context, filepath, storagePath, cont } func (s *ociStorage) location(storagePath string) string { - return (&url.URL{ - Scheme: "https", - Host: s.host, - Path: path.Join("/n", s.namespace, "b", s.conf.Bucket, "o", storagePath), - }).String() + return s.url(path.Join("/n", s.namespace, "b", s.conf.Bucket, "o", storagePath)) +} + +func (s *ociStorage) url(p string) string { + scheme := "https" + if s.insecure { + scheme = "http" + } + return (&url.URL{Scheme: scheme, Host: s.host, Path: p}).String() } func (s *ociStorage) DownloadFile(ctx context.Context, filepath, storagePath string) (int64, error) { @@ -176,11 +184,7 @@ func (s *ociStorage) presignedUrl(ctx context.Context, storagePath string, expir return "", wrapOCIError(err) } - return (&url.URL{ - Scheme: "https", - Host: s.host, - Path: *resp.AccessUri, - }).String(), nil + return s.url(*resp.AccessUri), nil } func wrapOCIError(err error) error { @@ -223,6 +227,18 @@ func (s *ociStorage) PutObject(ctx context.Context, storagePath string, data []b // multipart upload of sequential parts through one buffer. IfAbsent is checked // at create and again at commit, so it is atomic here. func (s *ociStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + // The condition is checked again at create and commit; this check spares a + // refused upload its traffic, and refuses it where a store ignores + // if-none-match on multipart uploads, as floci-oci does. + if opts.IfAbsent { + exists, err := s.ObjectExists(ctx, storagePath) + if err != nil { + return err + } + if exists { + return ErrObjectExists + } + } part := make([]byte, ociUploadPartSize) n, err := io.ReadFull(body, part) if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { @@ -316,6 +332,18 @@ func (s *ociStorage) GetObjectRange(ctx context.Context, storagePath string, off if errors.As(err, &se) && se.GetHTTPStatusCode() == http.StatusRequestedRangeNotSatisfiable { return []byte{}, nil } + // A range starting at the end can also come back 400 InvalidParameter, as on + // floci-oci; the object's size tells it from a bad request. + if errors.As(err, &se) && se.GetHTTPStatusCode() == http.StatusBadRequest { + head, headErr := s.client.HeadObject(ctx, objectstorage.HeadObjectRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &storagePath, + }) + if headErr == nil && head.ContentLength != nil && offset >= *head.ContentLength { + return []byte{}, nil + } + } if err != nil { return nil, err } From 8d4aae6e371b95e12707d70c2abf0a00dfbc5de7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 09:26:03 -0700 Subject: [PATCH 10/29] CI: a floci job per provider; the unit test job skips them --- .github/workflows/test.yaml | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f2b7641..b369c53 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -42,4 +42,35 @@ jobs: - name: Test run: | set -euo pipefail - go test -race -json -v ./... 2>&1 | gotestfmt + go test -race -json -v -skip '^TestFloci' ./... 2>&1 | gotestfmt + + # Every backend end to end against its floci emulator (floci_test.go), one + # check per provider. + floci: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + provider: [S3, Azure, GCP, OCI] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + ~/go/pkg/mod + ~/go/bin + ~/.cache + key: livekit-storage + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: "go.mod" + + - name: Set up gotestfmt + run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@9eae5abc81d6d08f73268741ef4a8b97f29b60d8 # v2.4.1 + + - name: Test against floci + run: | + set -euo pipefail + go test -race -json -v -run '^TestFloci${{ matrix.provider }}$' ./... 2>&1 | gotestfmt From 3f678656110e5760ef9b665ece5ed21a4927cb35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 10:30:43 -0700 Subject: [PATCH 11/29] Storage: main's wire behavior kept, S3 semantics, cleaner API, e2e module API: - UploadData removed: PutObject with a content type does the same - GeneratePresignedUrl is PresignedURL; ErrObjectNotFound/ErrObjectExists are ErrNotFound/ErrExists, as in objects - ProxyConfig.URL, S3Config.AssumeRoleARN/AssumeRoleExternalID (YAML unchanged) - S3Logger unexported Behavior: - DeleteObjects follows S3: a missing key succeeds, any key still present fails the call (S3 per-key errors were ignored on main) - Connections are cut after 5 minutes without bytes either way, replacing the response header timeout; S3 pinned to legacy defaults mode so the SDK keeps the dialer - OCI: UploadData/UploadFile one PutObject as on main, Content-Disposition inline, SDK TLS wrapper kept, OCI_CUSTOM_CLIENT_TIMEOUT honored - S3 GetObject uses the ranged downloader again - GCS: V2 signing as on main, V4 only for an http:// emulator - Local: keys outside StorageDir refused, byte-prefix listing, a directory is no object, fsync before publish, os.Create's file mode - A wrapped http.DefaultTransport no longer panics Tests: - Suites move to the e2e module, so users don't inherit dockertest's deps - New: refused deletes, conditional commit on the wire, idle timeout, proxy and GCS endpoint credentials, local keys, CI matrix completeness - floci images pinned by digest --- .github/workflows/test.yaml | 34 ++- README.md | 13 +- alioss.go | 17 +- azure.go | 140 ++++++---- config/config.go | 12 +- contract_test.go | 273 +++++++++++++++++++ e2e/ci_test.go | 56 ++++ e2e/floci_test.go | 356 ++++++++++++++++++++++++ e2e/go.mod | 130 +++++++++ e2e/go.sum | 361 +++++++++++++++++++++++++ e2e/minio_test.go | 100 +++++++ e2e/object_test.go | 273 +++++++++++++++++++ storage_test.go => e2e/storage_test.go | 43 ++- floci_test.go | 197 -------------- gcp.go | 89 +++--- go.mod | 27 -- go.sum | 104 ------- local.go | 327 ++++++++++++---------- location_test.go | 11 +- logging.go | 12 +- object_test.go | 106 -------- objects/objects.go | 15 +- oci.go | 102 ++++--- s3.go | 123 ++++----- storage.go | 119 +++++++- transport_test.go | 108 ++++++++ 26 files changed, 2288 insertions(+), 860 deletions(-) create mode 100644 contract_test.go create mode 100644 e2e/ci_test.go create mode 100644 e2e/floci_test.go create mode 100644 e2e/go.mod create mode 100644 e2e/go.sum create mode 100644 e2e/minio_test.go create mode 100644 e2e/object_test.go rename storage_test.go => e2e/storage_test.go (88%) delete mode 100644 floci_test.go delete mode 100644 object_test.go create mode 100644 transport_test.go diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b369c53..e821323 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,26 +32,37 @@ jobs: run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@9eae5abc81d6d08f73268741ef4a8b97f29b60d8 # v2.4.1 - name: Download Go modules - run: go mod download + run: go mod download && (cd e2e && go mod download) - name: Lint uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 with: version: v2.14.0 + - name: Lint e2e + uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 + with: + version: v2.14.0 + working-directory: e2e + + # The e2e module holds the suites: here they run on the local backend, and + # on any real account whose secrets are set. - name: Test run: | set -euo pipefail - go test -race -json -v -skip '^TestFloci' ./... 2>&1 | gotestfmt + go test -race -json -v ./... 2>&1 | gotestfmt + cd e2e && go test -race -json -v -skip '^Test(Floci|MinIO)' ./... 2>&1 | gotestfmt - # Every backend end to end against its floci emulator (floci_test.go), one - # check per provider. - floci: + # Every backend end to end against its floci emulator (e2e/floci_test.go), + # and S3 against MinIO (e2e/minio_test.go), whose conditional writes are + # atomic where floci's are not. One check each; TestWorkflowRunsEveryEmulatorTest + # fails when a test is missing here. + e2e: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - provider: [S3, Azure, GCP, OCI] + test: [FlociS3, FlociAzure, FlociGCP, FlociOCI, MinIO] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -70,7 +81,14 @@ jobs: - name: Set up gotestfmt run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@9eae5abc81d6d08f73268741ef4a8b97f29b60d8 # v2.4.1 - - name: Test against floci + - name: Install MinIO + if: matrix.test == 'MinIO' + run: | + go install github.com/minio/minio@v0.0.0-20260212201848-7aac2a2c5b7c + echo "MINIO_BIN=$(go env GOPATH)/bin/minio" >> "$GITHUB_ENV" + + - name: Test + working-directory: e2e run: | set -euo pipefail - go test -race -json -v -run '^TestFloci${{ matrix.provider }}$' ./... 2>&1 | gotestfmt + go test -race -json -v -run '^Test${{ matrix.test }}$' ./... 2>&1 | gotestfmt diff --git a/README.md b/README.md index e379643..158d20b 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,16 @@ A GORM-style storage wrapper with unified API for different cloud storage backen `storage.New(conf)` returns a `Storage` for S3 and S3-compatible stores, GCS, Azure, OCI, AliOSS or a local directory. Every call takes a context. -- `Bucket` (`storage/objects`): `PutObject`, `UploadObject` (can refuse to replace an object, `PutOptions.IfAbsent`), `GetObject`, `GetObjectRange`, `DownloadObject` (to any `io.Writer`), `ObjectExists`, `ListObjects`, `DeleteObjects`, `GeneratePresignedUrl`. A missing object is `ErrObjectNotFound` and a refused write `ErrObjectExists` (`errors.Is`). -- `Storage` is `Bucket` plus `UploadData`, `UploadFile` and `DownloadFile`, which publish and fetch whole files and return their location. +- `Bucket` (`storage/objects`): `PutObject`, `UploadObject` (can refuse to replace an object, `PutOptions.IfAbsent`), `GetObject`, `GetObjectRange`, `DownloadObject` (to any `io.Writer`), `ObjectExists`, `ListObjects`, `DeleteObjects`, `PresignedURL`. A missing object is `ErrNotFound` and a refused write `ErrExists` (`errors.Is`). +- `Storage` is `Bucket` plus `UploadFile` and `DownloadFile`, which publish and fetch whole files. +- Every backend follows S3's semantics where providers differ. `DeleteObjects` of a missing path succeeds, as S3's delete does, and fails while any path it was given still holds an object, so a retry after a partial failure is safe. - `storage/objects` imports no provider SDK, so a fake or a library can implement or accept the interfaces without pulling in every SDK. -- The clients this package builds time out a connection that stalls before answering, never a transfer in progress; the context bounds the call. +- The clients this package builds cut a connection that moves no bytes for 5 minutes, never a slow but live transfer; the context bounds the call. ## Tests -`go test ./...` runs every backend except AliOSS end to end against [floci](https://github.com/floci-io) emulators in Docker (S3, Azure Blob, GCS, OCI Object Storage), plus the local directory. Without Docker those tests skip, except when `CI` is set. The provider tests (`TestS3`, `TestGCP`, ...) reach real accounts when their environment variables are set; AliOSS has no emulator and is covered only that way. +The suites live in the `e2e` module, so this module's users do not inherit Docker's dependencies. + +- `go test ./...` here runs the unit tests. +- `cd e2e && go test ./...` runs every backend except AliOSS end to end: against [floci](https://github.com/floci-io) emulators in Docker (S3, Azure Blob, GCS, OCI Object Storage), against MinIO for S3's conditional writes (`MINIO_BIN` or `minio` on `PATH`), and on the local directory. Without Docker or MinIO those tests skip, except when `CI` is set. +- The provider tests (`TestS3`, `TestGCP`, ...) reach real accounts when their environment variables are set; AliOSS has no emulator and is covered only that way. diff --git a/alioss.go b/alioss.go index 453f789..f7032ae 100644 --- a/alioss.go +++ b/alioss.go @@ -72,13 +72,6 @@ func NewAliOSS(conf *AliOSSConfig) (Storage, error) { }, nil } -func (s *aliOSSStorage) UploadData(ctx context.Context, data []byte, storagePath, contentType string) (string, int64, error) { - if err := s.PutObject(ctx, storagePath, data, PutOptions{ContentType: contentType}); err != nil { - return "", 0, err - } - return s.location(storagePath), int64(len(data)), nil -} - func (s *aliOSSStorage) UploadFile(ctx context.Context, filepath, storagePath, contentType string) (string, int64, error) { info, err := os.Stat(filepath) if err != nil { @@ -108,7 +101,7 @@ func (s *aliOSSStorage) DownloadFile(ctx context.Context, filepath, storagePath return info.Size(), nil } -func (s *aliOSSStorage) presignedUrl(storagePath string, expiration time.Duration) (string, error) { +func (s *aliOSSStorage) presignedURL(storagePath string, expiration time.Duration) (string, error) { u, err := s.bucket.SignURL(storagePath, oss.HTTPGet, int64(expiration.Seconds())) if err != nil { return "", wrapAliOSSError(err) @@ -225,8 +218,8 @@ func (s *aliOSSStorage) DeleteObjects(ctx context.Context, storagePaths []string return nil } -func (s *aliOSSStorage) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { - return s.presignedUrl(storagePath, expiration) +func (s *aliOSSStorage) PresignedURL(_ context.Context, storagePath string, expiration time.Duration) (string, error) { + return s.presignedURL(storagePath, expiration) } func wrapAliOSSObjectError(err error) error { @@ -238,9 +231,9 @@ func wrapAliOSSObjectError(err error) error { switch { // A HEAD answers a missing key with a bare 404. case svcErr.Code == "NoSuchKey" || (svcErr.StatusCode == http.StatusNotFound && svcErr.Code == ""): - return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapAliOSSError(err)) + return fmt.Errorf("%w: %w", ErrNotFound, wrapAliOSSError(err)) case svcErr.Code == "FileAlreadyExists": - return fmt.Errorf("%w: %w", ErrObjectExists, wrapAliOSSError(err)) + return fmt.Errorf("%w: %w", ErrExists, wrapAliOSSError(err)) } } return wrapAliOSSError(err) diff --git a/azure.go b/azure.go index 0b61d61..4ccdd53 100644 --- a/azure.go +++ b/azure.go @@ -23,6 +23,7 @@ import ( "net/url" "os" "path" + "strings" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -54,7 +55,8 @@ const azureParallelism = 16 type azureBLOBStorage struct { conf *AzureConfig client *azblob.Client - serviceUrl string + cred *azblob.SharedKeyCredential + serviceURL string } func NewAzure(conf *AzureConfig) (Storage, error) { @@ -63,11 +65,11 @@ func NewAzure(conf *AzureConfig) (Storage, error) { return nil, err } - serviceUrl := azureServiceURL(conf) - if _, err := url.Parse(serviceUrl); err != nil { + serviceURL := azureServiceURL(conf) + if _, err := url.Parse(serviceURL); err != nil { return nil, fmt.Errorf("azure endpoint: %w", err) } - client, err := azblob.NewClientWithSharedKeyCredential(serviceUrl, cred, nil) + client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, nil) if err != nil { return nil, err } @@ -75,7 +77,8 @@ func NewAzure(conf *AzureConfig) (Storage, error) { return &azureBLOBStorage{ conf: conf, client: client, - serviceUrl: serviceUrl, + cred: cred, + serviceURL: serviceURL, }, nil } @@ -98,13 +101,6 @@ func (s *azureBLOBStorage) blobURL(storagePath string) *url.URL { return u } -func (s *azureBLOBStorage) UploadData(ctx context.Context, data []byte, storagePath, contentType string) (string, int64, error) { - if err := s.PutObject(ctx, storagePath, data, PutOptions{ContentType: contentType}); err != nil { - return "", 0, err - } - return s.location(storagePath), int64(len(data)), nil -} - func (s *azureBLOBStorage) UploadFile(ctx context.Context, filepath, storagePath, contentType string) (string, int64, error) { file, err := os.Open(filepath) if err != nil { @@ -161,7 +157,7 @@ func (s *azureBLOBStorage) presignedUrlWithToken(ctx context.Context, storagePat now := time.Now().UTC() exp := now.Add(expiration) - svcClient, err := service.NewClient(s.serviceUrl, s.conf.TokenCredential, nil) + svcClient, err := service.NewClient(s.serviceURL, s.conf.TokenCredential, nil) if err != nil { return "", err } @@ -191,36 +187,50 @@ func (s *azureBLOBStorage) presignedUrlWithToken(ctx context.Context, storagePat return loc.String(), nil } +// PutObject sends data up to 256 MiB in one Put Blob, IfAbsent as its +// condition, and larger data in blocks. func (s *azureBLOBStorage) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { - return s.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) + headers, conditions := azureWriteOptions(opts) + _, err := s.client.UploadBuffer(ctx, s.conf.ContainerName, storagePath, data, &azblob.UploadBufferOptions{ + BlockSize: azureBlockSize, + Concurrency: azureParallelism, + HTTPHeaders: headers, + AccessConditions: conditions, + }) + return wrapAzureObjectError(err) } -// UploadObject stages blocks and commits them; IfAbsent is a condition on the -// commit, so it is atomic here. The check before staging only spares a refused -// upload its traffic. -func (s *azureBLOBStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { - if opts.IfAbsent { - exists, err := s.ObjectExists(ctx, storagePath) - if err != nil { - return err - } - if exists { - return ErrObjectExists - } - } - uploadOpts := &azblob.UploadStreamOptions{BlockSize: azureBlockSize, Concurrency: azureParallelism} +func azureWriteOptions(opts PutOptions) (*blob.HTTPHeaders, *blob.AccessConditions) { + var headers *blob.HTTPHeaders if opts.ContentType != "" { - uploadOpts.HTTPHeaders = &blob.HTTPHeaders{BlobContentType: to.Ptr(opts.ContentType)} + headers = &blob.HTTPHeaders{BlobContentType: to.Ptr(opts.ContentType)} } + var conditions *blob.AccessConditions if opts.IfAbsent { - uploadOpts.AccessConditions = &blob.AccessConditions{ + conditions = &blob.AccessConditions{ ModifiedAccessConditions: &blob.ModifiedAccessConditions{IfNoneMatch: to.Ptr(azcore.ETagAny)}, } } - _, err := s.client.UploadStream(ctx, s.conf.ContainerName, storagePath, body, uploadOpts) + return headers, conditions +} + +// UploadObject stages blocks and commits them; IfAbsent is a condition on the +// commit, so it is atomic here. The check before staging only spares a refused +// upload its traffic. +func (s *azureBLOBStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + if opts.IfAbsent && presentBeforeUpload(ctx, s, storagePath) { + return ErrExists + } + headers, conditions := azureWriteOptions(opts) + _, err := s.client.UploadStream(ctx, s.conf.ContainerName, storagePath, body, &azblob.UploadStreamOptions{ + BlockSize: azureBlockSize, + Concurrency: azureParallelism, + HTTPHeaders: headers, + AccessConditions: conditions, + }) // A racing upload's commit discards this one's staged blocks. if opts.IfAbsent && bloberror.HasCode(err, bloberror.InvalidBlockList) { - return fmt.Errorf("%w: %w", ErrObjectExists, wrapAzureError(err)) + return fmt.Errorf("%w: %w", ErrExists, wrapAzureError(err)) } return wrapAzureObjectError(err) } @@ -259,17 +269,37 @@ func (s *azureBLOBStorage) download(ctx context.Context, storagePath string, w i return n, wrapAzureObjectError(err) } +// ObjectExists reports a directory of an account with a hierarchical namespace +// (ADLS Gen2) as no object, as a store reports a prefix. func (s *azureBLOBStorage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { blobClient := s.client.ServiceClient().NewContainerClient(s.conf.ContainerName).NewBlobClient(storagePath) - _, err := blobClient.GetProperties(ctx, nil) + props, err := blobClient.GetProperties(ctx, nil) if bloberror.HasCode(err, bloberror.BlobNotFound) { return false, nil } - return err == nil, wrapAzureObjectError(err) + if err != nil { + return false, wrapAzureObjectError(err) + } + return !azureFolder(props.Metadata), nil +} + +// azureFolder reports the metadata a hierarchical namespace gives its +// directories. Metadata keys come back in any case. +func azureFolder(metadata map[string]*string) bool { + for k, v := range metadata { + if strings.EqualFold(k, "hdi_isfolder") && v != nil && strings.EqualFold(*v, "true") { + return true + } + } + return false } func (s *azureBLOBStorage) ListObjects(ctx context.Context, prefix string) ([]string, error) { - pager := s.client.NewListBlobsFlatPager(s.conf.ContainerName, &azblob.ListBlobsFlatOptions{Prefix: &prefix}) + // With metadata, to leave out a hierarchical namespace's directories. + pager := s.client.NewListBlobsFlatPager(s.conf.ContainerName, &azblob.ListBlobsFlatOptions{ + Prefix: &prefix, + Include: azblob.ListBlobsInclude{Metadata: true}, + }) var keys []string for pager.More() { page, err := pager.NextPage(ctx) @@ -277,7 +307,9 @@ func (s *azureBLOBStorage) ListObjects(ctx context.Context, prefix string) ([]st return nil, wrapAzureObjectError(err) } for _, item := range page.Segment.BlobItems { - keys = append(keys, *item.Name) + if item.Name != nil && !azureFolder(item.Metadata) { + keys = append(keys, *item.Name) + } } } return keys, nil @@ -285,9 +317,9 @@ func (s *azureBLOBStorage) ListObjects(ctx context.Context, prefix string) ([]st func (s *azureBLOBStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { for _, p := range storagePaths { - _, err := s.client.DeleteBlob(ctx, s.conf.ContainerName, p, &azblob.DeleteBlobOptions{ - DeleteSnapshots: to.Ptr(azblob.DeleteSnapshotsOptionTypeInclude), - }) + // A blob with snapshots is not deleted, as on Storage before: deleting + // someone's snapshots along with it is not this call's to decide. + _, err := s.client.DeleteBlob(ctx, s.conf.ContainerName, p, nil) if err != nil && !bloberror.HasCode(err, bloberror.BlobNotFound) { return wrapAzureObjectError(err) } @@ -295,15 +327,31 @@ func (s *azureBLOBStorage) DeleteObjects(ctx context.Context, storagePaths []str return nil } -// GeneratePresignedUrl signs with the account key unless a token credential is +// PresignedURL signs with the account key unless a token credential is // set, in which case it signs with a user delegation key as Storage does. -func (s *azureBLOBStorage) GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { +func (s *azureBLOBStorage) PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { if s.conf.TokenCredential != nil { return s.presignedUrlWithToken(ctx, storagePath, expiration) } - b := s.client.ServiceClient().NewContainerClient(s.conf.ContainerName).NewBlobClient(storagePath) - u, err := b.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(expiration), nil) - return u, wrapAzureError(err) + // HTTPS only, as the user delegation SAS, unless the endpoint itself is + // plain HTTP, as an emulator's. + protocol := sas.ProtocolHTTPS + if strings.HasPrefix(s.serviceURL, "http://") { + protocol = sas.ProtocolHTTPSandHTTP + } + qp, err := sas.BlobSignatureValues{ + Protocol: protocol, + ExpiryTime: time.Now().UTC().Add(expiration), + Permissions: (&sas.BlobPermissions{Read: true}).String(), + ContainerName: s.conf.ContainerName, + BlobName: storagePath, + }.SignWithSharedKey(s.cred) + if err != nil { + return "", err + } + u := s.blobURL(storagePath) + u.RawQuery = qp.Encode() + return u.String(), nil } func wrapAzureObjectError(err error) error { @@ -311,9 +359,9 @@ func wrapAzureObjectError(err error) error { case err == nil: return nil case bloberror.HasCode(err, bloberror.BlobNotFound): - return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapAzureError(err)) + return fmt.Errorf("%w: %w", ErrNotFound, wrapAzureError(err)) case bloberror.HasCode(err, bloberror.BlobAlreadyExists, bloberror.ConditionNotMet): - return fmt.Errorf("%w: %w", ErrObjectExists, wrapAzureError(err)) + return fmt.Errorf("%w: %w", ErrExists, wrapAzureError(err)) } return wrapAzureError(err) } diff --git a/config/config.go b/config/config.go index 1abe09d..7a2c468 100644 --- a/config/config.go +++ b/config/config.go @@ -52,8 +52,10 @@ type GCPConfig struct { CredentialsJSON string `yaml:"credentials_json,omitempty"` // (env GOOGLE_APPLICATION_CREDENTIALS) Bucket string `yaml:"bucket,omitempty"` ProxyConfig *ProxyConfig `yaml:"proxy_config,omitempty"` - // Endpoint points the client at an emulator, e.g. http://localhost:4588. - // Requests to it are unauthenticated; CredentialsJSON still signs URLs. + // Endpoint replaces https://storage.googleapis.com, e.g. a Private Service + // Connect endpoint. An http:// one is taken for an emulator, such as + // http://localhost:4588, and gets no credentials; CredentialsJSON still + // signs URLs. Endpoint string `yaml:"endpoint,omitempty"` } @@ -88,8 +90,8 @@ type S3Config struct { AccessKey string `yaml:"access_key,omitempty"` Secret string `yaml:"secret,omitempty"` SessionToken string `yaml:"session_token,omitempty"` - AssumeRoleArn string `yaml:"assume_role_arn,omitempty"` // ARN of the role to assume for file upload. Egress will make an AssumeRole API call using the provided access_key and secret to assume that role - AssumeRoleExternalId string `yaml:"assume_role_external_id,omitempty"` // ExternalID to use when assuming role for upload + AssumeRoleARN string `yaml:"assume_role_arn,omitempty"` // ARN of the role to assume for file upload. Egress will make an AssumeRole API call using the provided access_key and secret to assume that role + AssumeRoleExternalID string `yaml:"assume_role_external_id,omitempty"` // ExternalID to use when assuming role for upload Region string `yaml:"region,omitempty"` Endpoint string `yaml:"endpoint,omitempty"` Bucket string `yaml:"bucket,omitempty"` @@ -108,7 +110,7 @@ type S3Config struct { func (*S3Config) storageConfig() {} type ProxyConfig struct { - Url string `yaml:"url,omitempty"` + URL string `yaml:"url,omitempty"` Username string `yaml:"username,omitempty"` Password string `yaml:"password,omitempty"` } diff --git a/contract_test.go b/contract_test.go new file mode 100644 index 0000000..95776ef --- /dev/null +++ b/contract_test.go @@ -0,0 +1,273 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage_test + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "fmt" + "math/big" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/livekit/storage" +) + +// These hold what a caller's environment relies on, which the emulators, +// reached with any credentials over plain HTTP, cannot show. + +// A custom CA bundle, as for a self-hosted store behind a private CA, can only +// be added to the AWS SDK's own HTTP client. +func TestS3CustomCABundle(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "ca.pem") + require.NoError(t, os.WriteFile(bundle, selfSignedCert(t), 0o600)) + t.Setenv("AWS_CA_BUNDLE", bundle) + + _, err := storage.New(&storage.S3Config{ + AccessKey: "key", + Secret: "secret", + Region: "us-east-1", + Endpoint: "https://127.0.0.1:1", + Bucket: "bucket", + }) + require.NoError(t, err) +} + +// Building a GCS client must not change the process's default transport, +// which every http.Get in the importing service uses. +func TestGCPLeavesDefaultTransport(t *testing.T) { + orig := http.DefaultTransport + before := orig.(*http.Transport).Clone() + + _, err := storage.New(&storage.GCPConfig{ + Bucket: "bucket", + Endpoint: "http://127.0.0.1:1", + ProxyConfig: &storage.ProxyConfig{URL: "http://127.0.0.1:2", Username: "u", Password: "p"}, + }) + require.NoError(t, err) + + require.Same(t, orig, http.DefaultTransport) + after := orig.(*http.Transport) + require.Nil(t, after.ProxyConnectHeader) + require.Equal(t, reflect.ValueOf(before.DialContext).Pointer(), reflect.ValueOf(after.DialContext).Pointer(), "dialer") + require.Equal(t, reflect.ValueOf(before.Proxy).Pointer(), reflect.ValueOf(after.Proxy).Pointer(), "proxy") +} + +// The proxy goes on the transport the client sends with: where it is dropped, +// traffic leaves around the proxy a network requires. +func TestProxyCarriesRequests(t *testing.T) { + for name, conf := range map[string]func(proxy *storage.ProxyConfig) storage.Config{ + "S3": func(proxy *storage.ProxyConfig) storage.Config { + return &storage.S3Config{ + AccessKey: "key", + Secret: "secret", + Region: "us-east-1", + Endpoint: "http://store.invalid", + ForcePathStyle: true, + Bucket: "bucket", + ProxyConfig: proxy, + } + }, + "GCP": func(proxy *storage.ProxyConfig) storage.Config { + return &storage.GCPConfig{Bucket: "bucket", Endpoint: "http://store.invalid", ProxyConfig: proxy} + }, + } { + t.Run(name, func(t *testing.T) { + proxied := make(chan string, 16) + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proxied <- r.Host + http.Error(w, "no", http.StatusForbidden) + })) + t.Cleanup(proxy.Close) + s, err := storage.New(conf(&storage.ProxyConfig{URL: proxy.URL})) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, _ = s.ObjectExists(ctx, "path") + select { + case host := <-proxied: + require.Equal(t, "store.invalid", host) + default: + t.Fatal("the request went around the proxy") + } + }) + } +} + +// Only an http:// endpoint, an emulator's, drops credentials: a private one, +// such as Private Service Connect, is still sent a token. +func TestGCPEndpointKeepsCredentials(t *testing.T) { + for endpoint, authenticated := range map[string]bool{ + "https://127.0.0.1:1": true, + "http://127.0.0.1:1": false, + } { + t.Run(endpoint, func(t *testing.T) { + var tokens atomic.Int32 + tokenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tokens.Add(1) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"access_token":"token","token_type":"Bearer","expires_in":3600}`) + })) + t.Cleanup(tokenServer.Close) + s, err := storage.New(&storage.GCPConfig{ + CredentialsJSON: serviceAccount(t, tokenServer.URL), + Bucket: "bucket", + Endpoint: endpoint, + }) + require.NoError(t, err) + + // The token is fetched before the request is sent, which then fails; + // GCS retries a refused connection until the context ends. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, _ = s.ObjectExists(ctx, "path") + require.Equal(t, authenticated, tokens.Load() > 0) + }) + } +} + +// Local objects get os.Create's mode, 0666 less the umask: egress records to +// a mounted volume a host user reads. +func TestLocalFileMode(t *testing.T) { + dir := t.TempDir() + s, err := storage.New(&storage.LocalConfig{StorageDir: dir}) + require.NoError(t, err) + ctx := context.Background() + + require.NoError(t, s.PutObject(ctx, "put", []byte("x"), storage.PutOptions{IfAbsent: true})) + file := filepath.Join(t.TempDir(), "file") + require.NoError(t, os.WriteFile(file, []byte("x"), 0o600)) + _, _, err = s.UploadFile(ctx, file, "file", "text/plain") + require.NoError(t, err) + for _, name := range []string{"put", "file"} { + info, err := os.Stat(filepath.Join(dir, name)) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o666)&^umask(t), info.Mode().Perm(), name) + } +} + +// An S3 bulk delete answers 200 and lists the keys it could not delete. A key +// already gone is deleted; any other is still there and must fail the call. +func TestS3DeleteObjectsKeyErrors(t *testing.T) { + for name, tc := range map[string]struct { + code string + deleted bool + }{ + "already gone": {code: "NoSuchKey", deleted: true}, + "access denied": {code: "AccessDenied"}, + "internal": {code: "InternalError"}, + } { + t.Run(name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `b%sm`, tc.code) + })) + t.Cleanup(srv.Close) + s, err := storage.New(&storage.S3Config{ + AccessKey: "key", + Secret: "secret", + Region: "us-east-1", + Endpoint: srv.URL, + ForcePathStyle: true, + Bucket: "bucket", + }) + require.NoError(t, err) + + err = s.DeleteObjects(context.Background(), []string{"a", "b"}) + if tc.deleted { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tc.code) + } + }) + } +} + +// Local keys drop a leading "/", in listings too, as egress's paths have one; +// a StorageDir that is a symlink lists what it points to. +func TestLocalKeys(t *testing.T) { + dir := t.TempDir() + link := filepath.Join(t.TempDir(), "link") + require.NoError(t, os.Symlink(dir, link)) + ctx := context.Background() + + for _, storageDir := range []string{dir, link} { + s, err := storage.New(&storage.LocalConfig{StorageDir: storageDir}) + require.NoError(t, err) + require.NoError(t, s.PutObject(ctx, "/rec/a.mp4", []byte("x"), storage.PutOptions{})) + for _, prefix := range []string{"", "/", "rec/", "/rec/", "/rec"} { + keys, err := s.ListObjects(ctx, prefix) + require.NoError(t, err, prefix) + require.Equal(t, []string{"rec/a.mp4"}, keys, prefix) + } + require.NoError(t, s.DeleteObjects(ctx, []string{"/rec/a.mp4"})) + } +} + +// umask is the process's, read the way os.Create applies it. +func umask(t *testing.T) os.FileMode { + f, err := os.Create(filepath.Join(t.TempDir(), "probe")) + require.NoError(t, err) + defer f.Close() + info, err := f.Stat() + require.NoError(t, err) + return 0o666 &^ info.Mode().Perm() +} + +func serviceAccount(t *testing.T, tokenURI string) string { + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + der, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(t, err) + b, err := json.Marshal(map[string]string{ + "type": "service_account", + "client_email": "livekit@test.iam.gserviceaccount.com", + "private_key": string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})), + "token_uri": tokenURI, + }) + require.NoError(t, err) + return string(b) +} + +func selfSignedCert(t *testing.T) []byte { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} diff --git a/e2e/ci_test.go b/e2e/ci_test.go new file mode 100644 index 0000000..80bce8b --- /dev/null +++ b/e2e/ci_test.go @@ -0,0 +1,56 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// Each emulator test is a check of its own in CI, from the e2e job's matrix; +// one left out of it would run nowhere. +func TestWorkflowRunsEveryEmulatorTest(t *testing.T) { + workflow, err := os.ReadFile("../.github/workflows/test.yaml") + require.NoError(t, err) + _, rest, ok := strings.Cut(string(workflow), "test: [") + require.True(t, ok, "no e2e matrix") + list, _, _ := strings.Cut(rest, "]") + var matrix []string + for _, name := range strings.Split(list, ",") { + matrix = append(matrix, "Test"+strings.TrimSpace(name)) + } + + files, err := filepath.Glob("*_test.go") + require.NoError(t, err) + var tests []string + for _, file := range files { + f, err := parser.ParseFile(token.NewFileSet(), file, nil, 0) + require.NoError(t, err) + for _, decl := range f.Decls { + fn, ok := decl.(*ast.FuncDecl) + if ok && (strings.HasPrefix(fn.Name.Name, "TestFloci") || fn.Name.Name == "TestMinIO") { + tests = append(tests, fn.Name.Name) + } + } + } + require.ElementsMatch(t, tests, matrix) +} diff --git a/e2e/floci_test.go b/e2e/floci_test.go new file mode 100644 index 0000000..4bf538d --- /dev/null +++ b/e2e/floci_test.go @@ -0,0 +1,356 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + gcs "cloud.google.com/go/storage" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/objectstorage" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" + "github.com/stretchr/testify/require" + "google.golang.org/api/option" + + "github.com/livekit/storage" +) + +// The emulators run with the tests on any machine with Docker, and must in CI. +// Each holds its state in memory and is removed when its test ends. +// Cleanups take a context of their own: the test's is canceled by then. + +const flociBucket = "livekit" + +func TestFlociS3(t *testing.T) { + endpoint := floci(t, "floci/floci:2.3.0@sha256:e0b292df4dc9559b7b1bf4f51bdda4bb8e269276236438cd03cae661260ea8a9", "4566") + conf := &storage.S3Config{ + AccessKey: "test", + Secret: "test", + Region: "us-east-1", + Endpoint: endpoint, + ForcePathStyle: true, + Bucket: flociBucket, + } + admin := s3.New(s3.Options{ + Region: conf.Region, + BaseEndpoint: &conf.Endpoint, + UsePathStyle: true, + Credentials: credentials.NewStaticCredentialsProvider(conf.AccessKey, conf.Secret, ""), + }) + _, err := admin.CreateBucket(t.Context(), &s3.CreateBucketInput{Bucket: aws.String(flociBucket)}) + require.NoError(t, err) + + testProvider(t, endpoint, func(endpoint string) storage.Config { + c := *conf + c.Endpoint = endpoint + return &c + }, objectsSuite{bulk: true, multipartIfAbsentRaces: true, commit: s3Commit}) +} + +// flociAzureKey is Azurite's well-known development key; floci-az accepts any. +const flociAzureKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + +func TestFlociAzure(t *testing.T) { + // An IP host: the SDK parses an account from the path only then. + endpoint := strings.Replace(floci(t, "floci/floci-az:0.15.0@sha256:530635d603223e120adbf9221a97560cffc6991d65fa0c72d0f79d1097974c89", "4577"), "localhost", "127.0.0.1", 1) + // floci-az gives devstoreaccount1 a hierarchical namespace (ADLS Gen2), + // where a path's directories are entries of their own, and other accounts + // a flat one: both are common on Azure. + for name, account := range map[string]string{"hierarchical namespace": "devstoreaccount1", "flat namespace": "flat"} { + t.Run(name, func(t *testing.T) { + conf := &storage.AzureConfig{ + AccountName: account, + AccountKey: flociAzureKey, + ContainerName: flociBucket, + Endpoint: endpoint + "/" + account + "/", + } + cred, err := azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey) + require.NoError(t, err) + admin, err := azblob.NewClientWithSharedKeyCredential(conf.Endpoint, cred, nil) + require.NoError(t, err) + _, err = admin.CreateContainer(t.Context(), flociBucket, nil) + require.NoError(t, err) + + testProvider(t, endpoint, func(endpoint string) storage.Config { + c := *conf + c.Endpoint = endpoint + "/" + account + "/" + return &c + }, objectsSuite{bulk: true, multipartIgnoresIfAbsent: true, commit: func(r *http.Request) (bool, bool) { + return r.Method == http.MethodPut && r.URL.Query().Get("comp") == "blocklist", r.Header.Get("If-None-Match") == "*" + }}) + }) + } +} + +func TestFlociGCP(t *testing.T) { + endpoint := floci(t, "floci/floci-gcp:0.11.0@sha256:eae9683d806ceb643bcb36aee97cdb8cad13c3e2959cd053bf40e481994fde12", "4588") + conf := &storage.GCPConfig{ + // A service account of its own, to sign URLs. + CredentialsJSON: gcpServiceAccount(t), + Bucket: flociBucket, + Endpoint: endpoint, + } + admin, err := gcs.NewClient(t.Context(), option.WithEndpoint(endpoint+"/storage/v1/"), option.WithoutAuthentication()) + require.NoError(t, err) + t.Cleanup(func() { _ = admin.Close() }) + require.NoError(t, admin.Bucket(flociBucket).Create(t.Context(), "floci-local", nil)) + + testProvider(t, endpoint, func(endpoint string) storage.Config { + c := *conf + c.Endpoint = endpoint + return &c + }, objectsSuite{bulk: true, commit: func(r *http.Request) (bool, bool) { + // A resumable upload takes its condition at the request that starts it, + // the one without an upload_id. + q := r.URL.Query() + return r.Method == http.MethodPost && q.Get("uploadType") == "resumable" && !q.Has("upload_id"), q.Get("ifGenerationMatch") == "0" + }}) +} + +// flociTenancy is floci-oci's tenancy, which is also the root compartment. +const flociTenancy = "ocid1.tenancy.oc1..flocilocaltenancy0000000000000000000000000000000000000000" + +func TestFlociOCI(t *testing.T) { + endpoint := floci(t, "floci/floci-oci:0.5.0@sha256:3cb5af867fb2a4e8b60f8cf6fd0acf4873108f01d481c0aac23c01ca00f935ae", "4599") + // floci-oci reads the tenancy and user from the request signature without + // verifying it, so any key signs. + conf := &storage.OCIConfig{ + TenancyOCID: flociTenancy, + UserOCID: "ocid1.user.oc1..livekit", + Fingerprint: "00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00", + PrivateKey: rsaKeyPEM(t), + Region: "us-ashburn-1", + Endpoint: endpoint, + Bucket: flociBucket, + } + provider := common.NewRawConfigurationProvider(conf.TenancyOCID, conf.UserOCID, conf.Region, conf.Fingerprint, conf.PrivateKey, nil) + admin, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(provider) + require.NoError(t, err) + admin.Host = endpoint + ns, err := admin.GetNamespace(t.Context(), objectstorage.GetNamespaceRequest{}) + require.NoError(t, err) + _, err = admin.CreateBucket(t.Context(), objectstorage.CreateBucketRequest{ + NamespaceName: ns.Value, + CreateBucketDetails: objectstorage.CreateBucketDetails{ + Name: common.String(flociBucket), + CompartmentId: common.String(flociTenancy), + }, + }) + require.NoError(t, err) + + testProvider(t, endpoint, func(endpoint string) storage.Config { + c := *conf + c.Endpoint = endpoint + // The namespace lookup is a read, so a write-only deployment names it. + c.Namespace = *ns.Value + return &c + }, objectsSuite{bulk: true, singlePartIfAbsentRaces: true, multipartIgnoresIfAbsent: true, commit: func(r *http.Request) (bool, bool) { + return r.Method == http.MethodPost && r.URL.Query().Has("uploadId"), r.Header.Get("If-None-Match") == "*" + }}) +} + +// testProvider runs every suite against the store at endpoint. at gives the +// provider's config for an endpoint, so the suites can reach the store through +// proxies that refuse some requests. +func testProvider(t *testing.T, endpoint string, at func(endpoint string) storage.Config, suite objectsSuite) { + s, err := storage.New(at(endpoint)) + require.NoError(t, err) + writeOnly, err := storage.New(at(refuse(t, endpoint, isRead))) + require.NoError(t, err) + noDeletes, err := storage.New(at(refuse(t, endpoint, isDelete))) + require.NoError(t, err) + t.Run("Storage", func(t *testing.T) { testStorage(t, s) }) + t.Run("Bucket", func(t *testing.T) { testObjectsWith(t, s, suite) }) + t.Run("write only", func(t *testing.T) { testWriteOnly(t, s, writeOnly, suite) }) + t.Run("deletes refused", func(t *testing.T) { testDeletesRefused(t, s, noDeletes) }) + if suite.commit != nil { + t.Run("multipart IfAbsent on the wire", func(t *testing.T) { + testConditionalCommit(t, s, endpoint, at, suite.commit) + }) + } +} + +func s3Commit(r *http.Request) (bool, bool) { + return r.Method == http.MethodPost && r.URL.Query().Has("uploadId"), r.Header.Get("If-None-Match") == "*" +} + +// testConditionalCommit uploads a multipart object IfAbsent through a proxy +// that sees every request: the one that makes the object must carry the +// condition, whether or not the store enforces it. +func testConditionalCommit(t *testing.T, s storage.Storage, endpoint string, at func(string) storage.Config, commit func(*http.Request) (bool, bool)) { + ctx := t.Context() + var commits, conditional atomic.Int32 + watched, err := storage.New(at(refuse(t, endpoint, func(r *http.Request) bool { + if isCommit, isConditional := commit(r); isCommit { + commits.Add(1) + if isConditional { + conditional.Add(1) + } + } + return false + }))) + require.NoError(t, err) + + path := fmt.Sprintf("conditional-commit-%d", time.Now().UnixNano()) + t.Cleanup(func() { _ = s.DeleteObjects(context.Background(), []string{path}) }) + body := bytes.Repeat([]byte("0123456789abcdef"), (17<<20)/16) + require.NoError(t, watched.UploadObject(ctx, path, bytes.NewReader(body), storage.PutOptions{IfAbsent: true})) + require.Positive(t, commits.Load(), "no commit request seen") + require.Equal(t, commits.Load(), conditional.Load(), "a commit went out without the condition") +} + +// testDeletesRefused deletes where the store refuses it: the object is still +// there, so the call must fail, missing keys alongside it or not. +func testDeletesRefused(t *testing.T, s, noDeletes storage.Storage) { + ctx := t.Context() + path := fmt.Sprintf("deletes-refused-%d", time.Now().UnixNano()) + require.NoError(t, s.PutObject(ctx, path, []byte("kept"), storage.PutOptions{})) + t.Cleanup(func() { _ = s.DeleteObjects(context.Background(), []string{path}) }) + + require.Error(t, noDeletes.DeleteObjects(ctx, []string{path + "-missing", path})) + exists, err := s.ObjectExists(ctx, path) + require.NoError(t, err) + require.True(t, exists) +} + +// testWriteOnly writes IfAbsent with credentials that may write but not read: +// any check before uploading fails, and the store's condition alone refuses +// the second upload. +func testWriteOnly(t *testing.T, s, writeOnly storage.Storage, suite objectsSuite) { + ctx := t.Context() + prefix := fmt.Sprintf("write-only-%d/", time.Now().UnixNano()) + bodies := map[string][]byte{"single part": []byte("written without reads")} + if !suite.multipartIgnoresIfAbsent { + bodies["multipart"] = bytes.Repeat([]byte("0123456789abcdef"), (17<<20)/16) + } + for name, body := range bodies { + t.Run(name, func(t *testing.T) { + path := prefix + name + t.Cleanup(func() { _ = s.DeleteObjects(context.Background(), []string{path}) }) + require.NoError(t, writeOnly.UploadObject(ctx, path, bytes.NewReader(body), storage.PutOptions{IfAbsent: true})) + err := writeOnly.UploadObject(ctx, path, bytes.NewReader(body), storage.PutOptions{IfAbsent: true}) + require.ErrorIs(t, err, storage.ErrExists) + _, err = writeOnly.GetObject(ctx, path) + require.Error(t, err, "reads must be denied") + got, err := s.GetObject(ctx, path) + require.NoError(t, err) + require.Equal(t, body, got) + }) + } +} + +func isRead(r *http.Request) bool { + return r.Method == http.MethodGet || r.Method == http.MethodHead +} + +// isDelete is a single delete, or an S3 or AliOSS bulk one (POST ?delete). +func isDelete(r *http.Request) bool { + return r.Method == http.MethodDelete || (r.Method == http.MethodPost && r.URL.Query().Has("delete")) +} + +// refuse is a proxy to endpoint that answers the requests matched with 403, +// like a bucket policy that withholds that permission. It sees every request. +func refuse(t *testing.T, endpoint string, matched func(*http.Request) bool) string { + target, err := url.Parse(endpoint) + require.NoError(t, err) + proxy := httputil.NewSingleHostReverseProxy(target) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if matched(r) { + http.Error(w, "refused", http.StatusForbidden) + return + } + proxy.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + return srv.URL +} + +// floci starts image, repo:tag@digest, and returns its http:// endpoint once +// it answers. A tag pushed again fails the test rather than change what it +// runs. It skips the test where Docker is unavailable, except in CI. +func floci(t *testing.T, image, port string) string { + ref, digest, _ := strings.Cut(image, "@") + repo, tag, _ := strings.Cut(ref, ":") + pool, err := dockertest.NewPool("") + if err == nil { + err = pool.Client.Ping() + } + if err != nil { + if os.Getenv("CI") != "" { + t.Fatalf("docker: %v", err) + } + t.Skipf("docker unavailable: %v", err) + } + + res, err := pool.RunWithOptions(&dockertest.RunOptions{Repository: repo, Tag: tag}, func(hc *docker.HostConfig) { + hc.AutoRemove = true + }) + require.NoError(t, err) + t.Cleanup(func() { _ = pool.Purge(res) }) + img, err := pool.Client.InspectImage(ref) + require.NoError(t, err) + require.Contains(t, img.RepoDigests, repo+"@"+digest, "%s is not the image pinned", ref) + + endpoint := "http://" + res.GetHostPort(port+"/tcp") + require.NoError(t, pool.Retry(func() error { + resp, err := pool.Client.HTTPClient.Get(endpoint) + if err == nil { + resp.Body.Close() + } + return err + })) + return endpoint +} + +func rsaKeyPEM(t *testing.T) string { + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + der, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(t, err) + return string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})) +} + +func gcpServiceAccount(t *testing.T) string { + b, err := json.Marshal(map[string]string{ + "type": "service_account", + "project_id": "floci-local", + "client_email": "livekit@floci-local.iam.gserviceaccount.com", + "private_key": rsaKeyPEM(t), + "token_uri": "https://oauth2.googleapis.com/token", + }) + require.NoError(t, err) + return string(b) +} diff --git a/e2e/go.mod b/e2e/go.mod new file mode 100644 index 0000000..8a0967e --- /dev/null +++ b/e2e/go.mod @@ -0,0 +1,130 @@ +module github.com/livekit/storage/e2e + +go 1.26.0 + +require ( + cloud.google.com/go/storage v1.69.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 + github.com/aws/aws-sdk-go-v2 v1.47.1 + github.com/aws/aws-sdk-go-v2/credentials v1.20.6 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 + github.com/joho/godotenv v1.5.1 + github.com/livekit/storage v0.0.0 + github.com/oracle/oci-go-sdk/v65 v65.126.1 + github.com/ory/dockertest/v3 v3.12.0 + github.com/stretchr/testify v1.12.1 + google.golang.org/api v0.300.0 +) + +require ( + cel.dev/expr v0.25.2 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.24.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.3.0 // indirect + cloud.google.com/go/compute/metadata v0.10.0 // indirect + cloud.google.com/go/iam v1.12.0 // indirect + cloud.google.com/go/monitoring v1.30.0 // indirect + dario.cat/mergo v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect + github.com/apache/arrow-go/v18 v18.7.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect + github.com/aws/aws-sdk-go-v2/config v1.33.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 // indirect + github.com/aws/smithy-go v1.28.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/containerd/continuity v0.4.5 // indirect + github.com/docker/cli v27.4.1+incompatible // indirect + github.com/docker/docker v27.1.1+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/goccy/go-json v0.10.6 // indirect + github.com/gofrs/flock v0.10.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect + github.com/google/s2a-go v0.1.10 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.22 // indirect + github.com/googleapis/gax-go/v2 v2.26.2 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/livekit/protocol v1.52.1 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/sys/user v0.3.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/opencontainers/runc v1.2.3 // indirect + github.com/pierrec/lz4/v4 v4.1.28 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect + github.com/puzpuzpuz/xsync/v4 v4.5.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/sony/gobreaker/v2 v2.4.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.28.0 // indirect + go.uber.org/zap/exp v0.3.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/oauth2 v0.37.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + golang.org/x/time v0.16.0 // indirect + google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 // indirect + google.golang.org/grpc v1.84.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) + +replace github.com/livekit/storage => ../ diff --git a/e2e/go.sum b/e2e/go.sum new file mode 100644 index 0000000..7338446 --- /dev/null +++ b/e2e/go.sum @@ -0,0 +1,361 @@ +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 h1:s6hzCXtND/ICdGPTMGk7C+/BFlr2Jg5GyH0NKf4XGXg= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM= +buf.build/go/protovalidate v1.2.0 h1:DQVrUWkmGTBij+kOYv/x2LLxwcLaGKMdzShj1/6/3H0= +buf.build/go/protovalidate v1.2.0/go.mod h1:7rYiQEhqvAipoazpVNBBH2S2f8bjG4huMVy1V2Yofn4= +buf.build/go/protoyaml v0.7.0 h1:z4oVoFicbpPefhT7WAykxUdfp0yEQlhMQ2mCZOY5V38= +buf.build/go/protoyaml v0.7.0/go.mod h1:+a0cavd0uMvirb87xdu2ZMMmjlIQoiH/N2Ich5MGSQ0= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.24.0 h1:UYMbF8otPZnLAkNJ5/LYQYOq0ARcJS1P4JqTeMKbCYU= +cloud.google.com/go/auth v0.24.0/go.mod h1:IFG/AMA1VWfuTrdbieEsB2GcpJyJV/phGAvogkOoPR4= +cloud.google.com/go/auth/oauth2adapt v0.3.0 h1:FY8oSZpCYoUNv6QxVODuMjQz4IlSOVeiQtZ08vLPz88= +cloud.google.com/go/auth/oauth2adapt v0.3.0/go.mod h1:7+2uCm7++XFO+/lN06c2HXpDXb/NMNn2/UwyBPbTnkk= +cloud.google.com/go/compute/metadata v0.10.0 h1:pyKMUQSwchgkIBBJGdILqQbs/BNJXqwSA7Ej6LAvvtY= +cloud.google.com/go/compute/metadata v0.10.0/go.mod h1:rGFHRrIif570kSibjFTMbt6/4/tzgJWFGI/HVol4GIk= +cloud.google.com/go/iam v1.12.0 h1:Aki3bX9aHUDKPHfnRJfDcTdVedvy6quGBQcTqx3DRXk= +cloud.google.com/go/iam v1.12.0/go.mod h1:FEZ4lXpADAC2AIpQY7LANNjjwyQ2jK439CI2VaD+sLY= +cloud.google.com/go/logging v1.19.0 h1:NCqhdVUg3wQ8Cobdf16FDSuTGi3+6+hdSBHrY5TsR6Q= +cloud.google.com/go/logging v1.19.0/go.mod h1:i40NZCHC9Gqvod4yE+yQfDWwlgwW/SrshkkGibCHxcA= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= +cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY= +cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM= +cloud.google.com/go/storage v1.69.0 h1:jAAMC1411HEh78nKsU0Zns+eFj3TnhjAWIhg5Ud/XBM= +cloud.google.com/go/storage v1.69.0/go.mod h1:PELYsxTYm2peE4mwLEC1+mS1dA/kUSRUxNv56rOy44g= +cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2 h1:utpeoEeZjd+A8J41zvoLsOOrqXHhX1Kx/X/tCW9dEYQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 h1:tQJdgQjbKKffEiMOKJmcQPkY01eFHRqWIZVefqP44nI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2/go.mod h1:kTkpJchXAdO7LGkKZP97a+Ou8TgaF4NT6MHwOIy5uSQ= +github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 h1:X/bTwe0qpvmvPPutc+LWo7T4cKMqDP9JHP4k+Co9ZB4= +github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0/go.mod h1:+kX7ahyFelQ42F3RScM2CFgHJljp0phc/FieKLYVpFI= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 h1:bN1gA3of5bXtbnLsRPrwfmbbe7A5UWFlcTHseujLnpc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0/go.mod h1:Yj5vHEz/aAepZGliRJsA6uvHAVAQyEwajq9ORCHPxzM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 h1:jLdiS1vO+XJFyDSWRHBx56r4s/NNtcl5J6KyCcWUX/w= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0/go.mod h1:8lmpHY+1VRoteiOwyrQMDt1YGXOrFKCz+1wJW7n3ODY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0 h1:cSjUzZ7KU8hicTgzaSv9NmSyM9fTVK3y5lsBUl3wOis= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 h1:RoO5+d7uCmDqovLrHCr2/BuViUXvdcrNxyNM1pN9dDQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= +github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= +github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= +github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= +github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= +github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= +github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug= +github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE= +github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw= +github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.11 h1:wgxEej5cFj+EfutuAPZPIFcMvQ3Doamt01lMtPoMpls= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.11/go.mod h1:dMcCQXtMtzVmEUO7YO+1xtYAvo8BcKgnN3Wppo8hbmA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4/go.mod h1:wm04I5DMuNVvZHFe/dHnUxincvNbbK7AiNBbYsQivek= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZjfhz0zXtjs6gdE8s= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.1/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM= +github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= +github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= +github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dennwc/iters v1.2.2 h1:XH2/Etihiy9ZvPOVCR+icQXeYlhbvS7k0qro4x/2qQo= +github.com/dennwc/iters v1.2.2/go.mod h1:M9KuuMBeyEXYTmB7EnI9SCyALFCmPWOIxn5W1L0CjGg= +github.com/docker/cli v27.4.1+incompatible h1:VzPiUlRJ/xh+otB75gva3r05isHMo5wXDfPRi5/b4hI= +github.com/docker/cli v27.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/gofrs/flock v0.10.0 h1:SHMXenfaB03KbroETaCMtbBg3Yn29v4w1r+tgy4ff4k= +github.com/gofrs/flock v0.10.0/go.mod h1:FirDy1Ing0mI2+kB6wk+vyyAH+e6xiE+EYA0jnzV9jc= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.29.0 h1:fEG+Ja3YRwNOqnQxTyJwoByAUAvTuxUGiro/jhrm4F4= +github.com/google/cel-go v0.29.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.10 h1:EMp+aOuXN6l8cE/gjF5Bt+vyZxsUuyCWe9chDWR/+uU= +github.com/google/s2a-go v0.1.10/go.mod h1:pz4tyvwXvJLLbyrkh6FW1eS2zPUXMaTmyNhYtyP2tNw= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.22 h1:NU4XpII6jD+Dxcot94fqjE+AfJoE/lQP9q3faYGzC/c= +github.com/googleapis/enterprise-certificate-proxy v0.3.22/go.mod h1:L3D/IQExI6LqEjBdXcZQ1WluSgigQmSwBboFstVPM4w= +github.com/googleapis/gax-go/v2 v2.26.2 h1:ydkmNXxj7bEmmeK5AihkKnWxyOyBR9TDebvp5L5izk8= +github.com/googleapis/gax-go/v2 v2.26.2/go.mod h1:sMKqnMesnKH+3wiRJROcttA+cJoZoGbZl1vDQ8XYtGk= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5ATTo469PQPkqzdoU7be46ryiCDO3boc= +github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= +github.com/livekit/protocol v1.52.1 h1:w8wRjRlj/57AJBxwomEzpEMzf+lT0qOoLQLO7awdTMU= +github.com/livekit/protocol v1.52.1/go.mod h1:t5CV/LCsWbI9zsSTHTJKDyLtByqvvVkioSP7jgqpIGA= +github.com/livekit/psrpc v0.8.0 h1:uo3x+DL8pK2OIaCxg/s9sLUO2cIocS5HwdPhhI+m+fQ= +github.com/livekit/psrpc v0.8.0/go.mod h1:6KnU8vULoizX0Er9NNfeKJDV8ug87j3fuo19M0bhj6Q= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= +github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/runc v1.2.3 h1:fxE7amCzfZflJO2lHXf4y/y8M1BoAqp+FVmG19oYB80= +github.com/opencontainers/runc v1.2.3/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM= +github.com/oracle/oci-go-sdk/v65 v65.126.1 h1:WmQ2Igq7/L/cHlR2jZnJkBC2UI51Ams4DJZW3CZGVBo= +github.com/oracle/oci-go-sdk/v65 v65.126.1/go.mod h1:YmvgbsnUfrsiJ410XAGQ1BRsrUYJJ7W3O5wUikmeI8w= +github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCyRCw= +github.com/ory/dockertest/v3 v3.12.0/go.mod h1:aKNDTva3cp8dwOWwb9cWuX84aH5akkxXRvO7KCwWVjE= +github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio= +github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 h1:S1hI5JiKP7883xBzZAr1ydcxrKNSVNm7+3+JwjxZEsg= +github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25/go.mod h1:ZQntvDG8TkPgljxtA0R9frDoND4QORU1VXz015N5Ks4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= +github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg= +github.com/sony/gobreaker/v2 v2.4.0/go.mod h1:pTyFJgcZ3h2tdQVLZZruK2C0eoFL1fb/G83wK1ZQl+s= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJXP61mNV3/7iuU= +github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0 h1:9jR0ZPRok9ryaOQ2Wx8rg5F7Aon59mxrqbVI60/vlBk= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0/go.mod h1:VSme3o2fvSg5bVg0dRzyHaj4Z5EVhG+g2Fde6LKzmQA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 h1:2yEATaop1/a1I4psnSLgWVPLWwCzkqWakgJy7xTDVy0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0/go.mod h1:D7J12YRapIekYyPWgGPlA/23pRmpSEZC5xJC/TTLI9U= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0 h1:dm9iyzn6tioYZtwqaiBSU0TSI8Yu/8dTIbfG0+B49DY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0/go.mod h1:xAvxYjYK28qvt+yu4BYZ/zMmAjwMXINXD6JiMyeB8iI= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= +go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= +go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= +golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.300.0 h1:2rvPV2bqnPuHOaF4gGOBiT1IIc6JVXYyHCkZeqdzjNk= +google.golang.org/api v0.300.0/go.mod h1:tKfTSDfK+0FlOVl8N30VL5fU5TuaEkJjvdyTIKNwzPg= +google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d h1:C9v1o0/4quuhOAfmRXA2j+we0PqZIp8traLdeogF3Ms= +google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d/go.mod h1:Wz2wFJntZFmLGo7pLDXZ3wYk5hyc0Mb+SkHhDDXT+lU= +google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d h1:QwnJwPte4XXAkhPu26LTDIahnsMSUV0kK8HkxbC+Pc4= +google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d/go.mod h1:WRrQ7/7N19PypuT0fxLOL5Lq0waoiRri4FbtHDEKrGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 h1:b0xCahf3FK2m2Cv0p4vTozGPWncCvLfwV86UNg8xWU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/e2e/minio_test.go b/e2e/minio_test.go new file mode 100644 index 0000000..2bd94e7 --- /dev/null +++ b/e2e/minio_test.go @@ -0,0 +1,100 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "net" + "net/http" + "os" + "os/exec" + "strconv" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/require" + + "github.com/livekit/storage" +) + +// TestMinIO runs the suites on a real S3 implementation, whose conditional +// writes are atomic where floci's are not, so the IfAbsent races run in full. +// It takes the minio binary from MINIO_BIN or PATH; CI installs it. +func TestMinIO(t *testing.T) { + endpoint := minio(t) + conf := &storage.S3Config{ + AccessKey: "minioadmin", + Secret: "minioadmin", + Region: "us-east-1", + Endpoint: endpoint, + ForcePathStyle: true, + Bucket: flociBucket, + } + admin := s3.New(s3.Options{ + Region: conf.Region, + BaseEndpoint: &conf.Endpoint, + UsePathStyle: true, + Credentials: credentials.NewStaticCredentialsProvider(conf.AccessKey, conf.Secret, ""), + }) + _, err := admin.CreateBucket(t.Context(), &s3.CreateBucketInput{Bucket: aws.String(flociBucket)}) + require.NoError(t, err) + + testProvider(t, endpoint, func(endpoint string) storage.Config { + c := *conf + c.Endpoint = endpoint + return &c + }, objectsSuite{bulk: true, commit: s3Commit}) +} + +// minio starts a MinIO server on a free port and returns its endpoint once it +// is live. It skips the test without a binary, except in CI. +func minio(t *testing.T) string { + bin := os.Getenv("MINIO_BIN") + if bin == "" { + bin, _ = exec.LookPath("minio") + } + if bin == "" { + if os.Getenv("CI") != "" { + t.Fatal("minio binary not found: set MINIO_BIN") + } + t.Skip("minio binary not found: set MINIO_BIN") + } + + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + addr := "127.0.0.1:" + strconv.Itoa(l.Addr().(*net.TCPAddr).Port) + require.NoError(t, l.Close()) + + cmd := exec.Command(bin, "server", t.TempDir(), "--address", addr, "--console-address", "127.0.0.1:0", "--quiet") + cmd.Env = append(os.Environ(), "MINIO_ROOT_USER=minioadmin", "MINIO_ROOT_PASSWORD=minioadmin") + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + + endpoint := "http://" + addr + require.Eventually(t, func() bool { + resp, err := http.Get(endpoint + "/minio/health/live") + if err != nil { + return false + } + resp.Body.Close() + return resp.StatusCode == http.StatusOK + }, 30*time.Second, 100*time.Millisecond) + return endpoint +} diff --git a/e2e/object_test.go b/e2e/object_test.go new file mode 100644 index 0000000..1395a12 --- /dev/null +++ b/e2e/object_test.go @@ -0,0 +1,273 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "path/filepath" + "sort" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/livekit/storage" +) + +// objectsSuite is what testObjectsWith runs beyond the core cases. Its gaps +// name what an emulator gets wrong, and skip the cases that would show it; a +// real store has none. +type objectsSuite struct { + // bulk deletes more keys than one batch takes, too slow for a real account. + bulk bool + // singlePartIfAbsentRaces: two racing IfAbsent puts can both win. + singlePartIfAbsentRaces bool + // multipartIfAbsentRaces: two racing IfAbsent multipart uploads can both win. + multipartIfAbsentRaces bool + // multipartIgnoresIfAbsent: a multipart upload's commit ignores IfAbsent. + multipartIgnoresIfAbsent bool + // commit tells the request that makes a multipart upload's object, and + // whether it carries IfAbsent's condition, which a store that ignores the + // condition cannot show. + commit func(r *http.Request) (commit, conditional bool) +} + +func testObjects(t *testing.T, s storage.Storage) { + testObjectsWith(t, s, objectsSuite{}) +} + +func testObjectsWith(t *testing.T, s storage.Storage, suite objectsSuite) { + ctx := context.Background() + prefix := fmt.Sprintf("objects-test-%d/", time.Now().UnixNano()) + small := prefix + "small" + large := prefix + "large" + missing := prefix + "missing" + payload := []byte("hello from PutObject") + // Larger than one upload part, so UploadObject takes the multipart path. + big := bytes.Repeat([]byte("0123456789abcdef"), (17<<20)/16) + + t.Cleanup(func() { + _ = s.DeleteObjects(ctx, []string{small, large}) + }) + + t.Run("PutObject IfAbsent", func(t *testing.T) { + require.NoError(t, s.PutObject(ctx, small, payload, storage.PutOptions{IfAbsent: true})) + err := s.PutObject(ctx, small, []byte("other"), storage.PutOptions{IfAbsent: true}) + require.ErrorIs(t, err, storage.ErrExists) + got, err := s.GetObject(ctx, small) + require.NoError(t, err) + require.Equal(t, payload, got) + }) + + t.Run("UploadObject", func(t *testing.T) { + require.NoError(t, s.UploadObject(ctx, large, bytes.NewReader(big), storage.PutOptions{IfAbsent: true})) + err := s.UploadObject(ctx, large, bytes.NewReader(big), storage.PutOptions{IfAbsent: true}) + require.ErrorIs(t, err, storage.ErrExists) + var buf bytes.Buffer + n, err := s.DownloadObject(ctx, large, &buf) + require.NoError(t, err) + require.Equal(t, int64(len(big)), n) + require.Equal(t, big, buf.Bytes()) + }) + + t.Run("GetObjectRange", func(t *testing.T) { + got, err := s.GetObjectRange(ctx, small, 6, 4) + require.NoError(t, err) + require.Equal(t, payload[6:10], got) + got, err = s.GetObjectRange(ctx, small, 6, 1000) + require.NoError(t, err) + require.Equal(t, payload[6:], got) + got, err = s.GetObjectRange(ctx, small, int64(len(payload)), 4) + require.NoError(t, err) + require.Empty(t, got) + }) + + t.Run("not found", func(t *testing.T) { + _, err := s.GetObject(ctx, missing) + require.ErrorIs(t, err, storage.ErrNotFound) + _, err = s.GetObjectRange(ctx, missing, 0, 1) + require.ErrorIs(t, err, storage.ErrNotFound) + _, err = s.DownloadFile(ctx, filepath.Join(t.TempDir(), "missing"), missing) + require.ErrorIs(t, err, storage.ErrNotFound) + exists, err := s.ObjectExists(ctx, missing) + require.NoError(t, err) + require.False(t, exists) + exists, err = s.ObjectExists(ctx, small) + require.NoError(t, err) + require.True(t, exists) + }) + + t.Run("ListObjects and DeleteObjects", func(t *testing.T) { + keys, err := s.ListObjects(ctx, prefix) + require.NoError(t, err) + require.Equal(t, []string{large, small}, keys) + require.NoError(t, s.DeleteObjects(ctx, []string{small, large, missing})) + keys, err = s.ListObjects(ctx, prefix) + require.NoError(t, err) + require.Empty(t, keys) + }) + + t.Run("PresignedURL", func(t *testing.T) { + url, err := s.PresignedURL(ctx, small, time.Minute) + require.NoError(t, err) + require.NotEmpty(t, url) + }) + + t.Run("IfAbsent race", func(t *testing.T) { + sizes := map[string][]byte{} + if !suite.singlePartIfAbsentRaces { + sizes["single part"] = payload + } + if !suite.multipartIfAbsentRaces && !suite.multipartIgnoresIfAbsent { + sizes["multipart"] = big + } + for name, body := range sizes { + t.Run(name, func(t *testing.T) { + testIfAbsentRace(t, s, prefix+"race-"+name, body) + }) + } + }) + + t.Run("edge objects", func(t *testing.T) { + empty := prefix + "edge/empty" + odd := prefix + "edge/a dir/caf\u00e9 & r\u00e9sum\u00e9+1.txt" + nested := prefix + "edge/a dir/deeper/leaf" + t.Cleanup(func() { _ = s.DeleteObjects(ctx, []string{empty, odd, nested}) }) + + require.NoError(t, s.PutObject(ctx, empty, nil, storage.PutOptions{})) + got, err := s.GetObject(ctx, empty) + require.NoError(t, err) + require.Empty(t, got) + got, err = s.GetObjectRange(ctx, empty, 0, 4) + require.NoError(t, err) + require.Empty(t, got) + + require.NoError(t, s.PutObject(ctx, odd, []byte("first"), storage.PutOptions{})) + require.NoError(t, s.PutObject(ctx, odd, []byte("second"), storage.PutOptions{})) + got, err = s.GetObject(ctx, odd) + require.NoError(t, err) + require.Equal(t, []byte("second"), got) + require.NoError(t, s.PutObject(ctx, nested, []byte("leaf"), storage.PutOptions{})) + + keys, err := s.ListObjects(ctx, prefix+"edge/") + require.NoError(t, err) + require.Equal(t, []string{odd, nested, empty}, keys) + exists, err := s.ObjectExists(ctx, prefix+"edge/a dir") + require.NoError(t, err) + require.False(t, exists, "a prefix is not an object") + + // A prefix matches keys byte for byte, as on S3. + sibling := prefix + "edge/a dirt" + t.Cleanup(func() { _ = s.DeleteObjects(ctx, []string{sibling}) }) + require.NoError(t, s.PutObject(ctx, sibling, []byte("sibling"), storage.PutOptions{})) + keys, err = s.ListObjects(ctx, prefix+"edge/a dir/") + require.NoError(t, err) + require.Equal(t, []string{odd, nested}, keys) + keys, err = s.ListObjects(ctx, prefix+"edge/a dir") + require.NoError(t, err) + require.Equal(t, []string{odd, nested, sibling}, keys) + + _, err = s.GetObject(ctx, odd+"/child") + require.ErrorIs(t, err, storage.ErrNotFound, "a path through an object") + // A prefix holds no object, so deleting it succeeds and removes nothing. + require.NoError(t, s.DeleteObjects(ctx, []string{prefix + "edge/a dir"})) + exists, err = s.ObjectExists(ctx, nested) + require.NoError(t, err) + require.True(t, exists) + require.NoError(t, s.DeleteObjects(ctx, []string{sibling})) + + require.NoError(t, s.DeleteObjects(ctx, []string{empty, odd, nested})) + keys, err = s.ListObjects(ctx, prefix+"edge/") + require.NoError(t, err) + require.Empty(t, keys) + }) + + if suite.bulk { + t.Run("bulk delete", func(t *testing.T) { + // One past S3's and OSS's 1000 keys per request. + paths := make([]string, 1001) + for i := range paths { + paths[i] = fmt.Sprintf("%sbulk/%04d", prefix, i) + require.NoError(t, s.PutObject(ctx, paths[i], []byte("x"), storage.PutOptions{})) + } + keys, err := s.ListObjects(ctx, prefix+"bulk/") + require.NoError(t, err) + require.True(t, sort.StringsAreSorted(keys)) + require.Equal(t, paths, keys) + require.NoError(t, s.DeleteObjects(ctx, paths)) + keys, err = s.ListObjects(ctx, prefix+"bulk/") + require.NoError(t, err) + require.Empty(t, keys) + }) + } +} + +// testIfAbsentRace sends two IfAbsent uploads of body to path that both pass +// any check before uploading: each body holds its first read until the other's +// starts. Only the store's condition can then refuse one, and exactly one must +// win. +func testIfAbsentRace(t *testing.T, s storage.Storage, path string, body []byte) { + ctx := context.Background() + t.Cleanup(func() { _ = s.DeleteObjects(ctx, []string{path}) }) + + var arrived sync.WaitGroup + var late atomic.Bool + arrived.Add(2) + errs := make(chan error, 2) + for i := range 2 { + r := &heldReader{r: bytes.NewReader(append([]byte{byte(i)}, body...)), arrived: &arrived, late: &late} + go func() { errs <- s.UploadObject(ctx, path, r, storage.PutOptions{IfAbsent: true}) }() + } + var won int + for range 2 { + err := <-errs + if err == nil { + won++ + continue + } + require.ErrorIs(t, err, storage.ErrExists) + } + require.Equal(t, 1, won) + require.False(t, late.Load(), "the uploads never overlapped, so nothing raced") +} + +// heldReader holds its first read until every reader sharing arrived has +// started one, or a while has passed, when it sets late. +type heldReader struct { + r io.Reader + arrived *sync.WaitGroup + late *atomic.Bool + once sync.Once +} + +func (h *heldReader) Read(p []byte) (int, error) { + h.once.Do(func() { + h.arrived.Done() + done := make(chan struct{}) + go func() { h.arrived.Wait(); close(done) }() + select { + case <-done: + case <-time.After(30 * time.Second): + h.late.Store(true) + } + }) + return h.r.Read(p) +} diff --git a/storage_test.go b/e2e/storage_test.go similarity index 88% rename from storage_test.go rename to e2e/storage_test.go index 1aa58d6..7cdff9c 100644 --- a/storage_test.go +++ b/e2e/storage_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package storage_test +package e2e import ( "bytes" @@ -22,6 +22,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" "strings" "testing" "time" @@ -93,6 +94,8 @@ func TestGCP(t *testing.T) { } func TestLocal(t *testing.T) { + // An empty StorageDir is the working directory. + t.Chdir(t.TempDir()) s, err := storage.New(&storage.LocalConfig{}) require.NoError(t, err) @@ -212,7 +215,7 @@ func testStorage(t *testing.T, s storage.Storage) { pathBulk1 := prefix + "-bulk-1.txt" pathBulk2 := prefix + "-bulk-2.txt" - dataPayload := []byte("hello from UploadData") + dataPayload := []byte("hello from PutObject") filePayload := []byte("hello from UploadFile") bulk1Payload := []byte("bulk delete 1") bulk2Payload := []byte("bulk delete 2") @@ -222,11 +225,8 @@ func testStorage(t *testing.T, s storage.Storage) { _ = s.DeleteObjects(context.Background(), []string{pathData, pathFile, pathBulk1, pathBulk2}) }) - t.Run("UploadData", func(t *testing.T) { - loc, size, err := s.UploadData(context.Background(), dataPayload, pathData, "text/plain") - require.NoError(t, err) - require.Equal(t, int64(len(dataPayload)), size) - assertLocation(t, loc, pathData) + t.Run("PutObject", func(t *testing.T) { + require.NoError(t, s.PutObject(context.Background(), pathData, dataPayload, storage.PutOptions{ContentType: "text/plain"})) }) t.Run("UploadFile", func(t *testing.T) { @@ -274,14 +274,9 @@ func testStorage(t *testing.T, s storage.Storage) { require.Equal(t, filePayload, got) }) - t.Run("GeneratePresignedUrl", func(t *testing.T) { - rawURL, err := s.GeneratePresignedUrl(context.Background(), pathData, 5*time.Minute) - if err != nil { - // Some configurations (e.g. Azure without OAuth) don't support presigning; - // the method exists and reported a clear error, which is the contract. - t.Logf("presigned URL not available in this config: %v", err) - return - } + t.Run("PresignedURL", func(t *testing.T) { + rawURL, err := s.PresignedURL(context.Background(), pathData, 5*time.Minute) + require.NoError(t, err) require.NotEmpty(t, rawURL) // The URL should at least be parseable and reference the key we asked for. @@ -314,10 +309,8 @@ func testStorage(t *testing.T, s storage.Storage) { t.Run("DeleteObjects", func(t *testing.T) { // Upload two more objects, then bulk-delete them along with the leftover pathFile. - _, _, err := s.UploadData(context.Background(), bulk1Payload, pathBulk1, "text/plain") - require.NoError(t, err) - _, _, err = s.UploadData(context.Background(), bulk2Payload, pathBulk2, "text/plain") - require.NoError(t, err) + require.NoError(t, s.PutObject(context.Background(), pathBulk1, bulk1Payload, storage.PutOptions{})) + require.NoError(t, s.PutObject(context.Background(), pathBulk2, bulk2Payload, storage.PutOptions{})) require.NoError(t, s.DeleteObjects(context.Background(), []string{pathFile, pathBulk1, pathBulk2})) @@ -332,14 +325,16 @@ func testStorage(t *testing.T, s storage.Storage) { _ = s.DeleteObjects(context.Background(), []string{pathMultipart}) }) - // 6 MiB exceeds the S3 manager's default 5 MiB part size, forcing the - // multipart upload path (CreateMultipartUpload / UploadPart / Complete). - payload := make([]byte, 6<<20) + // 17 MiB is more than one 16 MiB part on S3, and more than one ranged + // GET reads it back. + payload := make([]byte, 17<<20) for i := range payload { payload[i] = byte(i) } + file := filepath.Join(t.TempDir(), "multipart") + require.NoError(t, os.WriteFile(file, payload, 0o600)) - loc, size, err := s.UploadData(context.Background(), payload, pathMultipart, "application/octet-stream") + loc, size, err := s.UploadFile(context.Background(), file, pathMultipart, "application/octet-stream") require.NoError(t, err) require.Equal(t, int64(len(payload)), size) assertLocation(t, loc, pathMultipart) @@ -360,7 +355,7 @@ func hasSuffixIn(items []string, suffix string) bool { return false } -// assertLocation verifies the URL returned by UploadData/UploadFile is non-empty, +// assertLocation verifies the URL returned by UploadFile is non-empty, // parseable, and references the key we just uploaded. func assertLocation(t *testing.T, loc, key string) { t.Helper() diff --git a/floci_test.go b/floci_test.go deleted file mode 100644 index 548325e..0000000 --- a/floci_test.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage_test - -import ( - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "encoding/json" - "encoding/pem" - "os" - "strings" - "testing" - - gcs "cloud.google.com/go/storage" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/oracle/oci-go-sdk/v65/common" - "github.com/oracle/oci-go-sdk/v65/objectstorage" - "github.com/ory/dockertest/v3" - "github.com/ory/dockertest/v3/docker" - "github.com/stretchr/testify/require" - "google.golang.org/api/option" - - "github.com/livekit/storage" -) - -// The emulators run with the tests on any machine with Docker, and must in CI. -// Each holds its state in memory and is removed when its test ends. - -const flociBucket = "livekit" - -func TestFlociS3(t *testing.T) { - endpoint := floci(t, "floci/floci", "2.3.0", "4566") - conf := &storage.S3Config{ - AccessKey: "test", - Secret: "test", - Region: "us-east-1", - Endpoint: endpoint, - ForcePathStyle: true, - Bucket: flociBucket, - } - admin := s3.New(s3.Options{ - Region: conf.Region, - BaseEndpoint: &conf.Endpoint, - UsePathStyle: true, - Credentials: credentials.NewStaticCredentialsProvider(conf.AccessKey, conf.Secret, ""), - }) - _, err := admin.CreateBucket(t.Context(), &s3.CreateBucketInput{Bucket: aws.String(flociBucket)}) - require.NoError(t, err) - - testProvider(t, conf) -} - -// flociAzureKey is Azurite's well-known development key; floci-az accepts any. -const flociAzureKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" - -func TestFlociAzure(t *testing.T) { - // An IP host: the SDK parses an account from the path only then. - endpoint := strings.Replace(floci(t, "floci/floci-az", "0.15.0", "4577"), "localhost", "127.0.0.1", 1) - conf := &storage.AzureConfig{ - AccountName: "devstoreaccount1", - AccountKey: flociAzureKey, - ContainerName: flociBucket, - Endpoint: endpoint + "/devstoreaccount1/", - } - cred, err := azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey) - require.NoError(t, err) - admin, err := azblob.NewClientWithSharedKeyCredential(conf.Endpoint, cred, nil) - require.NoError(t, err) - _, err = admin.CreateContainer(t.Context(), flociBucket, nil) - require.NoError(t, err) - - testProvider(t, conf) -} - -func TestFlociGCP(t *testing.T) { - endpoint := floci(t, "floci/floci-gcp", "0.11.0", "4588") - conf := &storage.GCPConfig{ - // A service account of its own, to sign URLs. - CredentialsJSON: gcpServiceAccount(t), - Bucket: flociBucket, - Endpoint: endpoint, - } - admin, err := gcs.NewClient(t.Context(), option.WithEndpoint(endpoint+"/storage/v1/"), option.WithoutAuthentication()) - require.NoError(t, err) - t.Cleanup(func() { _ = admin.Close() }) - require.NoError(t, admin.Bucket(flociBucket).Create(t.Context(), "floci-local", nil)) - - testProvider(t, conf) -} - -// flociTenancy is floci-oci's tenancy, which is also the root compartment. -const flociTenancy = "ocid1.tenancy.oc1..flocilocaltenancy0000000000000000000000000000000000000000" - -func TestFlociOCI(t *testing.T) { - endpoint := floci(t, "floci/floci-oci", "0.5.0", "4599") - // floci-oci reads the tenancy and user from the request signature without - // verifying it, so any key signs. - conf := &storage.OCIConfig{ - TenancyOCID: flociTenancy, - UserOCID: "ocid1.user.oc1..livekit", - Fingerprint: "00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00", - PrivateKey: rsaKeyPEM(t), - Region: "us-ashburn-1", - Endpoint: endpoint, - Bucket: flociBucket, - } - provider := common.NewRawConfigurationProvider(conf.TenancyOCID, conf.UserOCID, conf.Region, conf.Fingerprint, conf.PrivateKey, nil) - admin, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(provider) - require.NoError(t, err) - admin.Host = endpoint - ns, err := admin.GetNamespace(t.Context(), objectstorage.GetNamespaceRequest{}) - require.NoError(t, err) - _, err = admin.CreateBucket(t.Context(), objectstorage.CreateBucketRequest{ - NamespaceName: ns.Value, - CreateBucketDetails: objectstorage.CreateBucketDetails{ - Name: common.String(flociBucket), - CompartmentId: common.String(flociTenancy), - }, - }) - require.NoError(t, err) - - testProvider(t, conf) -} - -func testProvider(t *testing.T, conf storage.Config) { - s, err := storage.New(conf) - require.NoError(t, err) - t.Run("Storage", func(t *testing.T) { testStorage(t, s) }) - t.Run("Bucket", func(t *testing.T) { testObjects(t, s) }) -} - -// floci starts repo:tag and returns its http:// endpoint once it answers. It -// skips the test where Docker is unavailable, except in CI. -func floci(t *testing.T, repo, tag, port string) string { - pool, err := dockertest.NewPool("") - if err == nil { - err = pool.Client.Ping() - } - if err != nil { - if os.Getenv("CI") != "" { - t.Fatalf("docker: %v", err) - } - t.Skipf("docker unavailable: %v", err) - } - - res, err := pool.RunWithOptions(&dockertest.RunOptions{Repository: repo, Tag: tag}, func(hc *docker.HostConfig) { - hc.AutoRemove = true - }) - require.NoError(t, err) - t.Cleanup(func() { _ = pool.Purge(res) }) - - endpoint := "http://" + res.GetHostPort(port+"/tcp") - require.NoError(t, pool.Retry(func() error { - resp, err := pool.Client.HTTPClient.Get(endpoint) - if err == nil { - resp.Body.Close() - } - return err - })) - return endpoint -} - -func rsaKeyPEM(t *testing.T) string { - key, err := rsa.GenerateKey(rand.Reader, 2048) - require.NoError(t, err) - der, err := x509.MarshalPKCS8PrivateKey(key) - require.NoError(t, err) - return string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})) -} - -func gcpServiceAccount(t *testing.T) string { - b, err := json.Marshal(map[string]string{ - "type": "service_account", - "project_id": "floci-local", - "client_email": "livekit@floci-local.iam.gserviceaccount.com", - "private_key": rsaKeyPEM(t), - "token_uri": "https://oauth2.googleapis.com/token", - }) - require.NoError(t, err) - return string(b) -} diff --git a/gcp.go b/gcp.go index 545f504..11dd0a5 100644 --- a/gcp.go +++ b/gcp.go @@ -17,7 +17,6 @@ package storage import ( "bytes" "context" - "encoding/base64" "errors" "fmt" "io" @@ -25,6 +24,7 @@ import ( "net/url" "os" "strings" + "sync" "time" "cloud.google.com/go/storage" @@ -72,6 +72,9 @@ func NewGCP(conf *GCPConfig) (Storage, error) { conf: conf, } + // An http:// endpoint is an emulator, which takes no credentials; any other + // endpoint, such as Private Service Connect, keeps them. + emulator := strings.HasPrefix(conf.Endpoint, "http://") var opts []option.ClientOption if conf.CredentialsJSON != "" { jwtConfig, err := google.JWTConfigFromJSON([]byte(conf.CredentialsJSON), storageScope) @@ -79,37 +82,22 @@ func NewGCP(conf *GCPConfig) (Storage, error) { return nil, err } u.signer = jwtConfig - if conf.Endpoint == "" { + if !emulator { opts = append(opts, option.WithTokenSource(jwtConfig.TokenSource(context.Background()))) } } if conf.Endpoint != "" { - opts = append(opts, - option.WithEndpoint(strings.TrimSuffix(conf.Endpoint, "/")+"/storage/v1/"), - option.WithoutAuthentication(), - ) + opts = append(opts, option.WithEndpoint(strings.TrimSuffix(conf.Endpoint, "/")+"/storage/v1/")) } - - defaultTransport := http.DefaultTransport.(*http.Transport) - transportClone := transportWithTimeouts(defaultTransport) - - if conf.ProxyConfig != nil { - proxyUrl, err := url.Parse(conf.ProxyConfig.Url) - if err != nil { - return nil, err - } - defaultTransport.Proxy = http.ProxyURL(proxyUrl) - if conf.ProxyConfig.Username != "" && conf.ProxyConfig.Password != "" { - auth := fmt.Sprintf("%s:%s", conf.ProxyConfig.Username, conf.ProxyConfig.Password) - basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth)) - defaultTransport.ProxyConnectHeader = http.Header{} - defaultTransport.ProxyConnectHeader.Add("Proxy-Authorization", basicAuth) - } + if emulator { + opts = append(opts, option.WithoutAuthentication()) } - client, err := storage.NewClient(context.Background(), opts...) - // restore default transport - http.DefaultTransport = transportClone + setProxy, err := proxySetter(conf.ProxyConfig) + if err != nil { + return nil, err + } + client, err := newGCSClient(opts, setProxy) if err != nil { return nil, err } @@ -118,13 +106,6 @@ func NewGCP(conf *GCPConfig) (Storage, error) { return u, nil } -func (s *gcpStorage) UploadData(ctx context.Context, data []byte, storagePath, contentType string) (string, int64, error) { - if err := s.PutObject(ctx, storagePath, data, PutOptions{ContentType: contentType}); err != nil { - return "", 0, err - } - return s.location(storagePath), int64(len(data)), nil -} - func (s *gcpStorage) UploadFile(ctx context.Context, filepath, storagePath, contentType string) (string, int64, error) { file, err := os.Open(filepath) if err != nil { @@ -160,14 +141,16 @@ func (s *gcpStorage) location(storagePath string) string { }).String() } -func (s *gcpStorage) presignedUrl(storagePath string, expiration time.Duration) (string, error) { - // V4, which the SDK can address to an http:// emulator, signs for at most 7 - // days. +func (s *gcpStorage) presignedURL(storagePath string, expiration time.Duration) (string, error) { opts := &storage.SignedURLOptions{ - Scheme: storage.SigningSchemeV4, - Method: "GET", - Expires: time.Now().Add(expiration), - Insecure: strings.HasPrefix(s.conf.Endpoint, "http://"), + Method: "GET", + Expires: time.Now().Add(expiration), + } + // A V2 URL, which has no limit on its expiry, is always https; an http:// + // emulator takes V4, which signs for at most 7 days. + if strings.HasPrefix(s.conf.Endpoint, "http://") { + opts.Scheme = storage.SigningSchemeV4 + opts.Insecure = true } if s.signer != nil { opts.GoogleAccessID = s.signer.Email @@ -222,12 +205,12 @@ func (s *gcpStorage) write(ctx context.Context, storagePath string, body io.Read return s.writeError(w.Close(), opts.IfAbsent) } -// writeError maps a failed precondition to ErrObjectExists only where IfAbsent +// writeError maps a failed precondition to ErrExists only where IfAbsent // set one: GCS answers other refusals with 412 too. func (s *gcpStorage) writeError(err error, ifAbsent bool) error { var apiErr *googleapi.Error if ifAbsent && errors.As(err, &apiErr) && apiErr.Code == http.StatusPreconditionFailed { - return fmt.Errorf("%w: %w", ErrObjectExists, wrapGCPError(err)) + return fmt.Errorf("%w: %w", ErrExists, wrapGCPError(err)) } return wrapGCPObjectError(err) } @@ -295,8 +278,8 @@ func (s *gcpStorage) DeleteObjects(ctx context.Context, storagePaths []string) e return nil } -func (s *gcpStorage) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { - return s.presignedUrl(storagePath, expiration) +func (s *gcpStorage) PresignedURL(_ context.Context, storagePath string, expiration time.Duration) (string, error) { + return s.presignedURL(storagePath, expiration) } func wrapGCPObjectError(err error) error { @@ -304,7 +287,25 @@ func wrapGCPObjectError(err error) error { return nil } if errors.Is(err, storage.ErrObjectNotExist) { - return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapGCPError(err)) + return fmt.Errorf("%w: %w", ErrNotFound, wrapGCPError(err)) } return wrapGCPError(err) } + +// gcsTransportMu serializes newGCSClient's swap of http.DefaultTransport. +var gcsTransportMu sync.Mutex + +// newGCSClient builds a client on a transport with this package's timeouts and +// the proxy. The SDK copies http.DefaultTransport when it builds a client, and +// passing a client of our own would mean wiring its auth by hand, so the +// transport stands in as the default only while the client is built. +func newGCSClient(opts []option.ClientOption, setProxy func(*http.Transport)) (*storage.Client, error) { + gcsTransportMu.Lock() + defer gcsTransportMu.Unlock() + orig := http.DefaultTransport + t := transportWithTimeouts(orig) + setProxy(t) + http.DefaultTransport = t + defer func() { http.DefaultTransport = orig }() + return storage.NewClient(context.Background(), opts...) +} diff --git a/go.mod b/go.mod index a4bf58c..7d88909 100644 --- a/go.mod +++ b/go.mod @@ -15,10 +15,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 github.com/aws/smithy-go v1.28.1 github.com/googleapis/gax-go/v2 v2.26.2 - github.com/joho/godotenv v1.5.1 github.com/livekit/protocol v1.52.1 github.com/oracle/oci-go-sdk/v65 v65.126.1 - github.com/ory/dockertest/v3 v3.12.0 github.com/stretchr/testify v1.12.1 golang.org/x/oauth2 v0.37.0 google.golang.org/api v0.300.0 @@ -32,15 +30,11 @@ require ( cloud.google.com/go/compute/metadata v0.10.0 // indirect cloud.google.com/go/iam v1.12.0 // indirect cloud.google.com/go/monitoring v1.30.0 // indirect - dario.cat/mergo v1.0.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/apache/arrow-go/v18 v18.7.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect @@ -54,47 +48,27 @@ require ( github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect - github.com/containerd/continuity v0.4.5 // indirect - github.com/docker/cli v27.4.1+incompatible // indirect - github.com/docker/docker v27.1.1+incompatible // indirect - github.com/docker/go-connections v0.5.0 // indirect - github.com/docker/go-units v0.5.0 // indirect github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/gofrs/flock v0.10.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/google/s2a-go v0.1.10 // indirect - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.22 // indirect github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/sys/user v0.3.0 // indirect - github.com/moby/term v0.5.0 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/opencontainers/runc v1.2.3 // indirect github.com/pierrec/lz4/v4 v4.1.28 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/puzpuzpuz/xsync/v4 v4.5.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect github.com/sony/gobreaker/v2 v2.4.0 // indirect github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect - github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect @@ -123,5 +97,4 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 // indirect google.golang.org/grpc v1.84.0 // indirect google.golang.org/protobuf v1.36.12 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 7338446..43f3c04 100644 --- a/go.sum +++ b/go.sum @@ -26,10 +26,6 @@ cloud.google.com/go/storage v1.69.0 h1:jAAMC1411HEh78nKsU0Zns+eFj3TnhjAWIhg5Ud/X cloud.google.com/go/storage v1.69.0/go.mod h1:PELYsxTYm2peE4mwLEC1+mS1dA/kUSRUxNv56rOy44g= cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2 h1:utpeoEeZjd+A8J41zvoLsOOrqXHhX1Kx/X/tCW9dEYQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU= @@ -42,8 +38,6 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 h1:tQJdgQjbKKffEiMOK github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2/go.mod h1:kTkpJchXAdO7LGkKZP97a+Ou8TgaF4NT6MHwOIy5uSQ= github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 h1:X/bTwe0qpvmvPPutc+LWo7T4cKMqDP9JHP4k+Co9ZB4= github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0/go.mod h1:+kX7ahyFelQ42F3RScM2CFgHJljp0phc/FieKLYVpFI= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI= github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 h1:bN1gA3of5bXtbnLsRPrwfmbbe7A5UWFlcTHseujLnpc= @@ -54,10 +48,6 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 h1:RoO5+d7uCmDqovLrHCr2/BuViUXvdcrNxyNM1pN9dDQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= -github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= @@ -106,28 +96,12 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZj github.com/aws/aws-sdk-go-v2/service/sts v1.51.1/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM= github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= -github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= -github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dennwc/iters v1.2.2 h1:XH2/Etihiy9ZvPOVCR+icQXeYlhbvS7k0qro4x/2qQo= github.com/dennwc/iters v1.2.2/go.mod h1:M9KuuMBeyEXYTmB7EnI9SCyALFCmPWOIxn5W1L0CjGg= -github.com/docker/cli v27.4.1+incompatible h1:VzPiUlRJ/xh+otB75gva3r05isHMo5wXDfPRi5/b4hI= -github.com/docker/cli v27.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= -github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= @@ -145,16 +119,10 @@ github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= -github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= -github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= -github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/flock v0.10.0 h1:SHMXenfaB03KbroETaCMtbBg3Yn29v4w1r+tgy4ff4k= github.com/gofrs/flock v0.10.0/go.mod h1:FirDy1Ing0mI2+kB6wk+vyyAH+e6xiE+EYA0jnzV9jc= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -169,18 +137,12 @@ github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9 github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.10 h1:EMp+aOuXN6l8cE/gjF5Bt+vyZxsUuyCWe9chDWR/+uU= github.com/google/s2a-go v0.1.10/go.mod h1:pz4tyvwXvJLLbyrkh6FW1eS2zPUXMaTmyNhYtyP2tNw= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.22 h1:NU4XpII6jD+Dxcot94fqjE+AfJoE/lQP9q3faYGzC/c= github.com/googleapis/enterprise-certificate-proxy v0.3.22/go.mod h1:L3D/IQExI6LqEjBdXcZQ1WluSgigQmSwBboFstVPM4w= github.com/googleapis/gax-go/v2 v2.26.2 h1:ydkmNXxj7bEmmeK5AihkKnWxyOyBR9TDebvp5L5izk8= github.com/googleapis/gax-go/v2 v2.26.2/go.mod h1:sMKqnMesnKH+3wiRJROcttA+cJoZoGbZl1vDQ8XYtGk= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= @@ -191,69 +153,36 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5ATTo469PQPkqzdoU7be46ryiCDO3boc= github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/protocol v1.52.1 h1:w8wRjRlj/57AJBxwomEzpEMzf+lT0qOoLQLO7awdTMU= github.com/livekit/protocol v1.52.1/go.mod h1:t5CV/LCsWbI9zsSTHTJKDyLtByqvvVkioSP7jgqpIGA= github.com/livekit/psrpc v0.8.0 h1:uo3x+DL8pK2OIaCxg/s9sLUO2cIocS5HwdPhhI+m+fQ= github.com/livekit/psrpc v0.8.0/go.mod h1:6KnU8vULoizX0Er9NNfeKJDV8ug87j3fuo19M0bhj6Q= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= -github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opencontainers/runc v1.2.3 h1:fxE7amCzfZflJO2lHXf4y/y8M1BoAqp+FVmG19oYB80= -github.com/opencontainers/runc v1.2.3/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM= github.com/oracle/oci-go-sdk/v65 v65.126.1 h1:WmQ2Igq7/L/cHlR2jZnJkBC2UI51Ams4DJZW3CZGVBo= github.com/oracle/oci-go-sdk/v65 v65.126.1/go.mod h1:YmvgbsnUfrsiJ410XAGQ1BRsrUYJJ7W3O5wUikmeI8w= -github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCyRCw= -github.com/ory/dockertest/v3 v3.12.0/go.mod h1:aKNDTva3cp8dwOWwb9cWuX84aH5akkxXRvO7KCwWVjE= github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio= github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 h1:S1hI5JiKP7883xBzZAr1ydcxrKNSVNm7+3+JwjxZEsg= github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25/go.mod h1:ZQntvDG8TkPgljxtA0R9frDoND4QORU1VXz015N5Ks4= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg= github.com/sony/gobreaker/v2 v2.4.0/go.mod h1:pTyFJgcZ3h2tdQVLZZruK2C0eoFL1fb/G83wK1ZQl+s= github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJXP61mNV3/7iuU= github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= @@ -292,49 +221,22 @@ go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.300.0 h1:2rvPV2bqnPuHOaF4gGOBiT1IIc6JVXYyHCkZeqdzjNk= @@ -349,13 +251,7 @@ google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= -gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/local.go b/local.go index 533f49a..a4a5f4a 100644 --- a/local.go +++ b/local.go @@ -21,12 +21,14 @@ import ( "fmt" "io" "io/fs" + "math/rand/v2" "net/url" "os" - "path" "path/filepath" "slices" + "strconv" "strings" + "syscall" "time" ) @@ -45,6 +47,17 @@ func NewLocal(conf *LocalConfig) (Storage, error) { }, nil } +// file is the path that holds storagePath's object. A key that is empty or +// leads out of StorageDir names no object and is refused. +func (u *localUploader) file(storagePath string) (string, error) { + p := filepath.Join(u.StorageDir, filepath.FromSlash(storagePath)) + rel, err := filepath.Rel(u.StorageDir, p) + if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("storage: key %q is not under %s", storagePath, u.StorageDir) + } + return p, nil +} + func (u *localUploader) UploadFile(ctx context.Context, localPath, storagePath, _ string) (string, int64, error) { file, err := os.Open(localPath) if err != nil { @@ -52,166 +65,110 @@ func (u *localUploader) UploadFile(ctx context.Context, localPath, storagePath, } defer file.Close() - stat, err := file.Stat() - if err != nil { - return "", 0, err - } - if err = u.UploadObject(ctx, storagePath, file, PutOptions{}); err != nil { - return "", 0, err - } - return u.location(storagePath), stat.Size(), nil + dst, n, err := u.upload(ctx, storagePath, file, PutOptions{}) + return dst, n, err } -func (u *localUploader) UploadData(ctx context.Context, data []byte, storagePath, _ string) (string, int64, error) { - if err := u.PutObject(ctx, storagePath, data, PutOptions{}); err != nil { - return "", 0, err - } - return u.location(storagePath), int64(len(data)), nil -} - -func (u *localUploader) location(storagePath string) string { - return path.Join(u.StorageDir, storagePath) -} - -func (u *localUploader) listObjects(prefix string) ([]string, error) { - absPrefix := path.Join(u.StorageDir, prefix) - dir, filenamePrefix := path.Split(absPrefix) - - var files []string - entries, err := os.ReadDir(dir) +func (u *localUploader) DownloadFile(ctx context.Context, localPath, storagePath string) (int64, error) { + src, err := u.file(storagePath) if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, err - } - - for _, entry := range entries { - if !strings.HasPrefix(entry.Name(), filenamePrefix) { - continue - } - - entryPath := path.Join(dir, entry.Name()) - if entry.IsDir() { - if err = filepath.Walk(entryPath, func(p string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - rel, err := filepath.Rel(u.StorageDir, p) - if err != nil { - return err - } - files = append(files, rel) - } - return nil - }); err != nil { - return nil, err - } - } else { - rel, err := filepath.Rel(u.StorageDir, entryPath) - if err != nil { - return nil, err - } - files = append(files, rel) - } + return 0, err } - - return files, nil -} - -func (u *localUploader) DownloadFile(_ context.Context, localPath, storagePath string) (int64, error) { - storagePath = path.Join(u.StorageDir, storagePath) - - storage, err := os.Open(storagePath) + f, err := os.Open(src) if err != nil { return 0, localObjectError(err) } - defer storage.Close() + defer f.Close() - if dir, _ := path.Split(localPath); dir != "" { + if dir := filepath.Dir(localPath); dir != "." { if err = os.MkdirAll(dir, 0755); err != nil { return 0, err } } - local, err := os.Create(localPath) if err != nil { return 0, err } defer local.Close() - size, err := io.Copy(local, storage) - if err != nil { - return 0, err - } - - return size, nil -} - -func (u *localUploader) presignedUrl(storagePath string, _ time.Duration) (string, error) { - abs := filepath.Join(u.StorageDir, storagePath) - return (&url.URL{Scheme: "file", Path: filepath.ToSlash(abs)}).String(), nil -} - -// deleteObject removes the file, then each parent directory it leaves empty. -func (u *localUploader) deleteObject(storagePath string) error { - storagePath = path.Join(u.StorageDir, storagePath) - - for { - if err := os.Remove(storagePath); err != nil { - return err - } - - storagePath, _ = path.Split(storagePath) - storagePath = storagePath[:len(storagePath)-1] // remove trailing slash - entries, err := os.ReadDir(storagePath) - if err != nil { - return err - } - - if storagePath == u.StorageDir || len(entries) > 0 { - return nil - } - } + return io.Copy(local, f) } const localUploadPrefix = ".upload-" func (u *localUploader) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { - return u.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) + _, _, err := u.upload(ctx, storagePath, bytes.NewReader(data), opts) + return err } -// UploadObject writes a temporary file and links or renames it into place, so -// readers never see a partial object and IfAbsent is atomic. -func (u *localUploader) UploadObject(_ context.Context, storagePath string, body io.Reader, opts PutOptions) error { - dst := u.location(storagePath) - if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { - return err +func (u *localUploader) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + _, _, err := u.upload(ctx, storagePath, body, opts) + return err +} + +// upload writes a temporary file, syncs it and links or renames it into place, +// so readers never see a partial object, a crash leaves none, and IfAbsent is +// atomic. It returns the object's path and the bytes written. +func (u *localUploader) upload(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) (string, int64, error) { + dst, err := u.file(storagePath) + if err != nil { + return "", 0, err } - tmp, err := os.CreateTemp(filepath.Dir(dst), localUploadPrefix+"*") + if err = ctx.Err(); err != nil { + return "", 0, err + } + tmp, err := createUploadFile(filepath.Dir(dst)) if err != nil { - return err + return "", 0, err } defer os.Remove(tmp.Name()) - if _, err = io.Copy(tmp, body); err != nil { - tmp.Close() - return err + + n, err := io.Copy(tmp, body) + if err == nil { + err = tmp.Sync() } - if err = tmp.Close(); err != nil { - return err + if closeErr := tmp.Close(); err == nil { + err = closeErr } + if err != nil { + return "", 0, err + } + if !opts.IfAbsent { - return os.Rename(tmp.Name(), dst) + err = os.Rename(tmp.Name(), dst) + } else if err = os.Link(tmp.Name(), dst); errors.Is(err, fs.ErrExist) { + err = ErrExists } - if err = os.Link(tmp.Name(), dst); errors.Is(err, fs.ErrExist) { - return ErrObjectExists + if err != nil { + return "", 0, err + } + return dst, n, nil +} + +// createUploadFile is a new file in dir, made with its parents, with the mode +// os.Create gives, 0666 less the umask, which moving it into place keeps: +// os.CreateTemp's 0600 would hide objects from other users, such as a host +// reading a container's volume. A delete can remove dir once empty, before +// the file is in it, so it is made again. +func createUploadFile(dir string) (*os.File, error) { + for { + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, err + } + name := filepath.Join(dir, localUploadPrefix+strconv.FormatUint(rand.Uint64(), 36)) + f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o666) + if !errors.Is(err, fs.ErrExist) && !errors.Is(err, fs.ErrNotExist) { + return f, err + } } - return err } func (u *localUploader) GetObject(_ context.Context, storagePath string) ([]byte, error) { - data, err := os.ReadFile(u.location(storagePath)) + src, err := u.file(storagePath) + if err != nil { + return nil, err + } + data, err := os.ReadFile(src) return data, localObjectError(err) } @@ -219,12 +176,17 @@ func (u *localUploader) GetObjectRange(_ context.Context, storagePath string, of if offset < 0 || length <= 0 { return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) } - f, err := os.Open(u.location(storagePath)) + f, err := u.open(storagePath) if err != nil { - return nil, localObjectError(err) + return nil, err } defer f.Close() - buf := make([]byte, length) + info, err := f.Stat() + if err != nil { + return nil, err + } + + buf := make([]byte, max(0, min(length, info.Size()-offset))) n, err := f.ReadAt(buf, offset) if errors.Is(err, io.EOF) { err = nil @@ -233,49 +195,128 @@ func (u *localUploader) GetObjectRange(_ context.Context, storagePath string, of } func (u *localUploader) DownloadObject(_ context.Context, storagePath string, w io.Writer) (int64, error) { - f, err := os.Open(u.location(storagePath)) + f, err := u.open(storagePath) if err != nil { - return 0, localObjectError(err) + return 0, err } defer f.Close() return io.Copy(w, f) } +func (u *localUploader) open(storagePath string) (*os.File, error) { + src, err := u.file(storagePath) + if err != nil { + return nil, err + } + f, err := os.Open(src) + return f, localObjectError(err) +} + +// ObjectExists reports a directory as no object, as a store reports a prefix. func (u *localUploader) ObjectExists(_ context.Context, storagePath string) (bool, error) { - _, err := os.Stat(u.location(storagePath)) - if errors.Is(err, fs.ErrNotExist) { + src, err := u.file(storagePath) + if err != nil { + return false, err + } + info, err := os.Stat(src) + if errors.Is(localObjectError(err), ErrNotFound) { return false, nil } - return err == nil, err + return err == nil && !info.IsDir(), err } -// ListObjects leaves out UploadObject's temporary files, in progress or left by -// a crash. +// ListObjects walks the directory the prefix names and keeps the keys that +// start with it, as on S3: "a/" lists a's files, "a" those of a, ab and a.txt. +// A leading "/" is dropped, as from keys. UploadObject's temporary files, in +// progress or left by a crash, are left out. func (u *localUploader) ListObjects(_ context.Context, prefix string) ([]string, error) { - keys, err := u.listObjects(prefix) - keys = slices.DeleteFunc(keys, func(k string) bool { - return strings.HasPrefix(filepath.Base(k), localUploadPrefix) + prefix = strings.TrimLeft(prefix, "/") + root := u.StorageDir + if i := strings.LastIndex(prefix, "/"); i >= 0 { + dir, err := u.file(prefix[:i]) + if err != nil { + return nil, err + } + root = dir + } + + // The trailing separator has the walk follow a root that is a symlink. + var keys []string + err := filepath.WalkDir(root+string(filepath.Separator), func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || strings.HasPrefix(d.Name(), localUploadPrefix) { + return nil + } + rel, err := filepath.Rel(u.StorageDir, p) + if err != nil { + return err + } + if key := filepath.ToSlash(rel); strings.HasPrefix(key, prefix) { + keys = append(keys, key) + } + return nil }) + if errors.Is(localObjectError(err), ErrNotFound) { + return nil, nil + } slices.Sort(keys) return keys, err } -func (u *localUploader) DeleteObjects(_ context.Context, storagePaths []string) error { +func (u *localUploader) DeleteObjects(ctx context.Context, storagePaths []string) error { for _, p := range storagePaths { - if err := u.deleteObject(p); err != nil && !errors.Is(err, fs.ErrNotExist) { + if err := ctx.Err(); err != nil { + return err + } + if err := u.deleteObject(p); err != nil { return err } } return nil } -func (u *localUploader) GeneratePresignedUrl(_ context.Context, storagePath string, expiration time.Duration) (string, error) { - return u.presignedUrl(storagePath, expiration) +// deleteObject removes the file, then each parent directory it leaves empty. A +// key that holds no file, a directory among them, is already deleted. +func (u *localUploader) deleteObject(storagePath string) error { + p, err := u.file(storagePath) + if err != nil { + return err + } + info, err := os.Lstat(p) + if errors.Is(localObjectError(err), ErrNotFound) || (err == nil && info.IsDir()) { + return nil + } + if err != nil { + return err + } + if err = os.Remove(p); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + + // A directory another upload just wrote into is not empty, and stays. + for dir := filepath.Dir(p); dir != u.StorageDir; dir = filepath.Dir(dir) { + if os.Remove(dir) != nil { + return nil + } + } + return nil +} + +func (u *localUploader) PresignedURL(_ context.Context, storagePath string, _ time.Duration) (string, error) { + p, err := u.file(storagePath) + if err != nil { + return "", err + } + return (&url.URL{Scheme: "file", Path: filepath.ToSlash(p)}).String(), nil } +// localObjectError reports a missing file as ErrNotFound, and so a path +// through a file (ENOTDIR) or onto a directory (EISDIR), where no object is. func localObjectError(err error) error { - if errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("%w: %w", ErrObjectNotFound, err) + if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) || errors.Is(err, syscall.EISDIR) { + return fmt.Errorf("%w: %w", ErrNotFound, err) } return err } diff --git a/location_test.go b/location_test.go index 6ea3646..2fa8f46 100644 --- a/location_test.go +++ b/location_test.go @@ -377,7 +377,16 @@ func TestLocalLocation(t *testing.T) { } for _, tc := range cases { u := &localUploader{StorageDir: tc.storageDir} - require.Equal(t, tc.want, u.location(tc.storagePath)) + got, err := u.file(tc.storagePath) + require.NoError(t, err) + require.Equal(t, tc.want, got) + } + + // No key reaches outside the directory or names the directory itself. + u := &localUploader{StorageDir: "/var/storage"} + for _, key := range []string{"", ".", "/", "..", "../x", "a/../../x", "a/.."} { + _, err := u.file(key) + require.Error(t, err, key) } } diff --git a/logging.go b/logging.go index 19eee3e..d1904aa 100644 --- a/logging.go +++ b/logging.go @@ -23,20 +23,20 @@ import ( "github.com/livekit/protocol/logger" ) -// S3Logger only logs aws messages on upload failure -type S3Logger struct { +// s3Logger only logs aws messages on upload failure +type s3Logger struct { mu sync.Mutex msgs []string idx int } -func NewS3Logger() *S3Logger { - return &S3Logger{ +func news3Logger() *s3Logger { + return &s3Logger{ msgs: make([]string, 10), } } -func (l *S3Logger) Logf(classification logging.Classification, format string, v ...interface{}) { +func (l *s3Logger) Logf(classification logging.Classification, format string, v ...interface{}) { format = "aws %s: " + format v = append([]interface{}{strings.ToLower(string(classification))}, v...) @@ -46,7 +46,7 @@ func (l *S3Logger) Logf(classification logging.Classification, format string, v l.mu.Unlock() } -func (l *S3Logger) WriteLogs() { +func (l *s3Logger) WriteLogs() { l.mu.Lock() size := len(l.msgs) for range size { diff --git a/object_test.go b/object_test.go deleted file mode 100644 index 3495672..0000000 --- a/object_test.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage_test - -import ( - "bytes" - "context" - "fmt" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/livekit/storage" -) - -func testObjects(t *testing.T, s storage.Storage) { - ctx := context.Background() - prefix := fmt.Sprintf("objects-test-%d/", time.Now().UnixNano()) - small := prefix + "small" - large := prefix + "large" - missing := prefix + "missing" - payload := []byte("hello from PutObject") - // Larger than one upload part, so UploadObject takes the multipart path. - big := bytes.Repeat([]byte("0123456789abcdef"), (17<<20)/16) - - t.Cleanup(func() { - _ = s.DeleteObjects(ctx, []string{small, large}) - }) - - t.Run("PutObject IfAbsent", func(t *testing.T) { - require.NoError(t, s.PutObject(ctx, small, payload, storage.PutOptions{IfAbsent: true})) - err := s.PutObject(ctx, small, []byte("other"), storage.PutOptions{IfAbsent: true}) - require.ErrorIs(t, err, storage.ErrObjectExists) - got, err := s.GetObject(ctx, small) - require.NoError(t, err) - require.Equal(t, payload, got) - }) - - t.Run("UploadObject", func(t *testing.T) { - require.NoError(t, s.UploadObject(ctx, large, bytes.NewReader(big), storage.PutOptions{IfAbsent: true})) - err := s.UploadObject(ctx, large, bytes.NewReader(big), storage.PutOptions{IfAbsent: true}) - require.ErrorIs(t, err, storage.ErrObjectExists) - var buf bytes.Buffer - n, err := s.DownloadObject(ctx, large, &buf) - require.NoError(t, err) - require.Equal(t, int64(len(big)), n) - require.Equal(t, big, buf.Bytes()) - }) - - t.Run("GetObjectRange", func(t *testing.T) { - got, err := s.GetObjectRange(ctx, small, 6, 4) - require.NoError(t, err) - require.Equal(t, payload[6:10], got) - got, err = s.GetObjectRange(ctx, small, 6, 1000) - require.NoError(t, err) - require.Equal(t, payload[6:], got) - got, err = s.GetObjectRange(ctx, small, int64(len(payload)), 4) - require.NoError(t, err) - require.Empty(t, got) - }) - - t.Run("not found", func(t *testing.T) { - _, err := s.GetObject(ctx, missing) - require.ErrorIs(t, err, storage.ErrObjectNotFound) - _, err = s.GetObjectRange(ctx, missing, 0, 1) - require.ErrorIs(t, err, storage.ErrObjectNotFound) - _, err = s.DownloadFile(ctx, filepath.Join(t.TempDir(), "missing"), missing) - require.ErrorIs(t, err, storage.ErrObjectNotFound) - exists, err := s.ObjectExists(ctx, missing) - require.NoError(t, err) - require.False(t, exists) - exists, err = s.ObjectExists(ctx, small) - require.NoError(t, err) - require.True(t, exists) - }) - - t.Run("ListObjects and DeleteObjects", func(t *testing.T) { - keys, err := s.ListObjects(ctx, prefix) - require.NoError(t, err) - require.Equal(t, []string{large, small}, keys) - require.NoError(t, s.DeleteObjects(ctx, []string{small, large, missing})) - keys, err = s.ListObjects(ctx, prefix) - require.NoError(t, err) - require.Empty(t, keys) - }) - - t.Run("GeneratePresignedUrl", func(t *testing.T) { - url, err := s.GeneratePresignedUrl(ctx, small, time.Minute) - require.NoError(t, err) - require.NotEmpty(t, url) - }) -} diff --git a/objects/objects.go b/objects/objects.go index bd22a8c..6513fb8 100644 --- a/objects/objects.go +++ b/objects/objects.go @@ -51,22 +51,21 @@ type Bucket interface { // ListObjects returns every key under prefix, recursively, as full paths, // in byte order. ListObjects(ctx context.Context, prefix string) ([]string, error) - // DeleteObjects succeeds for paths that hold nothing and fails when a path - // is not deleted; some may be deleted by then, and a retry is safe. GCS, - // Azure and OCI send one request per path. + // DeleteObjects succeeds for paths that hold nothing, as S3's delete does, + // and fails when a path is not deleted; some may be deleted by then, and a + // retry is safe. GCS, Azure and OCI send one request per path. DeleteObjects(ctx context.Context, storagePaths []string) error - // GeneratePresignedUrl is a GET URL; local returns a file:// URL. - GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) + // PresignedURL is a GET URL; local returns a file:// URL. + PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) } // Storage is Bucket and the calls that publish and fetch whole files. type Storage interface { Bucket - // UploadData and UploadFile replace any object at storagePath and return - // its location, a URL to it. - UploadData(ctx context.Context, data []byte, storagePath, contentType string) (location string, size int64, err error) + // UploadFile replaces any object at storagePath and returns its location, + // a URL to it. UploadFile(ctx context.Context, filepath, storagePath, contentType string) (location string, size int64, err error) DownloadFile(ctx context.Context, filepath, storagePath string) (size int64, err error) } diff --git a/oci.go b/oci.go index fb576fe..8c453f6 100644 --- a/oci.go +++ b/oci.go @@ -17,6 +17,7 @@ package storage import ( "bytes" "context" + "crypto/tls" "errors" "fmt" "io" @@ -60,8 +61,23 @@ func NewOCI(conf *OCIConfig) (Storage, error) { } // The SDK's 60s http.Client.Timeout covers reading the body and would cut a // long transfer; a response header timeout catches a dead connection instead. + // Its transport wrapper stays, as it loads OCI_DEFAULT_CERTS_PATH and + // refreshes the TLS config. if hc, ok := client.HTTPClient.(*http.Client); ok { - client.HTTPClient = &http.Client{Transport: transportWithTimeouts(hc.Transport), CheckRedirect: hc.CheckRedirect, Jar: hc.Jar} + if w, ok := hc.Transport.(*common.OciHTTPTransportWrapper); ok { + w.TransportTemplate = func(tlsConfig *tls.Config) (http.RoundTripper, error) { + t, err := common.DefaultTransport(tlsConfig) + if err != nil { + return nil, err + } + return transportWithTimeouts(t), nil + } + } + c := &http.Client{Transport: hc.Transport, CheckRedirect: hc.CheckRedirect, Jar: hc.Jar} + if os.Getenv(common.CustomClientTimeoutEnvVar) != "" { + c.Timeout = hc.Timeout + } + client.HTTPClient = c } // The SDK only prepends a scheme to client.Host on the first request, so @@ -116,12 +132,9 @@ func newConfigProvider(conf *OCIConfig) (common.ConfigurationProvider, error) { return common.DefaultConfigProvider(), nil } -func (s *ociStorage) UploadData(ctx context.Context, data []byte, storagePath, contentType string) (string, int64, error) { - if err := s.PutObject(ctx, storagePath, data, PutOptions{ContentType: contentType}); err != nil { - return "", 0, err - } - return s.location(storagePath), int64(len(data)), nil -} +// UploadFile sends the file in one PutObject, up to OCI's 50 GiB, which the SDK +// can retry by seeking it: a multipart upload would need more permissions and +// can leave uncommitted parts behind. func (s *ociStorage) UploadFile(ctx context.Context, filepath, storagePath, contentType string) (string, int64, error) { file, err := os.Open(filepath) @@ -134,7 +147,7 @@ func (s *ociStorage) UploadFile(ctx context.Context, filepath, storagePath, cont if err != nil { return "", 0, err } - if err = s.UploadObject(ctx, storagePath, file, PutOptions{ContentType: contentType}); err != nil { + if err = s.put(ctx, storagePath, file, stat.Size(), PutOptions{ContentType: contentType}); err != nil { return "", 0, err } return s.location(storagePath), stat.Size(), nil @@ -162,11 +175,11 @@ func (s *ociStorage) DownloadFile(ctx context.Context, filepath, storagePath str return s.DownloadObject(ctx, storagePath, file) } -// GeneratePresignedUrl creates a pre-authenticated request. Unlike an S3 +// PresignedURL creates a pre-authenticated request. Unlike an S3 // presigned URL, a PAR is a durable object: it is listable and revocable, it // costs an API call to mint, and it needs PAR_MANAGE on the bucket. Its access // URI is returned only here and cannot be read back afterwards. -func (s *ociStorage) presignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { +func (s *ociStorage) presignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { name := fmt.Sprintf("livekit-storage-%s-%d", storagePath, time.Now().UnixNano()) expires := common.SDKTime{Time: time.Now().Add(expiration)} @@ -204,14 +217,24 @@ func wrapOCIError(err error) error { const ociUploadPartSize = 16 << 20 func (s *ociStorage) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { - size := int64(len(data)) + body := common.NewOCIReadSeekCloser(io.NopCloser(bytes.NewReader(data))) + return s.put(ctx, storagePath, body, int64(len(data)), opts) +} + +// ociContentDisposition has browsers show an object rather than download it, +// as on S3. +var ociContentDisposition = common.String("inline") + +// put sends body in one request. It must be seekable, so the SDK can re-read +// it when it retries. +func (s *ociStorage) put(ctx context.Context, storagePath string, body io.ReadCloser, size int64, opts PutOptions) error { req := objectstorage.PutObjectRequest{ - NamespaceName: &s.namespace, - BucketName: &s.conf.Bucket, - ObjectName: &storagePath, - // Seekable, so the SDK can re-read it when it retries. - PutObjectBody: common.NewOCIReadSeekCloser(io.NopCloser(bytes.NewReader(data))), - ContentLength: &size, + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &storagePath, + PutObjectBody: body, + ContentLength: &size, + ContentDisposition: ociContentDisposition, } if opts.ContentType != "" { req.ContentType = &opts.ContentType @@ -223,30 +246,26 @@ func (s *ociStorage) PutObject(ctx context.Context, storagePath string, data []b return wrapOCIObjectError(err) } -// UploadObject sends a body that fits one part with PutObject, otherwise a +// UploadObject sends a body smaller than a part with PutObject, otherwise a // multipart upload of sequential parts through one buffer. IfAbsent is checked // at create and again at commit, so it is atomic here. func (s *ociStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { // The condition is checked again at create and commit; this check spares a // refused upload its traffic, and refuses it where a store ignores // if-none-match on multipart uploads, as floci-oci does. - if opts.IfAbsent { - exists, err := s.ObjectExists(ctx, storagePath) - if err != nil { - return err - } - if exists { - return ErrObjectExists - } - } - part := make([]byte, ociUploadPartSize) - n, err := io.ReadFull(body, part) - if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { - return s.PutObject(ctx, storagePath, part[:n], opts) + if opts.IfAbsent && presentBeforeUpload(ctx, s, storagePath) { + return ErrExists } - if err != nil { + // The buffer grows only as far as the body goes, so a small one costs no + // whole part. + var first bytes.Buffer + if _, err := first.ReadFrom(io.LimitReader(body, ociUploadPartSize)); err != nil { return err } + if first.Len() < ociUploadPartSize { + return s.PutObject(ctx, storagePath, first.Bytes(), opts) + } + part := first.Bytes() var ifNoneMatch, contentType *string if opts.IfAbsent { @@ -260,15 +279,16 @@ func (s *ociStorage) UploadObject(ctx context.Context, storagePath string, body BucketName: &s.conf.Bucket, IfNoneMatch: ifNoneMatch, CreateMultipartUploadDetails: objectstorage.CreateMultipartUploadDetails{ - Object: &storagePath, - ContentType: contentType, + Object: &storagePath, + ContentType: contentType, + ContentDisposition: ociContentDisposition, }, }) if err != nil { return wrapOCIObjectError(err) } - if err = s.uploadParts(ctx, storagePath, created.UploadId, body, part, n, ifNoneMatch); err != nil { + if err = s.uploadParts(ctx, storagePath, created.UploadId, body, part, len(part), ifNoneMatch); err != nil { _, _ = s.client.AbortMultipartUpload(context.WithoutCancel(ctx), objectstorage.AbortMultipartUploadRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, @@ -376,7 +396,7 @@ func (s *ociStorage) ObjectExists(ctx context.Context, storagePath string) (bool ObjectName: &storagePath, }) err = wrapOCIObjectError(err) - if errors.Is(err, ErrObjectNotFound) { + if errors.Is(err, ErrNotFound) { return false, nil } return err == nil, err @@ -412,15 +432,15 @@ func (s *ociStorage) DeleteObjects(ctx context.Context, storagePaths []string) e BucketName: &s.conf.Bucket, ObjectName: &p, }) - if err = wrapOCIObjectError(err); err != nil && !errors.Is(err, ErrObjectNotFound) { + if err = wrapOCIObjectError(err); err != nil && !errors.Is(err, ErrNotFound) { return err } } return nil } -func (s *ociStorage) GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { - return s.presignedUrl(ctx, storagePath, expiration) +func (s *ociStorage) PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { + return s.presignedURL(ctx, storagePath, expiration) } func wrapOCIObjectError(err error) error { @@ -435,10 +455,10 @@ func wrapOCIObjectError(err error) error { // NotAuthorizedOrNotFound stays an error. switch se.GetCode() { case "ObjectNotFound", "BadErrorResponse": - return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapOCIError(err)) + return fmt.Errorf("%w: %w", ErrNotFound, wrapOCIError(err)) } case http.StatusPreconditionFailed: - return fmt.Errorf("%w: %w", ErrObjectExists, wrapOCIError(err)) + return fmt.Errorf("%w: %w", ErrExists, wrapOCIError(err)) } } return wrapOCIError(err) diff --git a/s3.go b/s3.go index 760483a..a19f1c3 100644 --- a/s3.go +++ b/s3.go @@ -18,7 +18,6 @@ import ( "bytes" "cmp" "context" - "encoding/base64" "errors" "fmt" "io" @@ -31,6 +30,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/retry" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/credentials/stscreds" @@ -82,11 +82,11 @@ func NewS3(conf *S3Config) (Storage, error) { return nil, err } - if conf.AssumeRoleArn != "" { + if conf.AssumeRoleARN != "" { stsSvc := sts.NewFromConfig(*awsConf) - cp = stscreds.NewAssumeRoleProvider(stsSvc, conf.AssumeRoleArn, func(o *stscreds.AssumeRoleOptions) { - if conf.AssumeRoleExternalId != "" { - o.ExternalID = aws.String(conf.AssumeRoleExternalId) + cp = stscreds.NewAssumeRoleProvider(stsSvc, conf.AssumeRoleARN, func(o *stscreds.AssumeRoleOptions) { + if conf.AssumeRoleExternalID != "" { + o.ExternalID = aws.String(conf.AssumeRoleExternalID) } }) awsConf, err = getConf(conf, cp) @@ -129,21 +129,20 @@ func getConf(conf *S3Config, cp aws.CredentialsProvider) (*aws.Config, error) { }) } - s3Transport := transportWithTimeouts(nil) - if conf.ProxyConfig != nil { - proxyUrl, err := url.Parse(conf.ProxyConfig.Url) - if err != nil { - return err - } - s3Transport.Proxy = http.ProxyURL(proxyUrl) - if conf.ProxyConfig.Username != "" && conf.ProxyConfig.Password != "" { - auth := fmt.Sprintf("%s:%s", conf.ProxyConfig.Username, conf.ProxyConfig.Password) - basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth)) - s3Transport.ProxyConnectHeader = http.Header{} - s3Transport.ProxyConnectHeader.Add("Proxy-Authorization", basicAuth) - } + // The SDK's own client, which keeps its pool defaults and is the only + // kind a custom CA bundle (AWS_CA_BUNDLE) can be added to. + setProxy, err := proxySetter(conf.ProxyConfig) + if err != nil { + return err } - o.HTTPClient = &http.Client{Transport: s3Transport} + // The SDK's read timeout is off, as setTimeouts bounds reads and writes. + // Legacy defaults keep each client from replacing the dialer, and with + // it the idle timeout, as AWS_DEFAULTS_MODE would. + o.DefaultsModeOptions.Mode = aws.DefaultsModeLegacy + o.HTTPClient = awshttp.NewBuildableClient(). + WithDialerOptions(setDialTimeouts). + WithTransportOptions(setTimeouts, setProxy). + WithReadTimeout(0) return nil } @@ -194,13 +193,6 @@ func (s *s3Storage) getClient(l logging.Logger) *s3.Client { }) } -func (s *s3Storage) UploadData(ctx context.Context, data []byte, storagePath, contentType string) (string, int64, error) { - if err := s.upload(ctx, bytes.NewReader(data), storagePath, contentType); err != nil { - return "", 0, err - } - return s.location(storagePath), int64(len(data)), nil -} - func (s *s3Storage) UploadFile(ctx context.Context, filepath, storagePath, contentType string) (string, int64, error) { file, err := os.Open(filepath) if err != nil { @@ -219,7 +211,7 @@ func (s *s3Storage) UploadFile(ctx context.Context, filepath, storagePath, conte } func (s *s3Storage) upload(ctx context.Context, body io.Reader, storagePath, contentType string) error { - l := NewS3Logger() + l := news3Logger() if _, err := s.uploader(s.getClient(l)).Upload(ctx, s.putInput(storagePath, body, PutOptions{ContentType: contentType})); err != nil { l.WriteLogs() return wrapS3Error(err) @@ -275,7 +267,6 @@ func (s *s3Storage) location(storagePath string) string { return loc.String() } -// DownloadFile fetches ranges in parallel into the file. func (s *s3Storage) DownloadFile(ctx context.Context, filepath, storagePath string) (int64, error) { file, err := os.Create(filepath) if err != nil { @@ -283,7 +274,12 @@ func (s *s3Storage) DownloadFile(ctx context.Context, filepath, storagePath stri } defer file.Close() - n, err := manager.NewDownloader(s.getClient(nil)).Download(ctx, file, &s3.GetObjectInput{ + return s.downloadRanges(ctx, file, storagePath) +} + +// downloadRanges fetches the object in parallel ranges, each retried on its own. +func (s *s3Storage) downloadRanges(ctx context.Context, w io.WriterAt, storagePath string) (int64, error) { + n, err := manager.NewDownloader(s.getClient(nil)).Download(ctx, w, &s3.GetObjectInput{ Bucket: aws.String(s.conf.Bucket), Key: aws.String(storagePath), }) @@ -302,46 +298,23 @@ func (s *s3Storage) PutObject(ctx context.Context, storagePath string, data []by } // UploadObject checks IfAbsent before uploading, which spares a refused upload -// its traffic, and again when it completes. +// its traffic, and in the request that completes it: the uploader copies the +// input's IfNoneMatch into PutObject or CompleteMultipartUpload. func (s *s3Storage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { - if opts.IfAbsent { - exists, err := s.ObjectExists(ctx, storagePath) - if err != nil { - return err - } - if exists { - return ErrObjectExists - } + if opts.IfAbsent && presentBeforeUpload(ctx, s, storagePath) { + return ErrExists } input := s.putInput(storagePath, body, opts) uploader := s.uploader(s.getClient(nil), func(u *manager.Uploader) { u.PartSize = s3UploadPartSize - if opts.IfAbsent { - // A body that fits one part goes up as this PutObject; a larger one - // is completed with the same condition, which the uploader drops. - input.IfNoneMatch = aws.String("*") - u.ClientOptions = append(u.ClientOptions, completeIfAbsent) - } }) + if opts.IfAbsent { + input.IfNoneMatch = aws.String("*") + } _, err := uploader.Upload(ctx, input) return wrapS3ObjectError(err) } -func completeIfAbsent(o *s3.Options) { - o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { - return stack.Initialize.Add(middleware.InitializeMiddlewareFunc("CompleteIfAbsent", - func(ctx context.Context, in middleware.InitializeInput, next middleware.InitializeHandler) ( - middleware.InitializeOutput, middleware.Metadata, error, - ) { - if v, ok := in.Parameters.(*s3.CompleteMultipartUploadInput); ok { - v.IfNoneMatch = aws.String("*") - } - return next.HandleInitialize(ctx, in) - }, - ), middleware.Before) - }) -} - // putInput is every write's request: the content type, and the metadata, // tagging and content disposition (inline by default) the config sets. func (s *s3Storage) putInput(storagePath string, body io.Reader, opts PutOptions) *s3.PutObjectInput { @@ -362,7 +335,11 @@ func (s *s3Storage) putInput(storagePath string, body io.Reader, opts PutOptions } func (s *s3Storage) GetObject(ctx context.Context, storagePath string) ([]byte, error) { - return getObject(ctx, s, storagePath) + w := manager.NewWriteAtBuffer(nil) + if _, err := s.downloadRanges(ctx, w, storagePath); err != nil { + return nil, err + } + return w.Bytes(), nil } func (s *s3Storage) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { @@ -407,7 +384,7 @@ func (s *s3Storage) ObjectExists(ctx context.Context, storagePath string) (bool, Key: aws.String(storagePath), }) err = wrapS3ObjectError(err) - if errors.Is(err, ErrObjectNotFound) { + if errors.Is(err, ErrNotFound) { return false, nil } return err == nil, err @@ -451,16 +428,22 @@ func (s *s3Storage) DeleteObjects(ctx context.Context, storagePaths []string) er if err != nil { return wrapS3Error(err) } - if len(out.Errors) > 0 { - e := out.Errors[0] - return fmt.Errorf("storage: %d of %d deletes failed, first %s: %s %s", - len(out.Errors), len(batch), aws.ToString(e.Key), aws.ToString(e.Code), aws.ToString(e.Message)) + // Any key the store reports that is not already gone is still there, + // so the call fails rather than leave it behind unseen. + var errs []error + for _, e := range out.Errors { + if aws.ToString(e.Code) != "NoSuchKey" { + errs = append(errs, fmt.Errorf("storage: delete %s: %s %s", aws.ToString(e.Key), aws.ToString(e.Code), aws.ToString(e.Message))) + } + } + if err := errors.Join(errs...); err != nil { + return err } } return nil } -func (s *s3Storage) GeneratePresignedUrl(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { +func (s *s3Storage) PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { res, err := s3.NewPresignClient(s.getClient(nil)).PresignGetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(s.conf.Bucket), Key: aws.String(storagePath), @@ -471,7 +454,7 @@ func (s *s3Storage) GeneratePresignedUrl(ctx context.Context, storagePath string return res.URL, nil } -// wrapS3ObjectError is wrapS3Error with ErrObjectNotFound and ErrObjectExists +// wrapS3ObjectError is wrapS3Error with ErrNotFound and ErrExists // for the codes S3-compatible stores answer them with. func wrapS3ObjectError(err error) error { if err == nil { @@ -480,15 +463,15 @@ func wrapS3ObjectError(err error) error { var noSuchKey *types.NoSuchKey var notFound *types.NotFound if errors.As(err, &noSuchKey) || errors.As(err, ¬Found) { - return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapS3Error(err)) + return fmt.Errorf("%w: %w", ErrNotFound, wrapS3Error(err)) } var apiErr smithy.APIError if errors.As(err, &apiErr) { switch apiErr.ErrorCode() { case "NoSuchKey", "NotFound", "404": - return fmt.Errorf("%w: %w", ErrObjectNotFound, wrapS3Error(err)) + return fmt.Errorf("%w: %w", ErrNotFound, wrapS3Error(err)) case "PreconditionFailed": - return fmt.Errorf("%w: %w", ErrObjectExists, wrapS3Error(err)) + return fmt.Errorf("%w: %w", ErrExists, wrapS3Error(err)) } } return wrapS3Error(err) diff --git a/storage.go b/storage.go index b2c0047..687629c 100644 --- a/storage.go +++ b/storage.go @@ -17,8 +17,10 @@ package storage import ( "bytes" "context" + "encoding/base64" "net" "net/http" + "net/url" "time" "github.com/livekit/storage/objects" @@ -33,14 +35,23 @@ type ( ) var ( - ErrObjectNotFound = objects.ErrNotFound - ErrObjectExists = objects.ErrExists + ErrNotFound = objects.ErrNotFound + ErrExists = objects.ErrExists ) func New(conf Config) (Storage, error) { return newStorage(conf) } +// presentBeforeUpload reports whether IfAbsent can refuse an upload before it is +// sent, because the object is there. A check that fails, such as a 403 for +// credentials that may write but not read, leaves the refusal to the condition +// the store checks when the upload completes. +func presentBeforeUpload(ctx context.Context, b Bucket, storagePath string) bool { + exists, err := b.ObjectExists(ctx, storagePath) + return err == nil && exists +} + // getObject is GetObject for a backend whose reads are streams. func getObject(ctx context.Context, s Bucket, storagePath string) ([]byte, error) { var buf bytes.Buffer @@ -50,21 +61,101 @@ func getObject(ctx context.Context, s Bucket, storagePath string) ([]byte, error return buf.Bytes(), nil } -// responseHeaderTimeout bounds the wait for a response's headers on the clients -// this package builds. It catches a dead connection without a deadline on the -// body, so a long transfer is never cut; a call's context bounds the call. -const responseHeaderTimeout = time.Minute +// The clients this package builds cut a connection that stalls, never a slow +// but live transfer: a call's context bounds the call. +const ( + dialTimeout = 30 * time.Second + tlsHandshakeTimeout = 10 * time.Second +) + +// idleTimeout is how long a connection may move no bytes either way. A store +// can take minutes to answer a large commit. +var idleTimeout = 5 * time.Minute + +func setDialTimeouts(d *net.Dialer) { + d.Timeout = dialTimeout + d.KeepAlive = 30 * time.Second +} + +// setTimeouts sets t's TLS timeout and has its connections time out when idle. +func setTimeouts(t *http.Transport) { + t.TLSHandshakeTimeout = tlsHandshakeTimeout + setIdleTimeout(t) +} + +// setIdleTimeout has t's connections fail once idle for idleTimeout. A dialer t +// already has, such as one through a VPN, is kept. +func setIdleTimeout(t *http.Transport) { + dial := t.DialContext + if dial == nil { + d := &net.Dialer{} + setDialTimeouts(d) + dial = d.DialContext + } + t.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + conn, err := dial(ctx, network, addr) + if err != nil { + return nil, err + } + return idleConn{conn}, nil + } +} + +// idleConn fails once no bytes move either way for idleTimeout. Each read or +// write moves both deadlines: the transport's read for the response is already +// waiting while a long body is still being written. +type idleConn struct{ net.Conn } + +func (c idleConn) Read(p []byte) (int, error) { + if err := c.SetDeadline(time.Now().Add(idleTimeout)); err != nil { + return 0, err + } + return c.Conn.Read(p) +} + +func (c idleConn) Write(p []byte) (int, error) { + if err := c.SetDeadline(time.Now().Add(idleTimeout)); err != nil { + return 0, err + } + return c.Conn.Write(p) +} -// transportWithTimeouts is t, or Go's default transport, with dial, TLS and -// response header timeouts. +// transportWithTimeouts is a clone of t, or of Go's default transport, with +// every timeout set. func transportWithTimeouts(rt http.RoundTripper) *http.Transport { + // A wrapped transport, such as a tracing one, cannot be cloned, so the + // client gets a plain one with Go's defaults instead. t, ok := rt.(*http.Transport) - if !ok { - t = http.DefaultTransport.(*http.Transport) + if ok { + t = t.Clone() + } else { + t = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + ExpectContinueTimeout: time.Second, + } } - t = t.Clone() - t.DialContext = (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext - t.TLSHandshakeTimeout = 10 * time.Second - t.ResponseHeaderTimeout = responseHeaderTimeout + setTimeouts(t) return t } + +// proxySetter routes a transport through p, or leaves it as it is when p is +// nil. A bad URL fails here, so the setter itself can be a transport option. +func proxySetter(p *ProxyConfig) (func(*http.Transport), error) { + if p == nil { + return func(*http.Transport) {}, nil + } + u, err := url.Parse(p.URL) + if err != nil { + return nil, err + } + return func(t *http.Transport) { + t.Proxy = http.ProxyURL(u) + if p.Username != "" && p.Password != "" { + auth := base64.StdEncoding.EncodeToString([]byte(p.Username + ":" + p.Password)) + t.ProxyConnectHeader = http.Header{"Proxy-Authorization": {"Basic " + auth}} + } + }, nil +} diff --git a/transport_test.go b/transport_test.go new file mode 100644 index 0000000..c33cdef --- /dev/null +++ b/transport_test.go @@ -0,0 +1,108 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// A connection is cut when no bytes move for idleTimeout, and never while a +// slow transfer is still moving them. +func TestIdleTimeout(t *testing.T) { + idleTimeout = 300 * time.Millisecond + t.Cleanup(func() { idleTimeout = 5 * time.Minute }) + client := &http.Client{Transport: transportWithTimeouts(nil)} + + t.Run("slow upload", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + })) + t.Cleanup(srv.Close) + // The second request reuses the first's connection. + for range 2 { + resp, err := client.Post(srv.URL, "text/plain", &trickle{n: 10}) + require.NoError(t, err) + resp.Body.Close() + } + }) + + t.Run("slow download", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(w, &trickle{n: 10, flush: w.(http.Flusher)}) + })) + t.Cleanup(srv.Close) + resp, err := client.Get(srv.URL) + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Len(t, body, 10) + }) + + t.Run("stalled", func(t *testing.T) { + start := time.Now() + _, err := client.Get(stalled(t)) + require.Error(t, err) + require.Less(t, time.Since(start), 5*time.Second) + }) + + // Other defaults modes have the SDK replace the dialer the timeout is on. + t.Run("S3 in standard defaults mode", func(t *testing.T) { + t.Setenv("AWS_DEFAULTS_MODE", "standard") + s, err := NewS3(&S3Config{AccessKey: "key", Secret: "secret", Region: "us-east-1", Endpoint: stalled(t), ForcePathStyle: true, Bucket: "bucket"}) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + _, err = s.ObjectExists(ctx, "path") + require.Error(t, err) + require.NoError(t, ctx.Err(), "the context, not the idle timeout, ended the call") + }) +} + +// stalled is a server that accepts requests and never answers them. +func stalled(t *testing.T) string { + done := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-done + })) + t.Cleanup(func() { close(done); srv.Close() }) + return srv.URL +} + +// trickle yields n bytes, one every 100ms, a third of the idle timeout. +type trickle struct { + n int + flush http.Flusher +} + +func (r *trickle) Read(p []byte) (int, error) { + if r.n == 0 { + return 0, io.EOF + } + if r.flush != nil { + r.flush.Flush() + } + time.Sleep(100 * time.Millisecond) + r.n-- + p[0] = 'x' + return 1, nil +} From f0297f9e5d8439490b26bc963d40a262cc9ad66f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 10:37:18 -0700 Subject: [PATCH 12/29] Storage: deletes run 16 at a time on GCS, Azure and OCI; one S3 client per Storage; legacy defaults mode explained --- azure.go | 10 ++++----- gcp.go | 6 +++--- go.mod | 2 +- oci.go | 6 +++--- s3.go | 61 ++++++++++++++++++++++++++++++------------------------ storage.go | 32 ++++++++++++++++++++++++++++ 6 files changed, 78 insertions(+), 39 deletions(-) diff --git a/azure.go b/azure.go index 4ccdd53..d7a09ba 100644 --- a/azure.go +++ b/azure.go @@ -315,16 +315,16 @@ func (s *azureBLOBStorage) ListObjects(ctx context.Context, prefix string) ([]st return keys, nil } +// DeleteObjects fails on a blob with snapshots, as on main: deleting someone's +// snapshots along with it is not this call's to decide. func (s *azureBLOBStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { - for _, p := range storagePaths { - // A blob with snapshots is not deleted, as on Storage before: deleting - // someone's snapshots along with it is not this call's to decide. + return deleteEach(ctx, storagePaths, func(ctx context.Context, p string) error { _, err := s.client.DeleteBlob(ctx, s.conf.ContainerName, p, nil) if err != nil && !bloberror.HasCode(err, bloberror.BlobNotFound) { return wrapAzureObjectError(err) } - } - return nil + return nil + }) } // PresignedURL signs with the account key unless a token credential is diff --git a/gcp.go b/gcp.go index 11dd0a5..3ce511f 100644 --- a/gcp.go +++ b/gcp.go @@ -270,12 +270,12 @@ func (s *gcpStorage) ListObjects(ctx context.Context, prefix string) ([]string, } func (s *gcpStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { - for _, p := range storagePaths { + return deleteEach(ctx, storagePaths, func(ctx context.Context, p string) error { if err := s.object(p).Delete(ctx); err != nil && !errors.Is(err, storage.ErrObjectNotExist) { return wrapGCPObjectError(err) } - } - return nil + return nil + }) } func (s *gcpStorage) PresignedURL(_ context.Context, storagePath string, expiration time.Duration) (string, error) { diff --git a/go.mod b/go.mod index 7d88909..17f612a 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/oracle/oci-go-sdk/v65 v65.126.1 github.com/stretchr/testify v1.12.1 golang.org/x/oauth2 v0.37.0 + golang.org/x/sync v0.23.0 google.golang.org/api v0.300.0 ) @@ -88,7 +89,6 @@ require ( golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirect golang.org/x/net v0.59.0 // indirect - golang.org/x/sync v0.23.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.16.0 // indirect diff --git a/oci.go b/oci.go index 8c453f6..b8c2b47 100644 --- a/oci.go +++ b/oci.go @@ -426,7 +426,7 @@ func (s *ociStorage) ListObjects(ctx context.Context, prefix string) ([]string, } func (s *ociStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { - for _, p := range storagePaths { + return deleteEach(ctx, storagePaths, func(ctx context.Context, p string) error { _, err := s.client.DeleteObject(ctx, objectstorage.DeleteObjectRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, @@ -435,8 +435,8 @@ func (s *ociStorage) DeleteObjects(ctx context.Context, storagePaths []string) e if err = wrapOCIObjectError(err); err != nil && !errors.Is(err, ErrNotFound) { return err } - } - return nil + return nil + }) } func (s *ociStorage) PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { diff --git a/s3.go b/s3.go index a19f1c3..d0a9d1b 100644 --- a/s3.go +++ b/s3.go @@ -39,7 +39,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/aws/smithy-go" - "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" ) @@ -60,8 +59,8 @@ func wrapS3Error(err error) error { const defaultBucketLocation = "us-east-1" type s3Storage struct { - conf *S3Config - awsConf *aws.Config + conf *S3Config + client *s3.Client } func NewS3(conf *S3Config) (Storage, error) { @@ -102,8 +101,8 @@ func NewS3(conf *S3Config) (Storage, error) { } return &s3Storage{ - conf: conf, - awsConf: awsConf, + conf: conf, + client: newS3Client(awsConf, conf), }, nil } @@ -135,10 +134,14 @@ func getConf(conf *S3Config, cp aws.CredentialsProvider) (*aws.Config, error) { if err != nil { return err } - // The SDK's read timeout is off, as setTimeouts bounds reads and writes. - // Legacy defaults keep each client from replacing the dialer, and with - // it the idle timeout, as AWS_DEFAULTS_MODE would. + // "Legacy" is the SDK's default when AWS_DEFAULTS_MODE is unset, so + // this pins what main ran with. Every other mode only swaps in its own + // connect and TLS timeouts (1.1s to 30s), and does it by replacing the + // dialer, which drops the idle timeout setTimeouts puts on it. This + // package sets those timeouts itself, and retries with its own retryer, + // which no mode changes. o.DefaultsModeOptions.Mode = aws.DefaultsModeLegacy + // The SDK's read timeout is off: setTimeouts bounds reads and writes. o.HTTPClient = awshttp.NewBuildableClient(). WithDialerOptions(setDialTimeouts). WithTransportOptions(setTimeouts, setProxy). @@ -176,16 +179,13 @@ func updateRegion(awsConf *aws.Config, bucket string) error { return nil } -func (s *s3Storage) getClient(l logging.Logger) *s3.Client { - return s3.NewFromConfig(*s.awsConf, func(o *s3.Options) { - if l != nil { - o.Logger = l - o.ClientLogMode = aws.LogRequest | aws.LogResponse | aws.LogRetries - } +// newS3Client is the one client a Storage sends with, so calls share its +// connections. +func newS3Client(awsConf *aws.Config, conf *S3Config) *s3.Client { + return s3.NewFromConfig(*awsConf, func(o *s3.Options) { + o.UsePathStyle = conf.ForcePathStyle - o.UsePathStyle = s.conf.ForcePathStyle - - if s.conf.Endpoint != "" { + if conf.Endpoint != "" { // needed for compatibility with non-AWS S3-compatible providers (Oracle, etc.) o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired @@ -211,8 +211,15 @@ func (s *s3Storage) UploadFile(ctx context.Context, filepath, storagePath, conte } func (s *s3Storage) upload(ctx context.Context, body io.Reader, storagePath, contentType string) error { + // The SDK's request log is kept, and written only when the upload fails. l := news3Logger() - if _, err := s.uploader(s.getClient(l)).Upload(ctx, s.putInput(storagePath, body, PutOptions{ContentType: contentType})); err != nil { + logTo := func(u *manager.Uploader) { + u.ClientOptions = append(u.ClientOptions, func(o *s3.Options) { + o.Logger = l + o.ClientLogMode = aws.LogRequest | aws.LogResponse | aws.LogRetries + }) + } + if _, err := s.uploader(s.client, logTo).Upload(ctx, s.putInput(storagePath, body, PutOptions{ContentType: contentType})); err != nil { l.WriteLogs() return wrapS3Error(err) } @@ -279,7 +286,7 @@ func (s *s3Storage) DownloadFile(ctx context.Context, filepath, storagePath stri // downloadRanges fetches the object in parallel ranges, each retried on its own. func (s *s3Storage) downloadRanges(ctx context.Context, w io.WriterAt, storagePath string) (int64, error) { - n, err := manager.NewDownloader(s.getClient(nil)).Download(ctx, w, &s3.GetObjectInput{ + n, err := manager.NewDownloader(s.client).Download(ctx, w, &s3.GetObjectInput{ Bucket: aws.String(s.conf.Bucket), Key: aws.String(storagePath), }) @@ -293,7 +300,7 @@ func (s *s3Storage) PutObject(ctx context.Context, storagePath string, data []by if opts.IfAbsent { input.IfNoneMatch = aws.String("*") } - _, err := s.getClient(nil).PutObject(ctx, input) + _, err := s.client.PutObject(ctx, input) return wrapS3ObjectError(err) } @@ -305,7 +312,7 @@ func (s *s3Storage) UploadObject(ctx context.Context, storagePath string, body i return ErrExists } input := s.putInput(storagePath, body, opts) - uploader := s.uploader(s.getClient(nil), func(u *manager.Uploader) { + uploader := s.uploader(s.client, func(u *manager.Uploader) { u.PartSize = s3UploadPartSize }) if opts.IfAbsent { @@ -346,7 +353,7 @@ func (s *s3Storage) GetObjectRange(ctx context.Context, storagePath string, offs if offset < 0 || length <= 0 { return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) } - out, err := s.getClient(nil).GetObject(ctx, &s3.GetObjectInput{ + out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(s.conf.Bucket), Key: aws.String(storagePath), Range: aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)), @@ -367,7 +374,7 @@ func (s *s3Storage) GetObjectRange(ctx context.Context, storagePath string, offs } func (s *s3Storage) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { - out, err := s.getClient(nil).GetObject(ctx, &s3.GetObjectInput{ + out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(s.conf.Bucket), Key: aws.String(storagePath), }) @@ -379,7 +386,7 @@ func (s *s3Storage) DownloadObject(ctx context.Context, storagePath string, w io } func (s *s3Storage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { - _, err := s.getClient(nil).HeadObject(ctx, &s3.HeadObjectInput{ + _, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(s.conf.Bucket), Key: aws.String(storagePath), }) @@ -392,7 +399,7 @@ func (s *s3Storage) ObjectExists(ctx context.Context, storagePath string) (bool, func (s *s3Storage) ListObjects(ctx context.Context, prefix string) ([]string, error) { var keys []string - paginator := s3.NewListObjectsV2Paginator(s.getClient(nil), &s3.ListObjectsV2Input{ + paginator := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{ Bucket: aws.String(s.conf.Bucket), Prefix: aws.String(prefix), }) @@ -410,7 +417,7 @@ func (s *s3Storage) ListObjects(ctx context.Context, prefix string) ([]string, e // DeleteObjects fails when any key fails, even within a batch S3 accepted. func (s *s3Storage) DeleteObjects(ctx context.Context, storagePaths []string) error { - client := s.getClient(nil) + client := s.client for i := 0; i < len(storagePaths); i += 1000 { batch := storagePaths[i:min(i+1000, len(storagePaths))] objects := make([]types.ObjectIdentifier, 0, len(batch)) @@ -444,7 +451,7 @@ func (s *s3Storage) DeleteObjects(ctx context.Context, storagePaths []string) er } func (s *s3Storage) PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { - res, err := s3.NewPresignClient(s.getClient(nil)).PresignGetObject(ctx, &s3.GetObjectInput{ + res, err := s3.NewPresignClient(s.client).PresignGetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(s.conf.Bucket), Key: aws.String(storagePath), }, s3.WithPresignExpires(expiration)) diff --git a/storage.go b/storage.go index 687629c..3f6fc81 100644 --- a/storage.go +++ b/storage.go @@ -15,9 +15,12 @@ package storage import ( + "sync/atomic" + "bytes" "context" "encoding/base64" + "golang.org/x/sync/errgroup" "net" "net/http" "net/url" @@ -68,6 +71,35 @@ const ( tlsHandshakeTimeout = 10 * time.Second ) +// deleteConcurrency is how many deletes a DeleteObjects sends at once where a +// store takes one object per request (GCS, Azure, OCI): sequential, a large +// delete is one round trip per object. +const deleteConcurrency = 16 + +// deleteEach deletes every path with del, deleteConcurrency at a time, and +// returns the first error. A failure stops new deletes but lets those in +// flight finish, rather than cancel them: an OCI call canceled mid-flight +// races inside the SDK. A retry is safe, as del treats a missing path as +// deleted. +func deleteEach(ctx context.Context, paths []string, del func(ctx context.Context, path string) error) error { + var g errgroup.Group + g.SetLimit(deleteConcurrency) + var failed atomic.Bool + for _, p := range paths { + if failed.Load() { + break + } + g.Go(func() error { + err := del(ctx, p) + if err != nil { + failed.Store(true) + } + return err + }) + } + return g.Wait() +} + // idleTimeout is how long a connection may move no bytes either way. A store // can take minutes to answer a large commit. var idleTimeout = 5 * time.Minute From 90bc5ad6993e641e8a33c935e170f9670d89dc64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 11:03:49 -0700 Subject: [PATCH 13/29] Storage: fixes from seven reviews (API, backends, tests, best practices, enterprise) API: - UploadFile(ctx, storagePath, filepath, PutOptions) and DownloadFile(ctx, storagePath, filepath): key first, IfAbsent on files - ErrorWithStatusCode lives in objects, aliased in root - A precondition failure is ErrExists only on IfAbsent writes: S3's and Azure's readers pin the ETag, so a 412 on a read is no conflict - Listings sorted on every backend, as documented Robustness: - S3: no client-side retry quota (one client serves every call), backoff between MinRetryDelay and MaxRetryDelay, MaxRetries counts retries, multipart aborted on a context of its own, idle timeout kept under any AWS_DEFAULTS_MODE, checksums dropped only for non-AWS hosts (Object Lock behind VPC/FIPS endpoints), 16 MiB parts for files, bare 416, no session token in failure logs, bounded region lookup - GCS: own transport through the SDK hook (no swap of http.DefaultTransport, which raced with the process and leaked the proxy), tokens fetched through it, 10-minute chunk retry deadline, retried deletes, bounded presign retries, public host on presign behind a private endpoint - Azure: this package's transport (idle timeout, 32 idle connections), token-only credentials sign requests, SAS clock-skew allowance - AliOSS: retries on 5xx, 429 and network errors, which its SDK never does - OCI: abort in the background, bounded namespace lookup - Proxy: credentials for http targets too, NO_PROXY, metadata services never proxied - idleConn never extends a deadline set from above - DownloadFile writes through a temporary file, so a failed download leaves the destination as it was - Local: plain writes in place as on main (mounted volumes, file watchers), IfAbsent by link; listing survives concurrent deletes and prunes directories; directory keys read as not found; parent fsync Tests: - failed bodies store nothing, race winner read back, content type read back, DownloadObject not found, DownloadFile keeps the destination, multipart abort after a deadline, AliOSS retries, proxy credentials and NO_PROXY, idle deadlines from above, listing during deletes, no write to http.DefaultTransport under -race - CI caches through setup-go keyed on both modules --- .github/workflows/test.yaml | 28 +++--- README.md | 21 +++-- alioss.go | 93 ++++++++++++++++--- azure.go | 110 ++++++++++++----------- contract_test.go | 157 +++++++++++++++++++++++++++++--- e2e/object_test.go | 57 +++++++++++- e2e/storage_test.go | 9 +- gcp.go | 105 +++++++++++++--------- go.mod | 2 +- local.go | 142 +++++++++++++++-------------- logging.go | 3 +- objects/objects.go | 57 ++++++++---- oci.go | 70 ++++++++++----- s3.go | 172 +++++++++++++++++++++++++++--------- storage.go | 168 +++++++++++++++++++++++++++++++---- transport_test.go | 25 +++++- 16 files changed, 900 insertions(+), 319 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e821323..756a87d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,18 +15,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/go/pkg/mod - ~/go/bin - ~/.cache - key: livekit-storage - + # setup-go caches the module and build caches, keyed on both modules. - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: "go.mod" + cache-dependency-path: | + go.sum + e2e/go.sum - name: Set up gotestfmt run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@9eae5abc81d6d08f73268741ef4a8b97f29b60d8 # v2.4.1 @@ -45,8 +41,8 @@ jobs: version: v2.14.0 working-directory: e2e - # The e2e module holds the suites: here they run on the local backend, and - # on any real account whose secrets are set. + # The e2e module holds the suites: here they run on the local backend. The + # real-account tests skip, as CI passes them no credentials. - name: Test run: | set -euo pipefail @@ -65,18 +61,14 @@ jobs: test: [FlociS3, FlociAzure, FlociGCP, FlociOCI, MinIO] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/go/pkg/mod - ~/go/bin - ~/.cache - key: livekit-storage - + # setup-go caches the module and build caches, keyed on both modules. - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: "go.mod" + cache-dependency-path: | + go.sum + e2e/go.sum - name: Set up gotestfmt run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@9eae5abc81d6d08f73268741ef4a8b97f29b60d8 # v2.4.1 diff --git a/README.md b/README.md index 158d20b..c4c9d78 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,21 @@ # Storage -A GORM-style storage wrapper with unified API for different cloud storage backends +Object storage on S3 and S3-compatible stores, GCS, Azure Blob, OCI Object Storage, AliOSS and a local directory, behind one interface. `storage.New(conf)` returns a `Storage`; every call takes a context. -`storage.New(conf)` returns a `Storage` for S3 and S3-compatible stores, GCS, Azure, OCI, AliOSS or a local directory. Every call takes a context. - -- `Bucket` (`storage/objects`): `PutObject`, `UploadObject` (can refuse to replace an object, `PutOptions.IfAbsent`), `GetObject`, `GetObjectRange`, `DownloadObject` (to any `io.Writer`), `ObjectExists`, `ListObjects`, `DeleteObjects`, `PresignedURL`. A missing object is `ErrNotFound` and a refused write `ErrExists` (`errors.Is`). -- `Storage` is `Bucket` plus `UploadFile` and `DownloadFile`, which publish and fetch whole files. -- Every backend follows S3's semantics where providers differ. `DeleteObjects` of a missing path succeeds, as S3's delete does, and fails while any path it was given still holds an object, so a retry after a partial failure is safe. +- `Bucket` (`storage/objects`): `PutObject`, `UploadObject` (streams), `GetObject`, `GetObjectRange`, `DownloadObject` (to any `io.Writer`), `ObjectExists`, `ListObjects` (byte order), `DeleteObjects`, `PresignedURL`. +- `Storage` is `Bucket` plus `UploadFile` and `DownloadFile`. `DownloadFile` replaces the local file only once the whole object is read. +- `PutOptions{ContentType, IfAbsent}` on every write. `IfAbsent` refuses to replace an object, atomically where the store honors the condition. +- Errors: a missing object is `ErrNotFound`, a write `IfAbsent` refuses is `ErrExists` (`errors.Is`); `ErrorWithStatusCode` carries the store's HTTP status (`errors.As`). - `storage/objects` imports no provider SDK, so a fake or a library can implement or accept the interfaces without pulling in every SDK. -- The clients this package builds cut a connection that moves no bytes for 5 minutes, never a slow but live transfer; the context bounds the call. + +## Behavior + +- **S3 semantics** where stores differ. `DeleteObjects` of a missing path succeeds, as S3's delete does, and fails while any path it was given still holds an object, so a retry after a partial failure is safe. +- **Keys** are the store's keys as given. The local directory, as a filesystem, drops a leading `/`, cleans `//` and `.`, refuses keys outside it, and cannot hold both `a` and `a/b`. +- **Timeouts**: a connection that moves no bytes either way for 5 minutes is cut; a slow but live transfer never is. The context bounds the call. +- **Retries**: every store retries throttling, 5xx and network errors with backoff, including AliOSS, whose SDK does not. A stream given to `UploadObject` is sent once on AliOSS. +- **Proxy**: `ProxyConfig` (S3, GCS) carries its credentials for http and https targets, honors `NO_PROXY`, and never proxies the cloud metadata services. +- **Local writes** go in place, as a file is written, so mounted volumes and file watchers see one write; `IfAbsent` writes a temporary file (`.upload-*`, never listed) and links it into place. ## Tests diff --git a/alioss.go b/alioss.go index f7032ae..f4776d2 100644 --- a/alioss.go +++ b/alioss.go @@ -20,10 +20,14 @@ import ( "errors" "fmt" "io" + "math/rand/v2" + "net" "net/http" "net/url" "os" + "slices" "strings" + "syscall" "time" "github.com/aliyun/aliyun-oss-go-sdk/oss" @@ -72,13 +76,15 @@ func NewAliOSS(conf *AliOSSConfig) (Storage, error) { }, nil } -func (s *aliOSSStorage) UploadFile(ctx context.Context, filepath, storagePath, contentType string) (string, int64, error) { +func (s *aliOSSStorage) UploadFile(ctx context.Context, storagePath, filepath string, opts PutOptions) (string, int64, error) { info, err := os.Stat(filepath) if err != nil { return "", 0, err } - if err = s.bucket.PutObjectFromFile(storagePath, filepath, oss.ContentType(contentType), oss.WithContext(ctx)); err != nil { - return "", 0, wrapAliOSSError(err) + if err = aliOSSRetry(ctx, func() error { + return s.bucket.PutObjectFromFile(storagePath, filepath, aliOSSWriteOptions(ctx, opts)...) + }); err != nil { + return "", 0, wrapAliOSSObjectError(err) } return s.location(storagePath), info.Size(), nil } @@ -90,8 +96,12 @@ func (s *aliOSSStorage) location(storagePath string) string { return u.String() } -func (s *aliOSSStorage) DownloadFile(ctx context.Context, filepath, storagePath string) (int64, error) { - if err := s.bucket.GetObjectToFile(storagePath, filepath, oss.WithContext(ctx)); err != nil { +// DownloadFile has the SDK write a temporary file and rename it, as +// downloadFile does for the other stores. +func (s *aliOSSStorage) DownloadFile(ctx context.Context, storagePath, filepath string) (int64, error) { + if err := aliOSSRetry(ctx, func() error { + return s.bucket.GetObjectToFile(storagePath, filepath, oss.WithContext(ctx)) + }); err != nil { return 0, wrapAliOSSObjectError(err) } info, err := os.Stat(filepath) @@ -110,14 +120,23 @@ func (s *aliOSSStorage) presignedURL(storagePath string, expiration time.Duratio } func (s *aliOSSStorage) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { - return s.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) + return aliOSSRetry(ctx, func() error { + return s.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) + }) } // UploadObject streams body in one PutObject, chunked when its length is -// unknown, so it holds no part in memory and caps an object at 5 GB. IfAbsent -// sets x-oss-forbid-overwrite, atomic here, but OSS ignores it on a bucket with +// unknown, so it holds no part in memory and caps an object at 5 GB. It is +// sent once, as the body is read as it goes. IfAbsent sets +// x-oss-forbid-overwrite, atomic here, but OSS ignores it on a bucket with // versioning enabled or suspended. func (s *aliOSSStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + return wrapAliOSSObjectError(s.bucket.PutObject(storagePath, body, aliOSSWriteOptions(ctx, opts)...)) +} + +// aliOSSWriteOptions leaves out an empty content type, which the SDK then +// takes from the key's extension. +func aliOSSWriteOptions(ctx context.Context, opts PutOptions) []oss.Option { options := []oss.Option{oss.WithContext(ctx)} if opts.ContentType != "" { options = append(options, oss.ContentType(opts.ContentType)) @@ -125,7 +144,7 @@ func (s *aliOSSStorage) UploadObject(ctx context.Context, storagePath string, bo if opts.IfAbsent { options = append(options, oss.ForbidOverWrite(true)) } - return wrapAliOSSObjectError(s.bucket.PutObject(storagePath, body, options...)) + return options } func (s *aliOSSStorage) GetObject(ctx context.Context, storagePath string) ([]byte, error) { @@ -156,7 +175,11 @@ func (s *aliOSSStorage) DownloadObject(ctx context.Context, storagePath string, } func (s *aliOSSStorage) download(ctx context.Context, storagePath string, w io.Writer, options ...oss.Option) (int64, error) { - r, err := s.bucket.GetObject(storagePath, append(options, oss.WithContext(ctx))...) + var r io.ReadCloser + err := aliOSSRetry(ctx, func() (err error) { + r, err = s.bucket.GetObject(storagePath, append(options, oss.WithContext(ctx))...) + return err + }) if err != nil { return 0, wrapAliOSSObjectError(err) } @@ -166,7 +189,11 @@ func (s *aliOSSStorage) download(ctx context.Context, storagePath string, w io.W } func (s *aliOSSStorage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { - exists, err := s.bucket.IsObjectExist(storagePath, oss.WithContext(ctx)) + var exists bool + err := aliOSSRetry(ctx, func() (err error) { + exists, err = s.bucket.IsObjectExist(storagePath, oss.WithContext(ctx)) + return err + }) return exists, wrapAliOSSObjectError(err) } @@ -174,7 +201,11 @@ func (s *aliOSSStorage) ListObjects(ctx context.Context, prefix string) ([]strin var keys []string options := []oss.Option{oss.Prefix(prefix), oss.MaxKeys(1000), oss.WithContext(ctx)} for { - res, err := s.bucket.ListObjectsV2(options...) + var res oss.ListObjectsResultV2 + err := aliOSSRetry(ctx, func() (err error) { + res, err = s.bucket.ListObjectsV2(options...) + return err + }) if err != nil { return nil, wrapAliOSSObjectError(err) } @@ -182,6 +213,7 @@ func (s *aliOSSStorage) ListObjects(ctx context.Context, prefix string) ([]strin keys = append(keys, obj.Key) } if !res.IsTruncated || res.NextContinuationToken == "" { + slices.Sort(keys) return keys, nil } options = []oss.Option{oss.Prefix(prefix), oss.MaxKeys(1000), oss.ContinuationToken(res.NextContinuationToken), oss.WithContext(ctx)} @@ -194,7 +226,11 @@ func (s *aliOSSStorage) ListObjects(ctx context.Context, prefix string) ([]strin func (s *aliOSSStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { for i := 0; i < len(storagePaths); i += 1000 { batch := storagePaths[i:min(i+1000, len(storagePaths))] - res, err := s.bucket.DeleteObjects(batch, oss.WithContext(ctx)) + var res oss.DeleteObjectsResult + err := aliOSSRetry(ctx, func() (err error) { + res, err = s.bucket.DeleteObjects(batch, oss.WithContext(ctx)) + return err + }) if err != nil { return wrapAliOSSObjectError(err) } @@ -222,6 +258,37 @@ func (s *aliOSSStorage) PresignedURL(_ context.Context, storagePath string, expi return s.presignedURL(storagePath, expiration) } +// aliOSSAttempts is how many times a call is sent: the v1 SDK never retries. +const aliOSSAttempts = 3 + +// aliOSSRetry runs call again after a 5xx, a 429 or a network error, with +// backoff, until it succeeds or aliOSSAttempts is spent. Only calls whose body, +// if any, is sent anew each time are retried. +func aliOSSRetry(ctx context.Context, call func() error) error { + var err error + for attempt := range aliOSSAttempts { + if err = call(); err == nil || ctx.Err() != nil || !aliOSSRetryable(err) { + return err + } + backoff := 200 * time.Millisecond << attempt + select { + case <-ctx.Done(): + return err + case <-time.After(backoff/2 + rand.N(backoff)): + } + } + return err +} + +func aliOSSRetryable(err error) bool { + var svcErr oss.ServiceError + if errors.As(err, &svcErr) { + return svcErr.StatusCode >= 500 || svcErr.StatusCode == http.StatusTooManyRequests + } + var netErr net.Error + return errors.As(err, &netErr) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.ECONNRESET) +} + func wrapAliOSSObjectError(err error) error { if err == nil { return nil diff --git a/azure.go b/azure.go index d7a09ba..e1a0607 100644 --- a/azure.go +++ b/azure.go @@ -20,13 +20,16 @@ import ( "errors" "fmt" "io" + "net/http" "net/url" "os" "path" + "slices" "strings" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" @@ -57,29 +60,34 @@ type azureBLOBStorage struct { client *azblob.Client cred *azblob.SharedKeyCredential serviceURL string + options policy.ClientOptions } +// NewAzure signs requests with the account key, or with TokenCredential +// where no key is set, as with a managed or workload identity. func NewAzure(conf *AzureConfig) (Storage, error) { - cred, err := azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey) - if err != nil { - return nil, err - } - serviceURL := azureServiceURL(conf) if _, err := url.Parse(serviceURL); err != nil { return nil, fmt.Errorf("azure endpoint: %w", err) } - client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, nil) - if err != nil { - return nil, err - } - - return &azureBLOBStorage{ + s := &azureBLOBStorage{ conf: conf, - client: client, - cred: cred, serviceURL: serviceURL, - }, nil + // azcore's own transport has no idle timeout, and keeps fewer idle + // connections than an upload sends blocks at once. + options: policy.ClientOptions{Transport: &http.Client{Transport: transportWithTimeouts(http.DefaultTransport)}}, + } + + var err error + if conf.AccountKey == "" && conf.TokenCredential != nil { + s.client, err = azblob.NewClient(serviceURL, conf.TokenCredential, &azblob.ClientOptions{ClientOptions: s.options}) + return s, err + } + if s.cred, err = azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey); err != nil { + return nil, err + } + s.client, err = azblob.NewClientWithSharedKeyCredential(serviceURL, s.cred, &azblob.ClientOptions{ClientOptions: s.options}) + return s, err } func (s *azureBLOBStorage) location(storagePath string) string { @@ -101,52 +109,38 @@ func (s *azureBLOBStorage) blobURL(storagePath string) *url.URL { return u } -func (s *azureBLOBStorage) UploadFile(ctx context.Context, filepath, storagePath, contentType string) (string, int64, error) { +func (s *azureBLOBStorage) UploadFile(ctx context.Context, storagePath, filepath string, opts PutOptions) (string, int64, error) { file, err := os.Open(filepath) if err != nil { return "", 0, err } - defer func() { - _ = file.Close() - }() + defer file.Close() stat, err := file.Stat() if err != nil { return "", 0, err } - + headers, conditions := azureWriteOptions(opts) _, err = s.client.UploadFile(ctx, s.conf.ContainerName, storagePath, file, &azblob.UploadFileOptions{ - HTTPHeaders: &blob.HTTPHeaders{BlobContentType: &contentType}, - BlockSize: azureBlockSize, - Concurrency: azureParallelism, + HTTPHeaders: headers, + AccessConditions: conditions, + BlockSize: azureBlockSize, + Concurrency: azureParallelism, }) if err != nil { - return "", 0, wrapAzureError(err) + return "", 0, wrapAzureWriteError(err, opts.IfAbsent) } - return s.location(storagePath), stat.Size(), nil } -func (s *azureBLOBStorage) DownloadFile(ctx context.Context, filepath, storagePath string) (int64, error) { - file, err := os.Create(filepath) - if err != nil { - return 0, err - } - defer file.Close() - - _, err = s.client.DownloadFile(ctx, s.conf.ContainerName, storagePath, file, &azblob.DownloadFileOptions{ - BlockSize: azureBlockSize, - Concurrency: azureParallelism, +func (s *azureBLOBStorage) DownloadFile(ctx context.Context, storagePath, filepath string) (int64, error) { + return downloadFile(filepath, func(f *os.File) (int64, error) { + n, err := s.client.DownloadFile(ctx, s.conf.ContainerName, storagePath, f, &azblob.DownloadFileOptions{ + BlockSize: azureBlockSize, + Concurrency: azureParallelism, + }) + return n, wrapAzureObjectError(err) }) - if err != nil { - return 0, wrapAzureObjectError(err) - } - - stat, err := file.Stat() - if err != nil { - return 0, err - } - return stat.Size(), nil } func (s *azureBLOBStorage) presignedUrlWithToken(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { @@ -157,13 +151,15 @@ func (s *azureBLOBStorage) presignedUrlWithToken(ctx context.Context, storagePat now := time.Now().UTC() exp := now.Add(expiration) - svcClient, err := service.NewClient(s.serviceURL, s.conf.TokenCredential, nil) + svcClient, err := service.NewClient(s.serviceURL, s.conf.TokenCredential, &service.ClientOptions{ClientOptions: s.options}) if err != nil { return "", err } + // The key starts in the past and the SAS takes no start time, as Azure + // advises: a host clock ahead of Azure's would make the URL not valid yet. udc, err := svcClient.GetUserDelegationCredential(ctx, service.KeyInfo{ - Start: to.Ptr(now.Format(sas.TimeFormat)), + Start: to.Ptr(now.Add(-15 * time.Minute).Format(sas.TimeFormat)), Expiry: to.Ptr(exp.Format(sas.TimeFormat)), }, nil) if err != nil { @@ -172,7 +168,6 @@ func (s *azureBLOBStorage) presignedUrlWithToken(ctx context.Context, storagePat qp, err := sas.BlobSignatureValues{ Protocol: sas.ProtocolHTTPS, - StartTime: now, ExpiryTime: exp, Permissions: (&sas.BlobPermissions{Read: true}).String(), ContainerName: s.conf.ContainerName, @@ -197,7 +192,7 @@ func (s *azureBLOBStorage) PutObject(ctx context.Context, storagePath string, da HTTPHeaders: headers, AccessConditions: conditions, }) - return wrapAzureObjectError(err) + return wrapAzureWriteError(err, opts.IfAbsent) } func azureWriteOptions(opts PutOptions) (*blob.HTTPHeaders, *blob.AccessConditions) { @@ -228,11 +223,7 @@ func (s *azureBLOBStorage) UploadObject(ctx context.Context, storagePath string, HTTPHeaders: headers, AccessConditions: conditions, }) - // A racing upload's commit discards this one's staged blocks. - if opts.IfAbsent && bloberror.HasCode(err, bloberror.InvalidBlockList) { - return fmt.Errorf("%w: %w", ErrExists, wrapAzureError(err)) - } - return wrapAzureObjectError(err) + return wrapAzureWriteError(err, opts.IfAbsent) } func (s *azureBLOBStorage) GetObject(ctx context.Context, storagePath string) ([]byte, error) { @@ -312,6 +303,7 @@ func (s *azureBLOBStorage) ListObjects(ctx context.Context, prefix string) ([]st } } } + slices.Sort(keys) return keys, nil } @@ -360,8 +352,18 @@ func wrapAzureObjectError(err error) error { return nil case bloberror.HasCode(err, bloberror.BlobNotFound): return fmt.Errorf("%w: %w", ErrNotFound, wrapAzureError(err)) - case bloberror.HasCode(err, bloberror.BlobAlreadyExists, bloberror.ConditionNotMet): - return fmt.Errorf("%w: %w", ErrExists, wrapAzureError(err)) } return wrapAzureError(err) } + +// wrapAzureWriteError is wrapAzureObjectError with a failed condition as +// ErrExists where IfAbsent set one. A read can fail one too: the retry reader +// pins its retries to the first response's ETag. +func wrapAzureWriteError(err error, ifAbsent bool) error { + // InvalidBlockList is a racing upload whose commit discarded this one's + // staged blocks. + if ifAbsent && bloberror.HasCode(err, bloberror.BlobAlreadyExists, bloberror.ConditionNotMet, bloberror.InvalidBlockList) { + return fmt.Errorf("%w: %w", ErrExists, wrapAzureError(err)) + } + return wrapAzureObjectError(err) +} diff --git a/contract_test.go b/contract_test.go index 95776ef..eac5413 100644 --- a/contract_test.go +++ b/contract_test.go @@ -15,6 +15,7 @@ package storage_test import ( + "bytes" "context" "crypto/ecdsa" "crypto/elliptic" @@ -25,6 +26,7 @@ import ( "encoding/json" "encoding/pem" "fmt" + "io" "math/big" "net/http" "net/http/httptest" @@ -65,6 +67,19 @@ func TestS3CustomCABundle(t *testing.T) { func TestGCPLeavesDefaultTransport(t *testing.T) { orig := http.DefaultTransport before := orig.(*http.Transport).Clone() + // The rest of the process reads it meanwhile: under -race, a write races. + done := make(chan struct{}) + t.Cleanup(func() { close(done) }) + go func() { + for { + select { + case <-done: + return + default: + _ = http.DefaultTransport + } + } + }() _, err := storage.New(&storage.GCPConfig{ Bucket: "bucket", @@ -100,24 +115,40 @@ func TestProxyCarriesRequests(t *testing.T) { }, } { t.Run(name, func(t *testing.T) { - proxied := make(chan string, 16) + proxied := make(chan *http.Request, 16) proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - proxied <- r.Host + proxied <- r http.Error(w, "no", http.StatusForbidden) })) t.Cleanup(proxy.Close) - s, err := storage.New(conf(&storage.ProxyConfig{URL: proxy.URL})) - require.NoError(t, err) + send := func() { + s, err := storage.New(conf(&storage.ProxyConfig{URL: proxy.URL, Username: "user", Password: "pass"})) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, _ = s.ObjectExists(ctx, "path") + } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - _, _ = s.ObjectExists(ctx, "path") + // The credentials reach the proxy for a plain http target too, not + // only on CONNECT. + send() select { - case host := <-proxied: - require.Equal(t, "store.invalid", host) + case r := <-proxied: + require.Equal(t, "store.invalid", r.Host) + require.Equal(t, "Basic dXNlcjpwYXNz", r.Header.Get("Proxy-Authorization")) default: t.Fatal("the request went around the proxy") } + + // NO_PROXY is honored, as it is for the metadata services that hand + // out instance credentials. + t.Setenv("NO_PROXY", "store.invalid") + send() + select { + case <-proxied: + t.Fatal("a NO_PROXY host went through the proxy") + default: + } }) } } @@ -165,7 +196,7 @@ func TestLocalFileMode(t *testing.T) { require.NoError(t, s.PutObject(ctx, "put", []byte("x"), storage.PutOptions{IfAbsent: true})) file := filepath.Join(t.TempDir(), "file") require.NoError(t, os.WriteFile(file, []byte("x"), 0o600)) - _, _, err = s.UploadFile(ctx, file, "file", "text/plain") + _, _, err = s.UploadFile(ctx, "file", file, storage.PutOptions{ContentType: "text/plain"}) require.NoError(t, err) for _, name := range []string{"put", "file"} { info, err := os.Stat(filepath.Join(dir, name)) @@ -231,6 +262,39 @@ func TestLocalKeys(t *testing.T) { } } +// A delete that removes an empty directory while a listing walks it does not +// hide the other keys: an empty listing reads as no objects. +func TestLocalListingDuringDeletes(t *testing.T) { + dir := t.TempDir() + s, err := storage.New(&storage.LocalConfig{StorageDir: dir}) + require.NoError(t, err) + ctx := context.Background() + require.NoError(t, s.PutObject(ctx, "keep", []byte("x"), storage.PutOptions{})) + + done := make(chan struct{}) + churned := make(chan struct{}) + go func() { + defer close(churned) + for { + select { + case <-done: + return + default: + } + // What a delete's cleanup of emptied directories does. + _ = os.MkdirAll(filepath.Join(dir, "d", "e"), 0o755) + _ = os.RemoveAll(filepath.Join(dir, "d")) + } + }() + for range 5000 { + keys, err := s.ListObjects(ctx, "") + require.NoError(t, err) + require.Contains(t, keys, "keep") + } + close(done) + <-churned +} + // umask is the process's, read the way os.Create applies it. func umask(t *testing.T) os.FileMode { f, err := os.Create(filepath.Join(t.TempDir(), "probe")) @@ -256,6 +320,79 @@ func serviceAccount(t *testing.T, tokenURI string) string { return string(b) } +// A multipart upload that fails when its caller's context ends is aborted +// anyway: its parts would stay billed. +func TestS3AbortsAfterDeadline(t *testing.T) { + aborted := make(chan struct{}, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case r.Method == http.MethodPost && q.Has("uploads"): + fmt.Fprint(w, `bucketkup`) + case r.Method == http.MethodPut && q.Has("partNumber"): + _, _ = io.Copy(io.Discard, r.Body) + time.Sleep(time.Second) + case r.Method == http.MethodDelete && q.Get("uploadId") == "up": + aborted <- struct{}{} + w.WriteHeader(http.StatusNoContent) + } + })) + t.Cleanup(srv.Close) + s, err := storage.New(&storage.S3Config{AccessKey: "key", Secret: "secret", Region: "us-east-1", Endpoint: srv.URL, ForcePathStyle: true, Bucket: "bucket"}) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + require.Error(t, s.UploadObject(ctx, "k", bytes.NewReader(make([]byte, 40<<20)), storage.PutOptions{})) + select { + case <-aborted: + case <-time.After(10 * time.Second): + t.Fatal("the upload was not aborted") + } +} + +// The OSS SDK never retries, so this package does, for calls it can send +// again: a 5xx or 429 is retried, a 403 is not. +func TestAliOSSRetries(t *testing.T) { + var requests atomic.Int32 + failNext := atomic.Int32{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if code := failNext.Swap(0); code != 0 { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(int(code)) + fmt.Fprintf(w, `Failedmr`) + return + } + if r.Method == http.MethodGet { + fmt.Fprint(w, "hello") + } + })) + t.Cleanup(srv.Close) + s, err := storage.New(&storage.AliOSSConfig{AccessKey: "key", Secret: "secret", Endpoint: srv.URL, Bucket: "bucket"}) + require.NoError(t, err) + ctx := context.Background() + + for _, code := range []int32{http.StatusServiceUnavailable, http.StatusTooManyRequests} { + requests.Store(0) + failNext.Store(code) + require.NoError(t, s.PutObject(ctx, "k", []byte("hello"), storage.PutOptions{})) + require.EqualValues(t, 2, requests.Load(), "put after %d", code) + + requests.Store(0) + failNext.Store(code) + got, err := s.GetObject(ctx, "k") + require.NoError(t, err) + require.Equal(t, []byte("hello"), got) + require.EqualValues(t, 2, requests.Load(), "get after %d", code) + } + + requests.Store(0) + failNext.Store(http.StatusForbidden) + require.Error(t, s.PutObject(ctx, "k", []byte("hello"), storage.PutOptions{})) + require.EqualValues(t, 1, requests.Load(), "a 403 is not retried") +} + func selfSignedCert(t *testing.T) []byte { key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) require.NoError(t, err) diff --git a/e2e/object_test.go b/e2e/object_test.go index 1395a12..cb0e658 100644 --- a/e2e/object_test.go +++ b/e2e/object_test.go @@ -17,14 +17,18 @@ package e2e import ( "bytes" "context" + "errors" "fmt" "io" "net/http" + "os" "path/filepath" "sort" + "strings" "sync" "sync/atomic" "testing" + "testing/iotest" "time" "github.com/stretchr/testify/require" @@ -44,6 +48,9 @@ type objectsSuite struct { multipartIfAbsentRaces bool // multipartIgnoresIfAbsent: a multipart upload's commit ignores IfAbsent. multipartIgnoresIfAbsent bool + // overwritesInPlace: a write replaces an object as it goes, so a failed + // one leaves none (local, as a filesystem). + overwritesInPlace bool // commit tells the request that makes a multipart upload's object, and // whether it carries IfAbsent's condition, which a store that ignores the // condition cannot show. @@ -105,8 +112,16 @@ func testObjectsWith(t *testing.T, s storage.Storage, suite objectsSuite) { require.ErrorIs(t, err, storage.ErrNotFound) _, err = s.GetObjectRange(ctx, missing, 0, 1) require.ErrorIs(t, err, storage.ErrNotFound) - _, err = s.DownloadFile(ctx, filepath.Join(t.TempDir(), "missing"), missing) + _, err = s.DownloadObject(ctx, missing, io.Discard) require.ErrorIs(t, err, storage.ErrNotFound) + // A failed download leaves the file it would have written as it was. + kept := filepath.Join(t.TempDir(), "kept") + require.NoError(t, os.WriteFile(kept, []byte("kept"), 0o600)) + _, err = s.DownloadFile(ctx, missing, kept) + require.ErrorIs(t, err, storage.ErrNotFound) + got, err := os.ReadFile(kept) + require.NoError(t, err) + require.Equal(t, []byte("kept"), got) exists, err := s.ObjectExists(ctx, missing) require.NoError(t, err) require.False(t, exists) @@ -146,6 +161,35 @@ func testObjectsWith(t *testing.T, s storage.Storage, suite objectsSuite) { } }) + // An upload whose body fails partway stores nothing: a key it would create + // stays empty, and an object it would replace keeps its bytes. + t.Run("failed body", func(t *testing.T) { + for name, size := range map[string]int{"single part": 1 << 10, "multipart": 17 << 20} { + t.Run(name, func(t *testing.T) { + path := prefix + "failed-body/" + strings.ReplaceAll(name, " ", "-") + t.Cleanup(func() { _ = s.DeleteObjects(context.Background(), []string{path}) }) + failing := func() io.Reader { + return io.MultiReader(bytes.NewReader(make([]byte, size)), iotest.ErrReader(errors.New("body failed"))) + } + + require.Error(t, s.UploadObject(ctx, path, failing(), storage.PutOptions{})) + exists, err := s.ObjectExists(ctx, path) + require.NoError(t, err) + require.False(t, exists, "a failed upload created the object") + + require.NoError(t, s.PutObject(ctx, path, []byte("old"), storage.PutOptions{})) + require.Error(t, s.UploadObject(ctx, path, failing(), storage.PutOptions{})) + got, err := s.GetObject(ctx, path) + if suite.overwritesInPlace { + require.ErrorIs(t, err, storage.ErrNotFound) + return + } + require.NoError(t, err) + require.Equal(t, []byte("old"), got, "a failed upload replaced the object") + }) + } + }) + t.Run("edge objects", func(t *testing.T) { empty := prefix + "edge/empty" odd := prefix + "edge/a dir/caf\u00e9 & r\u00e9sum\u00e9+1.txt" @@ -236,17 +280,22 @@ func testIfAbsentRace(t *testing.T, s storage.Storage, path string, body []byte) r := &heldReader{r: bytes.NewReader(append([]byte{byte(i)}, body...)), arrived: &arrived, late: &late} go func() { errs <- s.UploadObject(ctx, path, r, storage.PutOptions{IfAbsent: true}) }() } - var won int + var won []byte for range 2 { err := <-errs if err == nil { - won++ + require.Nil(t, won, "both uploads won") + won = []byte{} continue } require.ErrorIs(t, err, storage.ErrExists) } - require.Equal(t, 1, won) + require.NotNil(t, won, "no upload won") require.False(t, late.Load(), "the uploads never overlapped, so nothing raced") + // The object is one upload's whole body: the loser wrote nothing. + got, err := s.GetObject(ctx, path) + require.NoError(t, err) + require.Contains(t, [][]byte{append([]byte{0}, body...), append([]byte{1}, body...)}, got) } // heldReader holds its first read until every reader sharing arrived has diff --git a/e2e/storage_test.go b/e2e/storage_test.go index 7cdff9c..dffcea1 100644 --- a/e2e/storage_test.go +++ b/e2e/storage_test.go @@ -100,7 +100,7 @@ func TestLocal(t *testing.T) { require.NoError(t, err) testStorage(t, s) - testObjects(t, s) + testObjectsWith(t, s, objectsSuite{overwritesInPlace: true}) } func TestOCI(t *testing.T) { @@ -238,7 +238,7 @@ func testStorage(t *testing.T, s storage.Storage) { require.NoError(t, err) require.NoError(t, tmp.Close()) - loc, size, err := s.UploadFile(context.Background(), tmp.Name(), pathFile, "text/plain") + loc, size, err := s.UploadFile(context.Background(), pathFile, tmp.Name(), storage.PutOptions{ContentType: "text/plain"}) require.NoError(t, err) require.Equal(t, int64(len(filePayload)), size) assertLocation(t, loc, pathFile) @@ -265,7 +265,7 @@ func testStorage(t *testing.T, s storage.Storage) { require.NoError(t, tmp.Close()) defer os.Remove(tmp.Name()) - size, err := s.DownloadFile(context.Background(), tmp.Name(), pathFile) + size, err := s.DownloadFile(context.Background(), pathFile, tmp.Name()) require.NoError(t, err) require.Equal(t, int64(len(filePayload)), size) @@ -293,6 +293,7 @@ func testStorage(t *testing.T, s storage.Storage) { require.NoError(t, err) defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, "text/plain", resp.Header.Get("Content-Type"), "the content type written") body, err := io.ReadAll(resp.Body) require.NoError(t, err) require.Equal(t, dataPayload, body) @@ -334,7 +335,7 @@ func testStorage(t *testing.T, s storage.Storage) { file := filepath.Join(t.TempDir(), "multipart") require.NoError(t, os.WriteFile(file, payload, 0o600)) - loc, size, err := s.UploadFile(context.Background(), file, pathMultipart, "application/octet-stream") + loc, size, err := s.UploadFile(context.Background(), pathMultipart, file, storage.PutOptions{ContentType: "application/octet-stream"}) require.NoError(t, err) require.Equal(t, int64(len(payload)), size) assertLocation(t, loc, pathMultipart) diff --git a/gcp.go b/gcp.go index 3ce511f..d24e890 100644 --- a/gcp.go +++ b/gcp.go @@ -23,17 +23,19 @@ import ( "net/http" "net/url" "os" + "slices" "strings" - "sync" "time" "cloud.google.com/go/storage" "github.com/googleapis/gax-go/v2" + "golang.org/x/oauth2" "golang.org/x/oauth2/google" "golang.org/x/oauth2/jwt" "google.golang.org/api/googleapi" "google.golang.org/api/iterator" "google.golang.org/api/option" + htransport "google.golang.org/api/transport/http" ) const storageScope = "https://www.googleapis.com/auth/devstorage.read_write" @@ -75,6 +77,13 @@ func NewGCP(conf *GCPConfig) (Storage, error) { // An http:// endpoint is an emulator, which takes no credentials; any other // endpoint, such as Private Service Connect, keeps them. emulator := strings.HasPrefix(conf.Endpoint, "http://") + setProxy, err := proxySetter(conf.ProxyConfig) + if err != nil { + return nil, err + } + t := transportWithTimeouts(http.DefaultTransport) + setProxy(t) + var opts []option.ClientOption if conf.CredentialsJSON != "" { jwtConfig, err := google.JWTConfigFromJSON([]byte(conf.CredentialsJSON), storageScope) @@ -83,7 +92,10 @@ func NewGCP(conf *GCPConfig) (Storage, error) { } u.signer = jwtConfig if !emulator { - opts = append(opts, option.WithTokenSource(jwtConfig.TokenSource(context.Background()))) + // Tokens are fetched through the same transport, so with the proxy + // and timeouts, not through http.DefaultClient. + tokens := &http.Client{Transport: t, Timeout: time.Minute} + opts = append(opts, option.WithTokenSource(jwtConfig.TokenSource(context.WithValue(context.Background(), oauth2.HTTPClient, tokens)))) } } if conf.Endpoint != "" { @@ -93,11 +105,7 @@ func NewGCP(conf *GCPConfig) (Storage, error) { opts = append(opts, option.WithoutAuthentication()) } - setProxy, err := proxySetter(conf.ProxyConfig) - if err != nil { - return nil, err - } - client, err := newGCSClient(opts, setProxy) + client, err := newGCSClient(opts, t) if err != nil { return nil, err } @@ -106,7 +114,7 @@ func NewGCP(conf *GCPConfig) (Storage, error) { return u, nil } -func (s *gcpStorage) UploadFile(ctx context.Context, filepath, storagePath, contentType string) (string, int64, error) { +func (s *gcpStorage) UploadFile(ctx context.Context, storagePath, filepath string, opts PutOptions) (string, int64, error) { file, err := os.Open(filepath) if err != nil { return "", 0, err @@ -117,20 +125,16 @@ func (s *gcpStorage) UploadFile(ctx context.Context, filepath, storagePath, cont if err != nil { return "", 0, err } - if err = s.UploadObject(ctx, storagePath, file, PutOptions{ContentType: contentType}); err != nil { + if err = s.UploadObject(ctx, storagePath, file, opts); err != nil { return "", 0, err } return s.location(storagePath), stat.Size(), nil } -func (s *gcpStorage) DownloadFile(ctx context.Context, filepath, storagePath string) (int64, error) { - file, err := os.Create(filepath) - if err != nil { - return 0, err - } - defer file.Close() - - return s.DownloadObject(ctx, storagePath, file) +func (s *gcpStorage) DownloadFile(ctx context.Context, storagePath, filepath string) (int64, error) { + return downloadFile(filepath, func(f *os.File) (int64, error) { + return s.DownloadObject(ctx, storagePath, f) + }) } func (s *gcpStorage) location(storagePath string) string { @@ -147,16 +151,22 @@ func (s *gcpStorage) presignedURL(storagePath string, expiration time.Duration) Expires: time.Now().Add(expiration), } // A V2 URL, which has no limit on its expiry, is always https; an http:// - // emulator takes V4, which signs for at most 7 days. + // emulator takes V4, which signs for at most 7 days. A private endpoint, + // such as Private Service Connect, is not where the URL's reader is, so it + // names the public host, as location does. if strings.HasPrefix(s.conf.Endpoint, "http://") { opts.Scheme = storage.SigningSchemeV4 opts.Insecure = true + } else if s.conf.Endpoint != "" { + opts.Hostname = "storage.googleapis.com" } if s.signer != nil { opts.GoogleAccessID = s.signer.Email opts.PrivateKey = s.signer.PrivateKey } - u, err := s.client.Bucket(s.conf.Bucket).SignedURL(storagePath, opts) + // Without a key of its own, signing calls IAM, which the SDK otherwise + // retries until the call's context ends, and SignedURL takes none. + u, err := s.client.Bucket(s.conf.Bucket).Retryer(storage.WithMaxAttempts(5)).SignedURL(storagePath, opts) if err != nil { return "", wrapGCPError(err) } @@ -179,17 +189,22 @@ func (s *gcpStorage) UploadObject(ctx context.Context, storagePath string, body return s.write(ctx, storagePath, body, opts, googleapi.DefaultUploadChunkSize) } +// gcsRetryAlways retries a call the SDK would not, as it is not idempotent +// on its own: an overwrite, or a delete, where a missing object counts as +// deleted. +var gcsRetryAlways = []storage.RetryOption{ + storage.WithBackoff(gax.Backoff{Initial: 100 * time.Millisecond, Max: 5 * time.Second, Multiplier: 2}), + storage.WithMaxAttempts(5), + storage.WithPolicy(storage.RetryAlways), +} + func (s *gcpStorage) write(ctx context.Context, storagePath string, body io.Reader, opts PutOptions, chunkSize int) error { obj := s.object(storagePath) if opts.IfAbsent { // Conditional, so the SDK retries it by default. obj = obj.If(storage.Conditions{DoesNotExist: true}) } else { - obj = obj.Retryer( - storage.WithBackoff(gax.Backoff{Initial: 100 * time.Millisecond, Max: 5 * time.Second, Multiplier: 2}), - storage.WithMaxAttempts(5), - storage.WithPolicy(storage.RetryAlways), - ) + obj = obj.Retryer(gcsRetryAlways...) } ctx, cancel := context.WithCancel(ctx) @@ -197,6 +212,9 @@ func (s *gcpStorage) write(ctx context.Context, storagePath string, body io.Read w := obj.NewWriter(ctx) w.ContentType = opts.ContentType w.ChunkSize = chunkSize + // The SDK gives up retrying a chunk after 32s, which a 16 MiB chunk on a + // slow link takes to send; past the idle timeout, a stalled one is retried. + w.ChunkRetryDeadline = 2 * idleTimeout if _, err := io.Copy(w, body); err != nil { // Cancelling aborts the upload; Close would store what was sent. cancel() @@ -255,11 +273,16 @@ func (s *gcpStorage) ObjectExists(ctx context.Context, storagePath string) (bool } func (s *gcpStorage) ListObjects(ctx context.Context, prefix string) ([]string, error) { - it := s.client.Bucket(s.conf.Bucket).Objects(ctx, &storage.Query{Prefix: prefix}) + q := &storage.Query{Prefix: prefix} + if err := q.SetAttrSelection([]string{"Name"}); err != nil { + return nil, err + } + it := s.client.Bucket(s.conf.Bucket).Objects(ctx, q) var keys []string for { attrs, err := it.Next() if errors.Is(err, iterator.Done) { + slices.Sort(keys) return keys, nil } if err != nil { @@ -271,7 +294,7 @@ func (s *gcpStorage) ListObjects(ctx context.Context, prefix string) ([]string, func (s *gcpStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { return deleteEach(ctx, storagePaths, func(ctx context.Context, p string) error { - if err := s.object(p).Delete(ctx); err != nil && !errors.Is(err, storage.ErrObjectNotExist) { + if err := s.object(p).Retryer(gcsRetryAlways...).Delete(ctx); err != nil && !errors.Is(err, storage.ErrObjectNotExist) { return wrapGCPObjectError(err) } return nil @@ -293,19 +316,19 @@ func wrapGCPObjectError(err error) error { } // gcsTransportMu serializes newGCSClient's swap of http.DefaultTransport. -var gcsTransportMu sync.Mutex - -// newGCSClient builds a client on a transport with this package's timeouts and -// the proxy. The SDK copies http.DefaultTransport when it builds a client, and -// passing a client of our own would mean wiring its auth by hand, so the -// transport stands in as the default only while the client is built. -func newGCSClient(opts []option.ClientOption, setProxy func(*http.Transport)) (*storage.Client, error) { - gcsTransportMu.Lock() - defer gcsTransportMu.Unlock() - orig := http.DefaultTransport - t := transportWithTimeouts(orig) - setProxy(t) - http.DefaultTransport = t - defer func() { http.DefaultTransport = orig }() - return storage.NewClient(context.Background(), opts...) +// newGCSClient builds a client that sends through t, with the SDK's auth +// added on top. The SDK chooses a client certificate and the mTLS endpoint only +// on a transport of its own, so where GOOGLE_API_USE_CLIENT_CERTIFICATE asks +// for them, the client keeps the SDK's, without this package's timeouts and +// proxy. +func newGCSClient(opts []option.ClientOption, t *http.Transport) (*storage.Client, error) { + ctx := context.Background() + if os.Getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE") == "true" { + return storage.NewClient(ctx, opts...) + } + authed, err := htransport.NewTransport(ctx, t, append(opts, option.WithScopes(storage.ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform"))...) + if err != nil { + return nil, err + } + return storage.NewClient(ctx, append(opts, option.WithHTTPClient(&http.Client{Transport: authed}))...) } diff --git a/go.mod b/go.mod index 17f612a..01e4893 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/livekit/protocol v1.52.1 github.com/oracle/oci-go-sdk/v65 v65.126.1 github.com/stretchr/testify v1.12.1 + golang.org/x/net v0.59.0 golang.org/x/oauth2 v0.37.0 golang.org/x/sync v0.23.0 google.golang.org/api v0.300.0 @@ -88,7 +89,6 @@ require ( go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirect - golang.org/x/net v0.59.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.16.0 // indirect diff --git a/local.go b/local.go index a4a5f4a..822d845 100644 --- a/local.go +++ b/local.go @@ -21,12 +21,10 @@ import ( "fmt" "io" "io/fs" - "math/rand/v2" "net/url" "os" "path/filepath" "slices" - "strconv" "strings" "syscall" "time" @@ -58,40 +56,24 @@ func (u *localUploader) file(storagePath string) (string, error) { return p, nil } -func (u *localUploader) UploadFile(ctx context.Context, localPath, storagePath, _ string) (string, int64, error) { +func (u *localUploader) UploadFile(ctx context.Context, storagePath, localPath string, opts PutOptions) (string, int64, error) { file, err := os.Open(localPath) if err != nil { return "", 0, err } defer file.Close() - - dst, n, err := u.upload(ctx, storagePath, file, PutOptions{}) - return dst, n, err + return u.upload(ctx, storagePath, file, opts) } -func (u *localUploader) DownloadFile(ctx context.Context, localPath, storagePath string) (int64, error) { - src, err := u.file(storagePath) +func (u *localUploader) DownloadFile(_ context.Context, storagePath, localPath string) (int64, error) { + f, err := u.open(storagePath) if err != nil { return 0, err } - f, err := os.Open(src) - if err != nil { - return 0, localObjectError(err) - } defer f.Close() - - if dir := filepath.Dir(localPath); dir != "." { - if err = os.MkdirAll(dir, 0755); err != nil { - return 0, err - } - } - local, err := os.Create(localPath) - if err != nil { - return 0, err - } - defer local.Close() - - return io.Copy(local, f) + return downloadFile(localPath, func(local *os.File) (int64, error) { + return io.Copy(local, f) + }) } const localUploadPrefix = ".upload-" @@ -106,9 +88,12 @@ func (u *localUploader) UploadObject(ctx context.Context, storagePath string, bo return err } -// upload writes a temporary file, syncs it and links or renames it into place, -// so readers never see a partial object, a crash leaves none, and IfAbsent is -// atomic. It returns the object's path and the bytes written. +// upload writes the object in place, as on main: a mounted volume (gcsfuse, +// s3fs) sends a rename to its store as a copy, and a consumer watching for the +// file to be closed sees it. A reader can see the object while it is written, +// and a failed write removes it. IfAbsent writes a temporary file and links it +// into place, which is atomic. It returns the object's path and the bytes +// written, synced. func (u *localUploader) upload(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) (string, int64, error) { dst, err := u.file(storagePath) if err != nil { @@ -117,59 +102,62 @@ func (u *localUploader) upload(ctx context.Context, storagePath string, body io. if err = ctx.Err(); err != nil { return "", 0, err } - tmp, err := createUploadFile(filepath.Dir(dst)) - if err != nil { - return "", 0, err + if opts.IfAbsent { + return uploadIfAbsent(dst, body) } - defer os.Remove(tmp.Name()) - n, err := io.Copy(tmp, body) - if err == nil { - err = tmp.Sync() + f, err := createFile(dst, os.O_TRUNC) + if err != nil { + return "", 0, err } - if closeErr := tmp.Close(); err == nil { - err = closeErr + n, err := writeFile(f, body) + if err != nil { + _ = os.Remove(dst) + return "", 0, err } + return dst, n, nil +} + +func uploadIfAbsent(dst string, body io.Reader) (string, int64, error) { + tmp, err := createTemp(filepath.Dir(dst), localUploadPrefix) if err != nil { return "", 0, err } + defer os.Remove(tmp.Name()) - if !opts.IfAbsent { - err = os.Rename(tmp.Name(), dst) - } else if err = os.Link(tmp.Name(), dst); errors.Is(err, fs.ErrExist) { - err = ErrExists + n, err := writeFile(tmp, body) + if err != nil { + return "", 0, err + } + // A hard link fails where it already exists, and on filesystems without + // hard links, such as SMB and most FUSE mounts. + if err = os.Link(tmp.Name(), dst); errors.Is(err, fs.ErrExist) { + return "", 0, ErrExists } if err != nil { return "", 0, err } + syncDir(filepath.Dir(dst)) return dst, n, nil } -// createUploadFile is a new file in dir, made with its parents, with the mode -// os.Create gives, 0666 less the umask, which moving it into place keeps: -// os.CreateTemp's 0600 would hide objects from other users, such as a host -// reading a container's volume. A delete can remove dir once empty, before -// the file is in it, so it is made again. -func createUploadFile(dir string) (*os.File, error) { - for { - if err := os.MkdirAll(dir, 0755); err != nil { - return nil, err - } - name := filepath.Join(dir, localUploadPrefix+strconv.FormatUint(rand.Uint64(), 36)) - f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o666) - if !errors.Is(err, fs.ErrExist) && !errors.Is(err, fs.ErrNotExist) { - return f, err - } +// syncDir persists the names in dir, so an object linked into it survives a +// crash. It is best effort: some filesystems, such as FUSE mounts, refuse to +// sync a directory, and the object's bytes are synced already. +func syncDir(dir string) { + if d, err := os.Open(dir); err == nil { + _ = d.Sync() + _ = d.Close() } } func (u *localUploader) GetObject(_ context.Context, storagePath string) ([]byte, error) { - src, err := u.file(storagePath) + f, err := u.open(storagePath) if err != nil { return nil, err } - data, err := os.ReadFile(src) - return data, localObjectError(err) + defer f.Close() + return io.ReadAll(f) } func (u *localUploader) GetObjectRange(_ context.Context, storagePath string, offset, length int64) ([]byte, error) { @@ -203,13 +191,25 @@ func (u *localUploader) DownloadObject(_ context.Context, storagePath string, w return io.Copy(w, f) } +// open is the object's file. A directory is no object, as a prefix is not on +// a store. func (u *localUploader) open(storagePath string) (*os.File, error) { src, err := u.file(storagePath) if err != nil { return nil, err } f, err := os.Open(src) - return f, localObjectError(err) + if err != nil { + return nil, localObjectError(err) + } + if info, err := f.Stat(); err != nil || info.IsDir() { + f.Close() + if err == nil { + err = syscall.EISDIR + } + return nil, localObjectError(err) + } + return f, nil } // ObjectExists reports a directory as no object, as a store reports a prefix. @@ -241,19 +241,31 @@ func (u *localUploader) ListObjects(_ context.Context, prefix string) ([]string, } // The trailing separator has the walk follow a root that is a symlink. + walkRoot := root + string(filepath.Separator) var keys []string - err := filepath.WalkDir(root+string(filepath.Separator), func(p string, d fs.DirEntry, err error) error { + err := filepath.WalkDir(walkRoot, func(p string, d fs.DirEntry, err error) error { if err != nil { + // A directory a delete removes mid-walk held no more keys; only a + // missing root means the prefix holds none. + if p != walkRoot && errors.Is(localObjectError(err), ErrNotFound) { + return nil + } return err } - if d.IsDir() || strings.HasPrefix(d.Name(), localUploadPrefix) { - return nil - } rel, err := filepath.Rel(u.StorageDir, p) if err != nil { return err } - if key := filepath.ToSlash(rel); strings.HasPrefix(key, prefix) { + key := filepath.ToSlash(rel) + if d.IsDir() { + // A directory none of whose keys can start with the prefix is not + // walked, so a listing reads only what it may return. + if p != walkRoot && !strings.HasPrefix(key+"/", prefix) && !strings.HasPrefix(prefix, key+"/") { + return fs.SkipDir + } + return nil + } + if !strings.HasPrefix(d.Name(), localUploadPrefix) && strings.HasPrefix(key, prefix) { keys = append(keys, key) } return nil diff --git a/logging.go b/logging.go index d1904aa..e39fe64 100644 --- a/logging.go +++ b/logging.go @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package storage import ( @@ -30,7 +31,7 @@ type s3Logger struct { idx int } -func news3Logger() *s3Logger { +func newS3Logger() *s3Logger { return &s3Logger{ msgs: make([]string, 10), } diff --git a/objects/objects.go b/objects/objects.go index 6513fb8..1680e2e 100644 --- a/objects/objects.go +++ b/objects/objects.go @@ -12,29 +12,35 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package objects is the storage interfaces without any provider SDK, for code -// that implements or accepts them without creating one, such as a fake: the -// root package imports every provider's SDK. config does the same for the -// config structs. +// Package objects is the storage interfaces and errors without any provider +// SDK, for code that implements or accepts them without creating one, such as +// a fake: the root package imports every provider's SDK. package objects import ( "context" "errors" + "fmt" "io" "time" ) // Bucket is object storage for a service that keeps and manages its own -// objects. A read of a missing object reports ErrNotFound and a refused write -// ErrExists; test them with errors.Is. +// objects. A read of a missing object reports ErrNotFound and a write IfAbsent +// refuses ErrExists; test them with errors.Is. Where a store differs, every +// backend follows S3. +// +// A key is a store's key as given. The local directory is the exception, as a +// filesystem: it drops a leading "/", cleans "//" and "." segments, refuses a +// key that leads out of it, cannot hold both "a" and "a/b", and keeps its +// temporary files under names starting with ".upload-", which it never lists. type Bucket interface { - // PutObject sends data in one request, which the provider's SDK can retry. + // PutObject sends data, which is retried as a whole where a call fails. PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error // UploadObject streams body in parts and does not rewind it, so a retry - // needs a new reader. Memory is the part size times the upload concurrency: - // 16 MiB x 5 on S3, 4 MiB x 16 on Azure, one 16 MiB part on GCS and OCI, - // none on AliOSS (one chunked request, up to 5 GB) and local. + // needs a new reader. Memory is about the part size times the parts in + // flight: 16 MiB x 6 on S3, 4 MiB x 16 on Azure, one 16 MiB part on GCS and + // OCI, none on AliOSS (one chunked request, up to 5 GB, sent once) and local. UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error GetObject(ctx context.Context, storagePath string) ([]byte, error) @@ -43,7 +49,8 @@ type Bucket interface { // one starting at or past it no bytes. GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) // DownloadObject streams the object to w, which holds part of it when an - // error comes back. + // error comes back. An object replaced during a read fails it, with no + // sentinel. DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) // ObjectExists reports a missing object as false, not ErrNotFound. ObjectExists(ctx context.Context, storagePath string) (bool, error) @@ -53,7 +60,7 @@ type Bucket interface { ListObjects(ctx context.Context, prefix string) ([]string, error) // DeleteObjects succeeds for paths that hold nothing, as S3's delete does, // and fails when a path is not deleted; some may be deleted by then, and a - // retry is safe. GCS, Azure and OCI send one request per path. + // retry is safe. GCS, Azure and OCI send one request per path, 16 at once. DeleteObjects(ctx context.Context, storagePaths []string) error // PresignedURL is a GET URL; local returns a file:// URL. @@ -64,10 +71,13 @@ type Bucket interface { type Storage interface { Bucket - // UploadFile replaces any object at storagePath and returns its location, - // a URL to it. - UploadFile(ctx context.Context, filepath, storagePath, contentType string) (location string, size int64, err error) - DownloadFile(ctx context.Context, filepath, storagePath string) (size int64, err error) + // UploadFile writes the file at storagePath and returns its location: a + // URL, or the file's path on local. + UploadFile(ctx context.Context, storagePath, filepath string, opts PutOptions) (location string, size int64, err error) + // DownloadFile writes the object to filepath, making its directory, only + // once the whole object is read: a failed download leaves filepath as it + // was. + DownloadFile(ctx context.Context, storagePath, filepath string) (size int64, err error) } type PutOptions struct { @@ -83,3 +93,18 @@ var ( ErrNotFound = errors.New("storage: object not found") ErrExists = errors.New("storage: object exists") ) + +// ErrorWithStatusCode carries the HTTP status a store answered with, under +// ErrNotFound or ErrExists where those apply; find it with errors.As. +type ErrorWithStatusCode struct { + Err error + StatusCode int +} + +func (e *ErrorWithStatusCode) Error() string { + return fmt.Sprintf("err: %s, status code: %d", e.Err, e.StatusCode) +} + +func (e *ErrorWithStatusCode) Unwrap() error { + return e.Err +} diff --git a/oci.go b/oci.go index b8c2b47..ebe4315 100644 --- a/oci.go +++ b/oci.go @@ -25,6 +25,7 @@ import ( "net/url" "os" "path" + "slices" "strings" "time" @@ -60,7 +61,7 @@ func NewOCI(conf *OCIConfig) (Storage, error) { client.Host = conf.Endpoint } // The SDK's 60s http.Client.Timeout covers reading the body and would cut a - // long transfer; a response header timeout catches a dead connection instead. + // long transfer; the idle timeout catches a dead connection instead. // Its transport wrapper stays, as it loads OCI_DEFAULT_CERTS_PATH and // refreshes the TLS config. if hc, ok := client.HTTPClient.(*http.Client); ok { @@ -99,16 +100,39 @@ func NewOCI(conf *OCIConfig) (Storage, error) { if conf.CompartmentID != "" { req.CompartmentId = &conf.CompartmentID } - resp, err := client.GetNamespace(context.Background(), req) - if err != nil { - return nil, wrapOCIError(err) + if s.namespace, err = lookupNamespace(client, req); err != nil { + return nil, err } - s.namespace = *resp.Value } return s, nil } +// lookupNamespace is the tenancy's namespace, or an error after a minute, as +// with the SDK's own client timeout. A canceled OCI call races in the SDK, so +// the lookup is not canceled but left to finish. +func lookupNamespace(client objectstorage.ObjectStorageClient, req objectstorage.GetNamespaceRequest) (string, error) { + type result struct { + namespace string + err error + } + done := make(chan result, 1) + go func() { + resp, err := client.GetNamespace(context.Background(), req) + if err != nil { + done <- result{err: wrapOCIError(err)} + return + } + done <- result{namespace: *resp.Value} + }() + select { + case r := <-done: + return r.namespace, r.err + case <-time.After(time.Minute): + return "", errors.New("storage: OCI namespace lookup took over a minute") + } +} + func newConfigProvider(conf *OCIConfig) (common.ConfigurationProvider, error) { if conf.UseWorkloadIdentity { return auth.OkeWorkloadIdentityConfigurationProvider() @@ -136,7 +160,10 @@ func newConfigProvider(conf *OCIConfig) (common.ConfigurationProvider, error) { // can retry by seeking it: a multipart upload would need more permissions and // can leave uncommitted parts behind. -func (s *ociStorage) UploadFile(ctx context.Context, filepath, storagePath, contentType string) (string, int64, error) { +// UploadFile sends the file in one PutObject, up to OCI's 50 GiB, which the SDK +// can retry by seeking it, as on main: a multipart upload, which OCI advises +// past 100 MiB, needs more permissions and can leave uncommitted parts behind. +func (s *ociStorage) UploadFile(ctx context.Context, storagePath, filepath string, opts PutOptions) (string, int64, error) { file, err := os.Open(filepath) if err != nil { return "", 0, err @@ -147,7 +174,7 @@ func (s *ociStorage) UploadFile(ctx context.Context, filepath, storagePath, cont if err != nil { return "", 0, err } - if err = s.put(ctx, storagePath, file, stat.Size(), PutOptions{ContentType: contentType}); err != nil { + if err = s.put(ctx, storagePath, file, stat.Size(), opts); err != nil { return "", 0, err } return s.location(storagePath), stat.Size(), nil @@ -165,14 +192,10 @@ func (s *ociStorage) url(p string) string { return (&url.URL{Scheme: scheme, Host: s.host, Path: p}).String() } -func (s *ociStorage) DownloadFile(ctx context.Context, filepath, storagePath string) (int64, error) { - file, err := os.Create(filepath) - if err != nil { - return 0, err - } - defer file.Close() - - return s.DownloadObject(ctx, storagePath, file) +func (s *ociStorage) DownloadFile(ctx context.Context, storagePath, filepath string) (int64, error) { + return downloadFile(filepath, func(f *os.File) (int64, error) { + return s.DownloadObject(ctx, storagePath, f) + }) } // PresignedURL creates a pre-authenticated request. Unlike an S3 @@ -288,13 +311,17 @@ func (s *ociStorage) UploadObject(ctx context.Context, storagePath string, body return wrapOCIObjectError(err) } + // The abort runs on: it cannot be bounded by canceling it, which races in + // the SDK, and the caller's deadline may be what failed the upload. if err = s.uploadParts(ctx, storagePath, created.UploadId, body, part, len(part), ifNoneMatch); err != nil { - _, _ = s.client.AbortMultipartUpload(context.WithoutCancel(ctx), objectstorage.AbortMultipartUploadRequest{ - NamespaceName: &s.namespace, - BucketName: &s.conf.Bucket, - ObjectName: &storagePath, - UploadId: created.UploadId, - }) + go func() { + _, _ = s.client.AbortMultipartUpload(context.WithoutCancel(ctx), objectstorage.AbortMultipartUploadRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &storagePath, + UploadId: created.UploadId, + }) + }() } return err } @@ -419,6 +446,7 @@ func (s *ociStorage) ListObjects(ctx context.Context, prefix string) ([]string, keys = append(keys, *obj.Name) } if resp.NextStartWith == nil { + slices.Sort(keys) return keys, nil } start = resp.NextStartWith diff --git a/s3.go b/s3.go index d0a9d1b..5922f2a 100644 --- a/s3.go +++ b/s3.go @@ -21,14 +21,17 @@ import ( "errors" "fmt" "io" + "math/rand/v2" "net/http" "net/url" "os" "path" + "slices" "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/ratelimit" "github.com/aws/aws-sdk-go-v2/aws/retry" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/config" @@ -117,8 +120,14 @@ func getConf(conf *S3Config, cp aws.CredentialsProvider) (*aws.Config, error) { o.Credentials = cp o.Retryer = func() aws.Retryer { return retry.NewStandard(func(o *retry.StandardOptions) { - o.MaxAttempts = conf.MaxRetries - o.MaxBackoff = conf.MaxRetryDelay + if conf.MaxRetries > 0 { + o.MaxAttempts = conf.MaxRetries + 1 + } + o.Backoff = retryBackoff{min: conf.MinRetryDelay, max: cmp.Or(conf.MaxRetryDelay, retry.DefaultMaxBackoff)} + // A Storage sends every call with one client, and the SDK's quota + // on retries is per client: one outage would use it up and fail + // every call at once. + o.RateLimiter = ratelimit.None o.Retryables = append(o.Retryables, retry.RetryableHTTPStatusCode{ Codes: map[int]struct{}{ http.StatusConflict: {}, @@ -134,17 +143,12 @@ func getConf(conf *S3Config, cp aws.CredentialsProvider) (*aws.Config, error) { if err != nil { return err } - // "Legacy" is the SDK's default when AWS_DEFAULTS_MODE is unset, so - // this pins what main ran with. Every other mode only swaps in its own - // connect and TLS timeouts (1.1s to 30s), and does it by replacing the - // dialer, which drops the idle timeout setTimeouts puts on it. This - // package sets those timeouts itself, and retries with its own retryer, - // which no mode changes. - o.DefaultsModeOptions.Mode = aws.DefaultsModeLegacy - // The SDK's read timeout is off: setTimeouts bounds reads and writes. + // The idle timeout goes on in newS3Client: a defaults mode + // (AWS_DEFAULTS_MODE) replaces the dialer when a client is made. The + // SDK's read timeout is off, as the idle timeout bounds reads and writes. o.HTTPClient = awshttp.NewBuildableClient(). WithDialerOptions(setDialTimeouts). - WithTransportOptions(setTimeouts, setProxy). + WithTransportOptions(func(t *http.Transport) { t.TLSHandshakeTimeout = tlsHandshakeTimeout }, setProxy). WithReadTimeout(0) return nil @@ -167,7 +171,9 @@ func updateRegion(awsConf *aws.Config, bucket string) error { Bucket: &bucket, } - resp, err := s3.NewFromConfig(*awsConf).GetBucketLocation(context.Background(), req) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + resp, err := s3.NewFromConfig(*awsConf).GetBucketLocation(ctx, req) if err != nil { return wrapS3Error(err) } @@ -179,21 +185,58 @@ func updateRegion(awsConf *aws.Config, bucket string) error { return nil } +// retryBackoff waits a random time between min and a ceiling that doubles +// from a second each attempt, up to max: the SDK's full jitter, with a floor. +type retryBackoff struct{ min, max time.Duration } + +func (b retryBackoff) BackoffDelay(attempt int, _ error) (time.Duration, error) { + ceiling := b.max + if attempt < 30 { + ceiling = min(b.max, time.Second< Date: Sat, 17 Oct 2026 11:04:39 -0700 Subject: [PATCH 14/29] Storage: ErrorWithStatusCode is objects' type, aliased, so errors.As finds it with either --- contract_test.go | 7 ++++++- error.go | 32 -------------------------------- storage.go | 7 ++++--- 3 files changed, 10 insertions(+), 36 deletions(-) delete mode 100644 error.go diff --git a/contract_test.go b/contract_test.go index eac5413..b8f8d9b 100644 --- a/contract_test.go +++ b/contract_test.go @@ -40,6 +40,7 @@ import ( "github.com/stretchr/testify/require" "github.com/livekit/storage" + "github.com/livekit/storage/objects" ) // These hold what a caller's environment relies on, which the emulators, @@ -389,8 +390,12 @@ func TestAliOSSRetries(t *testing.T) { requests.Store(0) failNext.Store(http.StatusForbidden) - require.Error(t, s.PutObject(ctx, "k", []byte("hello"), storage.PutOptions{})) + err = s.PutObject(ctx, "k", []byte("hello"), storage.PutOptions{}) require.EqualValues(t, 1, requests.Load(), "a 403 is not retried") + // The status is found with the SDK-free package's type. + var status *objects.ErrorWithStatusCode + require.ErrorAs(t, err, &status) + require.Equal(t, http.StatusForbidden, status.StatusCode) } func selfSignedCert(t *testing.T) []byte { diff --git a/error.go b/error.go deleted file mode 100644 index 99deaed..0000000 --- a/error.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import "fmt" - -var _ error = (*ErrorWithStatusCode)(nil) - -type ErrorWithStatusCode struct { - Err error - StatusCode int -} - -func (e *ErrorWithStatusCode) Error() string { - return fmt.Sprintf("err: %s, status code: %d", e.Err, e.StatusCode) -} - -func (e *ErrorWithStatusCode) Unwrap() error { - return e.Err -} diff --git a/storage.go b/storage.go index 6b9594c..e6ecd0e 100644 --- a/storage.go +++ b/storage.go @@ -43,9 +43,10 @@ import ( // The interfaces live in storage/objects so that code implementing or // accepting them without creating one doesn't pull in every provider SDK. type ( - Storage = objects.Storage - Bucket = objects.Bucket - PutOptions = objects.PutOptions + Storage = objects.Storage + Bucket = objects.Bucket + PutOptions = objects.PutOptions + ErrorWithStatusCode = objects.ErrorWithStatusCode ) var ( From 24da205e3645961ef66e80132f51eba702bff05d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 19:24:44 -0700 Subject: [PATCH 15/29] AliOSS: DownloadFile through downloadFile, CRC64 checked on every whole read GetObjectToFile wrote .temp without making its directory and left it behind when a download failed partway. The CRC64 check it made now runs in download, so GetObject and DownloadObject get it too. Stale comments in gcp.go and oci.go removed. --- alioss.go | 34 +++++++++++++++---------------- contract_test.go | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ gcp.go | 1 - oci.go | 4 ---- 4 files changed, 70 insertions(+), 22 deletions(-) diff --git a/alioss.go b/alioss.go index f4776d2..de7eca4 100644 --- a/alioss.go +++ b/alioss.go @@ -96,19 +96,10 @@ func (s *aliOSSStorage) location(storagePath string) string { return u.String() } -// DownloadFile has the SDK write a temporary file and rename it, as -// downloadFile does for the other stores. func (s *aliOSSStorage) DownloadFile(ctx context.Context, storagePath, filepath string) (int64, error) { - if err := aliOSSRetry(ctx, func() error { - return s.bucket.GetObjectToFile(storagePath, filepath, oss.WithContext(ctx)) - }); err != nil { - return 0, wrapAliOSSObjectError(err) - } - info, err := os.Stat(filepath) - if err != nil { - return 0, err - } - return info.Size(), nil + return downloadFile(filepath, func(f *os.File) (int64, error) { + return s.download(ctx, storagePath, f) + }) } func (s *aliOSSStorage) presignedURL(storagePath string, expiration time.Duration) (string, error) { @@ -174,18 +165,27 @@ func (s *aliOSSStorage) DownloadObject(ctx context.Context, storagePath string, return s.download(ctx, storagePath, w) } +// download copies the object to w and, for a whole object, checks the CRC64 +// OSS sends against what arrived, as the SDK's GetObjectToFile does. func (s *aliOSSStorage) download(ctx context.Context, storagePath string, w io.Writer, options ...oss.Option) (int64, error) { - var r io.ReadCloser + var result *oss.GetObjectResult err := aliOSSRetry(ctx, func() (err error) { - r, err = s.bucket.GetObject(storagePath, append(options, oss.WithContext(ctx))...) + result, err = s.bucket.DoGetObject(&oss.GetObjectRequest{ObjectKey: storagePath}, append(options, oss.WithContext(ctx))) return err }) if err != nil { return 0, wrapAliOSSObjectError(err) } - defer r.Close() - n, err := io.Copy(w, r) - return n, wrapAliOSSObjectError(err) + defer result.Response.Close() + n, err := io.Copy(w, result.Response.Body) + if err != nil { + return n, wrapAliOSSObjectError(err) + } + if result.ClientCRC != nil { + result.Response.ClientCRC = result.ClientCRC.Sum64() + err = oss.CheckCRC(result.Response, "GetObject") + } + return n, err } func (s *aliOSSStorage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { diff --git a/contract_test.go b/contract_test.go index b8f8d9b..271df2e 100644 --- a/contract_test.go +++ b/contract_test.go @@ -26,6 +26,7 @@ import ( "encoding/json" "encoding/pem" "fmt" + "hash/crc64" "io" "math/big" "net/http" @@ -33,6 +34,7 @@ import ( "os" "path/filepath" "reflect" + "strconv" "sync/atomic" "testing" "time" @@ -398,6 +400,57 @@ func TestAliOSSRetries(t *testing.T) { require.Equal(t, http.StatusForbidden, status.StatusCode) } +// AliOSS's DownloadFile keeps the contract: it makes the destination's +// directory, a body cut short or a CRC64 that does not match leaves the +// destination as it was and no temporary file behind. +func TestAliOSSDownloadFile(t *testing.T) { + var mode atomic.Value + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := []byte("hello") + crc := crc64.Checksum(body, crc64.MakeTable(crc64.ECMA)) + switch mode.Load() { + case "short": + w.Header().Set("Content-Length", "10") + _, _ = w.Write(body) + // Ends the response five bytes short. + conn, _, _ := w.(http.Hijacker).Hijack() + _ = conn.Close() + return + case "bad crc": + crc++ + } + w.Header().Set("X-Oss-Hash-Crc64ecma", strconv.FormatUint(crc, 10)) + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + s, err := storage.New(&storage.AliOSSConfig{AccessKey: "key", Secret: "secret", Endpoint: srv.URL, Bucket: "bucket"}) + require.NoError(t, err) + ctx := context.Background() + + dir := t.TempDir() + dst := filepath.Join(dir, "new", "dir", "file") + mode.Store("ok") + n, err := s.DownloadFile(ctx, "k", dst) + require.NoError(t, err) + require.EqualValues(t, 5, n) + got, err := os.ReadFile(dst) + require.NoError(t, err) + require.Equal(t, []byte("hello"), got) + + for _, m := range []string{"short", "bad crc"} { + mode.Store(m) + require.NoError(t, os.WriteFile(dst, []byte("kept"), 0o600)) + _, err = s.DownloadFile(ctx, "k", dst) + require.Error(t, err, m) + got, err := os.ReadFile(dst) + require.NoError(t, err) + require.Equal(t, []byte("kept"), got, m) + entries, err := os.ReadDir(filepath.Dir(dst)) + require.NoError(t, err) + require.Len(t, entries, 1, "%s left a temporary file", m) + } +} + func selfSignedCert(t *testing.T) []byte { key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) require.NoError(t, err) diff --git a/gcp.go b/gcp.go index d24e890..5948e8e 100644 --- a/gcp.go +++ b/gcp.go @@ -315,7 +315,6 @@ func wrapGCPObjectError(err error) error { return wrapGCPError(err) } -// gcsTransportMu serializes newGCSClient's swap of http.DefaultTransport. // newGCSClient builds a client that sends through t, with the SDK's auth // added on top. The SDK chooses a client certificate and the mTLS endpoint only // on a transport of its own, so where GOOGLE_API_USE_CLIENT_CERTIFICATE asks diff --git a/oci.go b/oci.go index ebe4315..7d78e99 100644 --- a/oci.go +++ b/oci.go @@ -156,10 +156,6 @@ func newConfigProvider(conf *OCIConfig) (common.ConfigurationProvider, error) { return common.DefaultConfigProvider(), nil } -// UploadFile sends the file in one PutObject, up to OCI's 50 GiB, which the SDK -// can retry by seeking it: a multipart upload would need more permissions and -// can leave uncommitted parts behind. - // UploadFile sends the file in one PutObject, up to OCI's 50 GiB, which the SDK // can retry by seeking it, as on main: a multipart upload, which OCI advises // past 100 MiB, needs more permissions and can leave uncommitted parts behind. From b831dffa01647fa0ea61feaa9bf5e652cddc6377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 19:30:21 -0700 Subject: [PATCH 16/29] Storage: no single-use helpers New takes newStorage's body; PresignedURL is the method itself on GCS, AliOSS and OCI rather than a forward; S3's abort and uploader, GCS's client builder, AliOSS's retry test, setTimeouts and syncDir are inlined where they were called. The CRC32-clearing middleware stays a named option. Azure's presign helper is presignWithUserDelegation. --- alioss.go | 26 ++++++++---------- azure.go | 10 +++---- config.go | 3 ++- gcp.go | 40 ++++++++++----------------- local.go | 14 ++++------ location_test.go | 2 +- oci.go | 6 +---- s3.go | 70 ++++++++++++++++++++++-------------------------- storage.go | 13 ++------- 9 files changed, 71 insertions(+), 113 deletions(-) diff --git a/alioss.go b/alioss.go index de7eca4..c4187e1 100644 --- a/alioss.go +++ b/alioss.go @@ -102,7 +102,7 @@ func (s *aliOSSStorage) DownloadFile(ctx context.Context, storagePath, filepath }) } -func (s *aliOSSStorage) presignedURL(storagePath string, expiration time.Duration) (string, error) { +func (s *aliOSSStorage) PresignedURL(_ context.Context, storagePath string, expiration time.Duration) (string, error) { u, err := s.bucket.SignURL(storagePath, oss.HTTPGet, int64(expiration.Seconds())) if err != nil { return "", wrapAliOSSError(err) @@ -254,10 +254,6 @@ func (s *aliOSSStorage) DeleteObjects(ctx context.Context, storagePaths []string return nil } -func (s *aliOSSStorage) PresignedURL(_ context.Context, storagePath string, expiration time.Duration) (string, error) { - return s.presignedURL(storagePath, expiration) -} - // aliOSSAttempts is how many times a call is sent: the v1 SDK never retries. const aliOSSAttempts = 3 @@ -267,7 +263,16 @@ const aliOSSAttempts = 3 func aliOSSRetry(ctx context.Context, call func() error) error { var err error for attempt := range aliOSSAttempts { - if err = call(); err == nil || ctx.Err() != nil || !aliOSSRetryable(err) { + if err = call(); err == nil || ctx.Err() != nil { + return err + } + var svcErr oss.ServiceError + var netErr net.Error + retryable := errors.As(err, &netErr) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.ECONNRESET) + if errors.As(err, &svcErr) { + retryable = svcErr.StatusCode >= 500 || svcErr.StatusCode == http.StatusTooManyRequests + } + if !retryable { return err } backoff := 200 * time.Millisecond << attempt @@ -280,15 +285,6 @@ func aliOSSRetry(ctx context.Context, call func() error) error { return err } -func aliOSSRetryable(err error) bool { - var svcErr oss.ServiceError - if errors.As(err, &svcErr) { - return svcErr.StatusCode >= 500 || svcErr.StatusCode == http.StatusTooManyRequests - } - var netErr net.Error - return errors.As(err, &netErr) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.ECONNRESET) -} - func wrapAliOSSObjectError(err error) error { if err == nil { return nil diff --git a/azure.go b/azure.go index e1a0607..05a018b 100644 --- a/azure.go +++ b/azure.go @@ -90,10 +90,6 @@ func NewAzure(conf *AzureConfig) (Storage, error) { return s, err } -func (s *azureBLOBStorage) location(storagePath string) string { - return s.blobURL(storagePath).String() -} - func azureServiceURL(conf *AzureConfig) string { if conf.Endpoint != "" { return conf.Endpoint @@ -130,7 +126,7 @@ func (s *azureBLOBStorage) UploadFile(ctx context.Context, storagePath, filepath if err != nil { return "", 0, wrapAzureWriteError(err, opts.IfAbsent) } - return s.location(storagePath), stat.Size(), nil + return s.blobURL(storagePath).String(), stat.Size(), nil } func (s *azureBLOBStorage) DownloadFile(ctx context.Context, storagePath, filepath string) (int64, error) { @@ -143,7 +139,7 @@ func (s *azureBLOBStorage) DownloadFile(ctx context.Context, storagePath, filepa }) } -func (s *azureBLOBStorage) presignedUrlWithToken(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { +func (s *azureBLOBStorage) presignWithUserDelegation(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { if s.conf.TokenCredential == nil { return "", errors.New("OAuth required") } @@ -323,7 +319,7 @@ func (s *azureBLOBStorage) DeleteObjects(ctx context.Context, storagePaths []str // set, in which case it signs with a user delegation key as Storage does. func (s *azureBLOBStorage) PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { if s.conf.TokenCredential != nil { - return s.presignedUrlWithToken(ctx, storagePath, expiration) + return s.presignWithUserDelegation(ctx, storagePath, expiration) } // HTTPS only, as the user delegation SAS, unless the endpoint itself is // plain HTTP, as an emulator's. diff --git a/config.go b/config.go index cd10d85..725d54b 100644 --- a/config.go +++ b/config.go @@ -34,7 +34,8 @@ type ( ProxyConfig = config.ProxyConfig ) -func newStorage(conf Config) (Storage, error) { +// New builds the Storage a config describes. +func New(conf Config) (Storage, error) { switch c := conf.(type) { case *AliOSSConfig: return NewAliOSS(c) diff --git a/gcp.go b/gcp.go index 5948e8e..ddf1301 100644 --- a/gcp.go +++ b/gcp.go @@ -105,12 +105,21 @@ func NewGCP(conf *GCPConfig) (Storage, error) { opts = append(opts, option.WithoutAuthentication()) } - client, err := newGCSClient(opts, t) - if err != nil { + // The client sends through t, with the SDK's auth added on top. The SDK + // chooses a client certificate and the mTLS endpoint only on a transport of + // its own, so where GOOGLE_API_USE_CLIENT_CERTIFICATE asks for them, the + // client keeps the SDK's, without this package's timeouts and proxy. + ctx := context.Background() + if os.Getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE") != "true" { + authed, err := htransport.NewTransport(ctx, t, append(opts, option.WithScopes(storage.ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform"))...) + if err != nil { + return nil, err + } + opts = append(opts, option.WithHTTPClient(&http.Client{Transport: authed})) + } + if u.client, err = storage.NewClient(ctx, opts...); err != nil { return nil, err } - - u.client = client return u, nil } @@ -145,7 +154,7 @@ func (s *gcpStorage) location(storagePath string) string { }).String() } -func (s *gcpStorage) presignedURL(storagePath string, expiration time.Duration) (string, error) { +func (s *gcpStorage) PresignedURL(_ context.Context, storagePath string, expiration time.Duration) (string, error) { opts := &storage.SignedURLOptions{ Method: "GET", Expires: time.Now().Add(expiration), @@ -301,10 +310,6 @@ func (s *gcpStorage) DeleteObjects(ctx context.Context, storagePaths []string) e }) } -func (s *gcpStorage) PresignedURL(_ context.Context, storagePath string, expiration time.Duration) (string, error) { - return s.presignedURL(storagePath, expiration) -} - func wrapGCPObjectError(err error) error { if err == nil { return nil @@ -314,20 +319,3 @@ func wrapGCPObjectError(err error) error { } return wrapGCPError(err) } - -// newGCSClient builds a client that sends through t, with the SDK's auth -// added on top. The SDK chooses a client certificate and the mTLS endpoint only -// on a transport of its own, so where GOOGLE_API_USE_CLIENT_CERTIFICATE asks -// for them, the client keeps the SDK's, without this package's timeouts and -// proxy. -func newGCSClient(opts []option.ClientOption, t *http.Transport) (*storage.Client, error) { - ctx := context.Background() - if os.Getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE") == "true" { - return storage.NewClient(ctx, opts...) - } - authed, err := htransport.NewTransport(ctx, t, append(opts, option.WithScopes(storage.ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform"))...) - if err != nil { - return nil, err - } - return storage.NewClient(ctx, append(opts, option.WithHTTPClient(&http.Client{Transport: authed}))...) -} diff --git a/local.go b/local.go index 822d845..d472d71 100644 --- a/local.go +++ b/local.go @@ -137,18 +137,14 @@ func uploadIfAbsent(dst string, body io.Reader) (string, int64, error) { if err != nil { return "", 0, err } - syncDir(filepath.Dir(dst)) - return dst, n, nil -} - -// syncDir persists the names in dir, so an object linked into it survives a -// crash. It is best effort: some filesystems, such as FUSE mounts, refuse to -// sync a directory, and the object's bytes are synced already. -func syncDir(dir string) { - if d, err := os.Open(dir); err == nil { + // The directory holds the new name: syncing it keeps the object across a + // crash. Best effort, as some filesystems, FUSE mounts among them, refuse + // to sync a directory. + if d, err := os.Open(filepath.Dir(dst)); err == nil { _ = d.Sync() _ = d.Close() } + return dst, n, nil } func (u *localUploader) GetObject(_ context.Context, storagePath string) ([]byte, error) { diff --git a/location_test.go b/location_test.go index 2fa8f46..acefa70 100644 --- a/location_test.go +++ b/location_test.go @@ -137,7 +137,7 @@ func TestAzureLocation(t *testing.T) { } for _, tc := range cases { s := &azureBLOBStorage{conf: &tc.conf} - got := s.location(tc.storagePath) + got := s.blobURL(tc.storagePath).String() require.Equal(t, tc.want, got, tc.name) requireWellFormedHTTPLocation(t, got, tc.name) } diff --git a/oci.go b/oci.go index 7d78e99..4ee95a4 100644 --- a/oci.go +++ b/oci.go @@ -198,7 +198,7 @@ func (s *ociStorage) DownloadFile(ctx context.Context, storagePath, filepath str // presigned URL, a PAR is a durable object: it is listable and revocable, it // costs an API call to mint, and it needs PAR_MANAGE on the bucket. Its access // URI is returned only here and cannot be read back afterwards. -func (s *ociStorage) presignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { +func (s *ociStorage) PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { name := fmt.Sprintf("livekit-storage-%s-%d", storagePath, time.Now().UnixNano()) expires := common.SDKTime{Time: time.Now().Add(expiration)} @@ -463,10 +463,6 @@ func (s *ociStorage) DeleteObjects(ctx context.Context, storagePaths []string) e }) } -func (s *ociStorage) PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { - return s.presignedURL(ctx, storagePath, expiration) -} - func wrapOCIObjectError(err error) error { if err == nil { return nil diff --git a/s3.go b/s3.go index 5922f2a..c502de6 100644 --- a/s3.go +++ b/s3.go @@ -285,54 +285,48 @@ func (s *s3Storage) send(ctx context.Context, input *s3.PutObjectInput, opts ... opts = append(opts, func(u *manager.Uploader) { u.PartSize = s3UploadPartSize u.LeavePartsOnError = true + if !awsEndpoint(s.conf.Endpoint) { + u.ClientOptions = append(u.ClientOptions, clearMultipartChecksum) + } }) - _, err := s.uploader(opts...).Upload(ctx, input) + _, err := manager.NewUploader(s.client, opts...).Upload(ctx, input) + var failed manager.MultiUploadFailure if errors.As(err, &failed) { - go s.abort(context.WithoutCancel(ctx), input.Key, failed.UploadID()) + go func() { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute) + defer cancel() + _, _ = s.client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: aws.String(s.conf.Bucket), + Key: input.Key, + UploadId: aws.String(failed.UploadID()), + }) + }() } return err } -func (s *s3Storage) abort(ctx context.Context, key *string, uploadID string) { - ctx, cancel := context.WithTimeout(ctx, time.Minute) - defer cancel() - _, _ = s.client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ - Bucket: aws.String(s.conf.Bucket), - Key: key, - UploadId: aws.String(uploadID), +// clearMultipartChecksum drops the CRC32 the uploader forces on multipart +// uploads, which UploadPart sends as an aws-chunked trailer that stores other +// than AWS may reject, before any checksum middleware sees it. +func clearMultipartChecksum(o *s3.Options) { + o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { + return stack.Initialize.Add(middleware.InitializeMiddlewareFunc("ClearMultipartChecksum", + func(ctx context.Context, in middleware.InitializeInput, next middleware.InitializeHandler) ( + middleware.InitializeOutput, middleware.Metadata, error, + ) { + switch v := in.Parameters.(type) { + case *s3.CreateMultipartUploadInput: + v.ChecksumAlgorithm = "" + case *s3.UploadPartInput: + v.ChecksumAlgorithm = "" + } + return next.HandleInitialize(ctx, in) + }, + ), middleware.Before) }) } -func (s *s3Storage) uploader(opts ...func(*manager.Uploader)) *manager.Uploader { - // Uploader forces a CRC32 ChecksumAlgorithm on multipart uploads, - // which makes UploadPart use a trailing checksum encoded as Content-Encoding: - // aws-chunked. Non-AWS S3-compatible providers (e.g. OCI) don't support - // this, so clear the algorithm before any checksum middleware sees it. - return manager.NewUploader(s.client, append(opts, func(u *manager.Uploader) { - if !awsEndpoint(s.conf.Endpoint) { - u.ClientOptions = append(u.ClientOptions, func(o *s3.Options) { - o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { - return stack.Initialize.Add(middleware.InitializeMiddlewareFunc( - "ClearMultipartChecksum", - func(ctx context.Context, in middleware.InitializeInput, next middleware.InitializeHandler) ( - middleware.InitializeOutput, middleware.Metadata, error, - ) { - switch v := in.Parameters.(type) { - case *s3.CreateMultipartUploadInput: - v.ChecksumAlgorithm = "" - case *s3.UploadPartInput: - v.ChecksumAlgorithm = "" - } - return next.HandleInitialize(ctx, in) - }, - ), middleware.Before) - }) - }) - } - })...) -} - func (s *s3Storage) location(storagePath string) string { endpoint := "s3.amazonaws.com" if s.conf.Endpoint != "" { diff --git a/storage.go b/storage.go index e6ecd0e..9df349f 100644 --- a/storage.go +++ b/storage.go @@ -54,10 +54,6 @@ var ( ErrExists = objects.ErrExists ) -func New(conf Config) (Storage, error) { - return newStorage(conf) -} - // presentBeforeUpload reports whether IfAbsent can refuse an upload before it is // sent, because the object is there. A check that fails, such as a 403 for // credentials that may write but not read, leaves the refusal to the condition @@ -178,12 +174,6 @@ func setDialTimeouts(d *net.Dialer) { d.KeepAlive = 30 * time.Second } -// setTimeouts sets t's TLS timeout and has its connections time out when idle. -func setTimeouts(t *http.Transport) { - t.TLSHandshakeTimeout = tlsHandshakeTimeout - setIdleTimeout(t) -} - // setIdleTimeout has t's connections fail once idle for idleTimeout. A dialer t // already has, such as one through a VPN, is kept. func setIdleTimeout(t *http.Transport) { @@ -290,7 +280,8 @@ func transportWithTimeouts(rt http.RoundTripper) *http.Transport { } } t.MaxIdleConnsPerHost = max(t.MaxIdleConnsPerHost, maxIdleConnsPerHost) - setTimeouts(t) + t.TLSHandshakeTimeout = tlsHandshakeTimeout + setIdleTimeout(t) return t } From 13c7894c3249d3b28320040ebfd582dcf6c186af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 19:37:47 -0700 Subject: [PATCH 17/29] AliOSS: aliOSSRetryable named again, as what retries is not obvious inline --- alioss.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/alioss.go b/alioss.go index c4187e1..204f3ba 100644 --- a/alioss.go +++ b/alioss.go @@ -263,16 +263,7 @@ const aliOSSAttempts = 3 func aliOSSRetry(ctx context.Context, call func() error) error { var err error for attempt := range aliOSSAttempts { - if err = call(); err == nil || ctx.Err() != nil { - return err - } - var svcErr oss.ServiceError - var netErr net.Error - retryable := errors.As(err, &netErr) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.ECONNRESET) - if errors.As(err, &svcErr) { - retryable = svcErr.StatusCode >= 500 || svcErr.StatusCode == http.StatusTooManyRequests - } - if !retryable { + if err = call(); err == nil || ctx.Err() != nil || !aliOSSRetryable(err) { return err } backoff := 200 * time.Millisecond << attempt @@ -285,6 +276,18 @@ func aliOSSRetry(ctx context.Context, call func() error) error { return err } +// aliOSSRetryable is a failure another attempt can get past: OSS throttling or +// a 5xx, or a connection that broke. A 4xx, as for a denied or bad request, +// would fail again. +func aliOSSRetryable(err error) bool { + var svcErr oss.ServiceError + if errors.As(err, &svcErr) { + return svcErr.StatusCode >= 500 || svcErr.StatusCode == http.StatusTooManyRequests + } + var netErr net.Error + return errors.As(err, &netErr) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.ECONNRESET) +} + func wrapAliOSSObjectError(err error) error { if err == nil { return nil From dc4109d71922c5b4f62562f773bb27159ad7de80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 19:55:21 -0700 Subject: [PATCH 18/29] DeleteObjects: *DeleteError names each path left, every backend tries every path Failed maps each path not deleted to why; every other path is deleted, so a caller can retry or report exactly those. GCS, Azure and OCI no longer stop at the first failure; once the context ends, the paths left fail without a request. --- README.md | 4 ++-- alioss.go | 16 ++++++++++------ contract_test.go | 8 ++++++-- e2e/floci_test.go | 25 ++++++++++++++++--------- local.go | 12 +++++++----- objects/objects.go | 33 ++++++++++++++++++++++++++++++--- s3.go | 21 ++++++++++----------- storage.go | 38 +++++++++++++++++++++++++------------- 8 files changed, 106 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index c4c9d78..c7a20d9 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,12 @@ Object storage on S3 and S3-compatible stores, GCS, Azure Blob, OCI Object Stora - `Bucket` (`storage/objects`): `PutObject`, `UploadObject` (streams), `GetObject`, `GetObjectRange`, `DownloadObject` (to any `io.Writer`), `ObjectExists`, `ListObjects` (byte order), `DeleteObjects`, `PresignedURL`. - `Storage` is `Bucket` plus `UploadFile` and `DownloadFile`. `DownloadFile` replaces the local file only once the whole object is read. - `PutOptions{ContentType, IfAbsent}` on every write. `IfAbsent` refuses to replace an object, atomically where the store honors the condition. -- Errors: a missing object is `ErrNotFound`, a write `IfAbsent` refuses is `ErrExists` (`errors.Is`); `ErrorWithStatusCode` carries the store's HTTP status (`errors.As`). +- Errors: a missing object is `ErrNotFound`, a write `IfAbsent` refuses is `ErrExists` (`errors.Is`); `ErrorWithStatusCode` carries the store's HTTP status and `DeleteError` the paths a delete left (`errors.As`). - `storage/objects` imports no provider SDK, so a fake or a library can implement or accept the interfaces without pulling in every SDK. ## Behavior -- **S3 semantics** where stores differ. `DeleteObjects` of a missing path succeeds, as S3's delete does, and fails while any path it was given still holds an object, so a retry after a partial failure is safe. +- **S3 semantics** where stores differ. `DeleteObjects` of a missing path succeeds, as S3's delete does. It tries every path, and when some are not deleted fails with a `*DeleteError` whose `Failed` names each one with its error (`errors.As`); every other path is deleted, and a retry is safe. - **Keys** are the store's keys as given. The local directory, as a filesystem, drops a leading `/`, cleans `//` and `.`, refuses keys outside it, and cannot hold both `a` and `a/b`. - **Timeouts**: a connection that moves no bytes either way for 5 minutes is cut; a slow but live transfer never is. The context bounds the call. - **Retries**: every store retries throttling, 5xx and network errors with backoff, including AliOSS, whose SDK does not. A stream given to `UploadObject` is sent once on AliOSS. diff --git a/alioss.go b/alioss.go index 204f3ba..41dbbdd 100644 --- a/alioss.go +++ b/alioss.go @@ -224,6 +224,7 @@ func (s *aliOSSStorage) ListObjects(ctx context.Context, prefix string) ([]strin // no per-key errors, only the keys it deleted, so a key it leaves out fails the // call unless it holds nothing. func (s *aliOSSStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { + failed := map[string]error{} for i := 0; i < len(storagePaths); i += 1000 { batch := storagePaths[i:min(i+1000, len(storagePaths))] var res oss.DeleteObjectsResult @@ -232,7 +233,10 @@ func (s *aliOSSStorage) DeleteObjects(ctx context.Context, storagePaths []string return err }) if err != nil { - return wrapAliOSSObjectError(err) + for _, k := range batch { + failed[k] = wrapAliOSSObjectError(err) + } + continue } deleted := make(map[string]bool, len(res.DeletedObjects)) for _, k := range res.DeletedObjects { @@ -243,15 +247,15 @@ func (s *aliOSSStorage) DeleteObjects(ctx context.Context, storagePaths []string continue } exists, err := s.ObjectExists(ctx, k) - if err != nil { - return err + if err == nil && exists { + err = errors.New("delete not confirmed") } - if exists { - return fmt.Errorf("storage: delete of %s not confirmed", k) + if err != nil { + failed[k] = err } } } - return nil + return deleteError(failed) } // aliOSSAttempts is how many times a call is sent: the v1 SDK never retries. diff --git a/contract_test.go b/contract_test.go index 271df2e..f044475 100644 --- a/contract_test.go +++ b/contract_test.go @@ -237,9 +237,13 @@ func TestS3DeleteObjectsKeyErrors(t *testing.T) { err = s.DeleteObjects(context.Background(), []string{"a", "b"}) if tc.deleted { require.NoError(t, err) - } else { - require.ErrorContains(t, err, tc.code) + return } + // The caller learns which keys are left, and only those. + var failed *storage.DeleteError + require.ErrorAs(t, err, &failed) + require.Len(t, failed.Failed, 1) + require.ErrorContains(t, failed.Failed["b"], tc.code) }) } } diff --git a/e2e/floci_test.go b/e2e/floci_test.go index 4bf538d..40defdb 100644 --- a/e2e/floci_test.go +++ b/e2e/floci_test.go @@ -232,18 +232,25 @@ func testConditionalCommit(t *testing.T, s storage.Storage, endpoint string, at require.Equal(t, commits.Load(), conditional.Load(), "a commit went out without the condition") } -// testDeletesRefused deletes where the store refuses it: the object is still -// there, so the call must fail, missing keys alongside it or not. +// testDeletesRefused deletes where the store refuses it: the objects are still +// there, so the call must fail, naming each of them, past the first refusal. func testDeletesRefused(t *testing.T, s, noDeletes storage.Storage) { ctx := t.Context() - path := fmt.Sprintf("deletes-refused-%d", time.Now().UnixNano()) - require.NoError(t, s.PutObject(ctx, path, []byte("kept"), storage.PutOptions{})) - t.Cleanup(func() { _ = s.DeleteObjects(context.Background(), []string{path}) }) + prefix := fmt.Sprintf("deletes-refused-%d/", time.Now().UnixNano()) + paths := []string{prefix + "a", prefix + "b"} + for _, p := range paths { + require.NoError(t, s.PutObject(ctx, p, []byte("kept"), storage.PutOptions{})) + } + t.Cleanup(func() { _ = s.DeleteObjects(context.Background(), paths) }) - require.Error(t, noDeletes.DeleteObjects(ctx, []string{path + "-missing", path})) - exists, err := s.ObjectExists(ctx, path) - require.NoError(t, err) - require.True(t, exists) + var failed *storage.DeleteError + require.ErrorAs(t, noDeletes.DeleteObjects(ctx, paths), &failed) + for _, p := range paths { + require.Contains(t, failed.Failed, p) + exists, err := s.ObjectExists(ctx, p) + require.NoError(t, err) + require.True(t, exists) + } } // testWriteOnly writes IfAbsent with credentials that may write but not read: diff --git a/local.go b/local.go index d472d71..65c2fb4 100644 --- a/local.go +++ b/local.go @@ -274,15 +274,17 @@ func (u *localUploader) ListObjects(_ context.Context, prefix string) ([]string, } func (u *localUploader) DeleteObjects(ctx context.Context, storagePaths []string) error { + failed := map[string]error{} for _, p := range storagePaths { - if err := ctx.Err(); err != nil { - return err + err := ctx.Err() + if err == nil { + err = u.deleteObject(p) } - if err := u.deleteObject(p); err != nil { - return err + if err != nil { + failed[p] = err } } - return nil + return deleteError(failed) } // deleteObject removes the file, then each parent directory it leaves empty. A diff --git a/objects/objects.go b/objects/objects.go index 1680e2e..c20e8b5 100644 --- a/objects/objects.go +++ b/objects/objects.go @@ -22,6 +22,8 @@ import ( "errors" "fmt" "io" + "maps" + "slices" "time" ) @@ -58,9 +60,11 @@ type Bucket interface { // ListObjects returns every key under prefix, recursively, as full paths, // in byte order. ListObjects(ctx context.Context, prefix string) ([]string, error) - // DeleteObjects succeeds for paths that hold nothing, as S3's delete does, - // and fails when a path is not deleted; some may be deleted by then, and a - // retry is safe. GCS, Azure and OCI send one request per path, 16 at once. + // DeleteObjects succeeds for paths that hold nothing, as S3's delete does. + // It tries every path, and when some are not deleted fails with a + // *DeleteError naming them; every other path is deleted, and a retry of the + // same paths is safe. GCS, Azure and OCI send one request per path, 16 at + // once. DeleteObjects(ctx context.Context, storagePaths []string) error // PresignedURL is a GET URL; local returns a file:// URL. @@ -94,6 +98,29 @@ var ( ErrExists = errors.New("storage: object exists") ) +// DeleteError is DeleteObjects' failure: Failed holds each path it did not +// delete, with why. Every other path it was given is deleted. +type DeleteError struct { + Failed map[string]error +} + +func (e *DeleteError) Error() string { + paths := slices.Sorted(maps.Keys(e.Failed)) + msg := fmt.Sprintf("storage: %d paths not deleted", len(paths)) + for i, p := range paths { + if i == 3 { + return msg + fmt.Sprintf(", and %d more", len(paths)-i) + } + msg += fmt.Sprintf(", %s: %v", p, e.Failed[p]) + } + return msg +} + +// Unwrap is the paths' errors, so errors.Is and errors.As see them. +func (e *DeleteError) Unwrap() []error { + return slices.Collect(maps.Values(e.Failed)) +} + // ErrorWithStatusCode carries the HTTP status a store answered with, under // ErrNotFound or ErrExists where those apply; find it with errors.As. type ErrorWithStatusCode struct { diff --git a/s3.go b/s3.go index c502de6..ea9d54e 100644 --- a/s3.go +++ b/s3.go @@ -485,7 +485,7 @@ func (s *s3Storage) ListObjects(ctx context.Context, prefix string) ([]string, e // DeleteObjects fails when any key fails, even within a batch S3 accepted. func (s *s3Storage) DeleteObjects(ctx context.Context, storagePaths []string) error { - client := s.client + failed := map[string]error{} for i := 0; i < len(storagePaths); i += 1000 { batch := storagePaths[i:min(i+1000, len(storagePaths))] objects := make([]types.ObjectIdentifier, 0, len(batch)) @@ -501,23 +501,22 @@ func (s *s3Storage) DeleteObjects(ctx context.Context, storagePaths []string) er if !awsEndpoint(s.conf.Endpoint) { input.ChecksumAlgorithm = types.ChecksumAlgorithmSha256 } - out, err := client.DeleteObjects(ctx, input) + out, err := s.client.DeleteObjects(ctx, input) if err != nil { - return wrapS3Error(err) + for _, p := range batch { + failed[p] = wrapS3Error(err) + } + continue } - // Any key the store reports that is not already gone is still there, - // so the call fails rather than leave it behind unseen. - var errs []error + // S3 answers 200 and lists the keys it could not delete. NoSuchKey is + // a key already gone; any other is still there. for _, e := range out.Errors { if aws.ToString(e.Code) != "NoSuchKey" { - errs = append(errs, fmt.Errorf("storage: delete %s: %s %s", aws.ToString(e.Key), aws.ToString(e.Code), aws.ToString(e.Message))) + failed[aws.ToString(e.Key)] = fmt.Errorf("%s: %s", aws.ToString(e.Code), aws.ToString(e.Message)) } } - if err := errors.Join(errs...); err != nil { - return err - } } - return nil + return deleteError(failed) } func (s *s3Storage) PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { diff --git a/storage.go b/storage.go index 9df349f..be0b0be 100644 --- a/storage.go +++ b/storage.go @@ -31,6 +31,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "sync/atomic" "time" @@ -47,6 +48,7 @@ type ( Bucket = objects.Bucket PutOptions = objects.PutOptions ErrorWithStatusCode = objects.ErrorWithStatusCode + DeleteError = objects.DeleteError ) var ( @@ -141,28 +143,38 @@ const ( // delete is one round trip per object. const deleteConcurrency = 16 -// deleteEach deletes every path with del, deleteConcurrency at a time, and -// returns the first error. A failure stops new deletes but lets those in -// flight finish, rather than cancel them: an OCI call canceled mid-flight -// races inside the SDK. A retry is safe, as del treats a missing path as -// deleted. +// deleteEach deletes every path with del, deleteConcurrency at a time, where a +// store takes one path per request. Once the context ends, the paths left are +// failed without a request: an OCI call canceled mid-flight races in its SDK. func deleteEach(ctx context.Context, paths []string, del func(ctx context.Context, path string) error) error { + var mu sync.Mutex + failed := map[string]error{} var g errgroup.Group g.SetLimit(deleteConcurrency) - var failed atomic.Bool for _, p := range paths { - if failed.Load() { - break - } g.Go(func() error { - err := del(ctx, p) + err := ctx.Err() + if err == nil { + err = del(ctx, p) + } if err != nil { - failed.Store(true) + mu.Lock() + failed[p] = err + mu.Unlock() } - return err + return nil }) } - return g.Wait() + _ = g.Wait() + return deleteError(failed) +} + +// deleteError is a *DeleteError for the paths in failed, or nil for none. +func deleteError(failed map[string]error) error { + if len(failed) == 0 { + return nil + } + return &DeleteError{Failed: failed} } // idleTimeout is how long a connection may move no bytes either way. A store From ceb53a4f715d7bc5c0f42bdf2b77b1f2459146cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 20:44:21 -0700 Subject: [PATCH 19/29] S3: a key DeleteObjects could not delete keeps its code as an API error, found with errors.As --- contract_test.go | 5 ++++- s3.go | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/contract_test.go b/contract_test.go index f044475..ab1d0f6 100644 --- a/contract_test.go +++ b/contract_test.go @@ -39,6 +39,7 @@ import ( "testing" "time" + "github.com/aws/smithy-go" "github.com/stretchr/testify/require" "github.com/livekit/storage" @@ -243,7 +244,9 @@ func TestS3DeleteObjectsKeyErrors(t *testing.T) { var failed *storage.DeleteError require.ErrorAs(t, err, &failed) require.Len(t, failed.Failed, 1) - require.ErrorContains(t, failed.Failed["b"], tc.code) + var apiErr smithy.APIError + require.ErrorAs(t, failed.Failed["b"], &apiErr) + require.Equal(t, tc.code, apiErr.ErrorCode()) }) } } diff --git a/s3.go b/s3.go index ea9d54e..6c723e4 100644 --- a/s3.go +++ b/s3.go @@ -508,11 +508,12 @@ func (s *s3Storage) DeleteObjects(ctx context.Context, storagePaths []string) er } continue } - // S3 answers 200 and lists the keys it could not delete. NoSuchKey is - // a key already gone; any other is still there. + // S3 answers 200 and lists the keys it could not delete, each with a + // code but no status, kept as the API error its other calls return. + // NoSuchKey is a key already gone; any other is still there. for _, e := range out.Errors { if aws.ToString(e.Code) != "NoSuchKey" { - failed[aws.ToString(e.Key)] = fmt.Errorf("%s: %s", aws.ToString(e.Code), aws.ToString(e.Message)) + failed[aws.ToString(e.Key)] = &smithy.GenericAPIError{Code: aws.ToString(e.Code), Message: aws.ToString(e.Message)} } } } From 42ef441ca5be8c81954c035d1f6b77756fbf4f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 21:26:31 -0700 Subject: [PATCH 20/29] objects: its own module, with no requirements, so a fake or library that implements the interfaces inherits no SDK's versions --- e2e/go.mod | 3 +++ go.mod | 3 +++ objects/go.mod | 3 +++ 3 files changed, 9 insertions(+) create mode 100644 objects/go.mod diff --git a/e2e/go.mod b/e2e/go.mod index 8a0967e..d9eb431 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -79,6 +79,7 @@ require ( github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/livekit/protocol v1.52.1 // indirect + github.com/livekit/storage/objects v0.0.0-00010101000000-000000000000 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/user v0.3.0 // indirect github.com/moby/term v0.5.0 // indirect @@ -128,3 +129,5 @@ require ( ) replace github.com/livekit/storage => ../ + +replace github.com/livekit/storage/objects => ../objects diff --git a/go.mod b/go.mod index 01e4893..baca2f9 100644 --- a/go.mod +++ b/go.mod @@ -66,6 +66,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.22 // indirect github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/livekit/storage/objects v0.0.0-00010101000000-000000000000 github.com/pierrec/lz4/v4 v4.1.28 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/puzpuzpuz/xsync/v4 v4.5.0 // indirect @@ -98,3 +99,5 @@ require ( google.golang.org/grpc v1.84.0 // indirect google.golang.org/protobuf v1.36.12 // indirect ) + +replace github.com/livekit/storage/objects => ./objects diff --git a/objects/go.mod b/objects/go.mod new file mode 100644 index 0000000..d3b4ffa --- /dev/null +++ b/objects/go.mod @@ -0,0 +1,3 @@ +module github.com/livekit/storage/objects + +go 1.26.0 From 9c1b445416f40dd3248167de6e0d26e6c7a97827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sat, 17 Oct 2026 21:29:02 -0700 Subject: [PATCH 21/29] Storage: require the objects module at the commit that made it; CI checks the requirement matches objects/ --- .github/workflows/test.yaml | 7 +++++++ README.md | 2 +- e2e/go.mod | 2 +- go.mod | 4 +--- go.sum | 2 ++ 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 756a87d..e7e276e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -30,6 +30,12 @@ jobs: - name: Download Go modules run: go mod download && (cd e2e && go mod download) + # The root module builds against the objects version it requires, which + # must be the interfaces in this tree: a change to objects/ needs the + # requirement bumped to the commit that made it. + - name: objects requirement matches objects/ + run: diff -r --exclude=go.sum --exclude=LICENSE "$(go list -m -f '{{.Dir}}' github.com/livekit/storage/objects)" objects + - name: Lint uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 with: @@ -47,6 +53,7 @@ jobs: run: | set -euo pipefail go test -race -json -v ./... 2>&1 | gotestfmt + (cd objects && go vet ./...) cd e2e && go test -race -json -v -skip '^Test(Floci|MinIO)' ./... 2>&1 | gotestfmt # Every backend end to end against its floci emulator (e2e/floci_test.go), diff --git a/README.md b/README.md index c7a20d9..028411c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Object storage on S3 and S3-compatible stores, GCS, Azure Blob, OCI Object Stora - `Storage` is `Bucket` plus `UploadFile` and `DownloadFile`. `DownloadFile` replaces the local file only once the whole object is read. - `PutOptions{ContentType, IfAbsent}` on every write. `IfAbsent` refuses to replace an object, atomically where the store honors the condition. - Errors: a missing object is `ErrNotFound`, a write `IfAbsent` refuses is `ErrExists` (`errors.Is`); `ErrorWithStatusCode` carries the store's HTTP status and `DeleteError` the paths a delete left (`errors.As`). -- `storage/objects` imports no provider SDK, so a fake or a library can implement or accept the interfaces without pulling in every SDK. +- `storage/objects` is a module of its own with no requirements, so a fake or a library can implement or accept the interfaces without inheriting any SDK or its versions. A change to it is two commits: `objects/` first, then this module's requirement bumped to that commit (CI checks they match). ## Behavior diff --git a/e2e/go.mod b/e2e/go.mod index d9eb431..8ee21cb 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -79,7 +79,7 @@ require ( github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/livekit/protocol v1.52.1 // indirect - github.com/livekit/storage/objects v0.0.0-00010101000000-000000000000 // indirect + github.com/livekit/storage/objects v0.0.0-20261018042631-42ef441ca5be // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/user v0.3.0 // indirect github.com/moby/term v0.5.0 // indirect diff --git a/go.mod b/go.mod index baca2f9..d0416a7 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.22 // indirect github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect - github.com/livekit/storage/objects v0.0.0-00010101000000-000000000000 + github.com/livekit/storage/objects v0.0.0-20261018042631-42ef441ca5be github.com/pierrec/lz4/v4 v4.1.28 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/puzpuzpuz/xsync/v4 v4.5.0 // indirect @@ -99,5 +99,3 @@ require ( google.golang.org/grpc v1.84.0 // indirect google.golang.org/protobuf v1.36.12 // indirect ) - -replace github.com/livekit/storage/objects => ./objects diff --git a/go.sum b/go.sum index 43f3c04..44ef553 100644 --- a/go.sum +++ b/go.sum @@ -159,6 +159,8 @@ github.com/livekit/protocol v1.52.1 h1:w8wRjRlj/57AJBxwomEzpEMzf+lT0qOoLQLO7awdT github.com/livekit/protocol v1.52.1/go.mod h1:t5CV/LCsWbI9zsSTHTJKDyLtByqvvVkioSP7jgqpIGA= github.com/livekit/psrpc v0.8.0 h1:uo3x+DL8pK2OIaCxg/s9sLUO2cIocS5HwdPhhI+m+fQ= github.com/livekit/psrpc v0.8.0/go.mod h1:6KnU8vULoizX0Er9NNfeKJDV8ug87j3fuo19M0bhj6Q= +github.com/livekit/storage/objects v0.0.0-20261018042631-42ef441ca5be h1:QchdOj9qHifeU69Mb/1+iPrItoaxYZX262eIDrw3osA= +github.com/livekit/storage/objects v0.0.0-20261018042631-42ef441ca5be/go.mod h1:S3+jo6fCZ+A7Z9N6VGTQoffHRGZyNBHn/Jlb/06pXuM= github.com/oracle/oci-go-sdk/v65 v65.126.1 h1:WmQ2Igq7/L/cHlR2jZnJkBC2UI51Ams4DJZW3CZGVBo= github.com/oracle/oci-go-sdk/v65 v65.126.1/go.mod h1:YmvgbsnUfrsiJ410XAGQ1BRsrUYJJ7W3O5wUikmeI8w= github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio= From 0d875ff129e17b2b2e5e60124f6f9f5e860d386f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sun, 18 Oct 2026 10:30:48 -0700 Subject: [PATCH 22/29] objects: released by tag on merge; go.mod names the version, go.work builds the tree go.mod requires github.com/livekit/storage/objects v0.1.0, which objects-release.yaml tags on main when this merges, so main never requires a commit only a deleted branch held. go.work builds the repo against objects/ until the tag exists. CI fails a PR that changes objects/ under an existing tag, or names something other than a release. --- .github/workflows/objects-release.yaml | 29 ++++++++++++++++++++++++++ .github/workflows/test.yaml | 22 ++++++++++++++----- README.md | 2 +- go.mod | 2 +- go.sum | 2 -- 5 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/objects-release.yaml diff --git a/.github/workflows/objects-release.yaml b/.github/workflows/objects-release.yaml new file mode 100644 index 0000000..7e811e4 --- /dev/null +++ b/.github/workflows/objects-release.yaml @@ -0,0 +1,29 @@ +name: objects release + +permissions: + contents: write + +on: + push: + branches: [main] + +jobs: + # Tags the objects version go.mod requires on this commit, so main never + # requires one that does not exist. An existing tag is left as it is: the + # test workflow kept objects/ from changing under it. + tag: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Tag objects + run: | + tag="objects/$(awk '$1 == "github.com/livekit/storage/objects" {print $2}' go.mod)" + if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then + git diff --quiet "refs/tags/$tag" HEAD -- objects || { echo "::error::objects/ differs from $tag"; exit 1; } + exit 0 + fi + git tag "$tag" HEAD + git push origin "$tag" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e7e276e..3f56768 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,6 +15,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 # the objects/ tags # setup-go caches the module and build caches, keyed on both modules. - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -30,11 +32,21 @@ jobs: - name: Download Go modules run: go mod download && (cd e2e && go mod download) - # The root module builds against the objects version it requires, which - # must be the interfaces in this tree: a change to objects/ needs the - # requirement bumped to the commit that made it. - - name: objects requirement matches objects/ - run: diff -r --exclude=go.sum --exclude=LICENSE "$(go list -m -f '{{.Dir}}' github.com/livekit/storage/objects)" objects + # objects/ is released by tag: objects-release.yaml tags the version + # go.mod requires on main after the merge, and go.work builds this tree + # against objects/ until then. A tag never moves, so once it exists, a + # change to objects/ needs the next version. + - name: objects version names these interfaces + run: | + v=$(awk '$1 == "github.com/livekit/storage/objects" {print $2}' go.mod) + if ! [[ "$v" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::go.mod requires github.com/livekit/storage/objects at $v: name a release, such as v0.2.0" + exit 1 + fi + if git rev-parse -q --verify "refs/tags/objects/$v" >/dev/null && ! git diff --quiet "refs/tags/objects/$v" HEAD -- objects; then + echo "::error::objects/ changed since objects/$v: raise github.com/livekit/storage/objects in go.mod to the next version" + exit 1 + fi - name: Lint uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 diff --git a/README.md b/README.md index 028411c..761fb3f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Object storage on S3 and S3-compatible stores, GCS, Azure Blob, OCI Object Stora - `Storage` is `Bucket` plus `UploadFile` and `DownloadFile`. `DownloadFile` replaces the local file only once the whole object is read. - `PutOptions{ContentType, IfAbsent}` on every write. `IfAbsent` refuses to replace an object, atomically where the store honors the condition. - Errors: a missing object is `ErrNotFound`, a write `IfAbsent` refuses is `ErrExists` (`errors.Is`); `ErrorWithStatusCode` carries the store's HTTP status and `DeleteError` the paths a delete left (`errors.As`). -- `storage/objects` is a module of its own with no requirements, so a fake or a library can implement or accept the interfaces without inheriting any SDK or its versions. A change to it is two commits: `objects/` first, then this module's requirement bumped to that commit (CI checks they match). +- `storage/objects` is a module of its own with no requirements, so a fake or a library can implement or accept the interfaces without inheriting any SDK or its versions. A PR that changes `objects/` raises the `objects` version in `go.mod` (CI says so when it is missing); `go.work` builds the repo against `objects/` until then, and the version is tagged `objects/vX.Y.Z` on main when the PR merges. ## Behavior diff --git a/go.mod b/go.mod index d0416a7..6ce2835 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.22 // indirect github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect - github.com/livekit/storage/objects v0.0.0-20261018042631-42ef441ca5be + github.com/livekit/storage/objects v0.1.0 github.com/pierrec/lz4/v4 v4.1.28 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/puzpuzpuz/xsync/v4 v4.5.0 // indirect diff --git a/go.sum b/go.sum index 44ef553..43f3c04 100644 --- a/go.sum +++ b/go.sum @@ -159,8 +159,6 @@ github.com/livekit/protocol v1.52.1 h1:w8wRjRlj/57AJBxwomEzpEMzf+lT0qOoLQLO7awdT github.com/livekit/protocol v1.52.1/go.mod h1:t5CV/LCsWbI9zsSTHTJKDyLtByqvvVkioSP7jgqpIGA= github.com/livekit/psrpc v0.8.0 h1:uo3x+DL8pK2OIaCxg/s9sLUO2cIocS5HwdPhhI+m+fQ= github.com/livekit/psrpc v0.8.0/go.mod h1:6KnU8vULoizX0Er9NNfeKJDV8ug87j3fuo19M0bhj6Q= -github.com/livekit/storage/objects v0.0.0-20261018042631-42ef441ca5be h1:QchdOj9qHifeU69Mb/1+iPrItoaxYZX262eIDrw3osA= -github.com/livekit/storage/objects v0.0.0-20261018042631-42ef441ca5be/go.mod h1:S3+jo6fCZ+A7Z9N6VGTQoffHRGZyNBHn/Jlb/06pXuM= github.com/oracle/oci-go-sdk/v65 v65.126.1 h1:WmQ2Igq7/L/cHlR2jZnJkBC2UI51Ams4DJZW3CZGVBo= github.com/oracle/oci-go-sdk/v65 v65.126.1/go.mod h1:YmvgbsnUfrsiJ410XAGQ1BRsrUYJJ7W3O5wUikmeI8w= github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio= From bc19ae524063d38d7ca8422fc117742b68873bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sun, 18 Oct 2026 10:31:00 -0700 Subject: [PATCH 23/29] go.work committed: the repo builds against objects/ until its version is tagged --- .gitignore | 2 - go.work | 7 + go.work.sum | 470 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 477 insertions(+), 2 deletions(-) create mode 100644 go.work create mode 100644 go.work.sum diff --git a/.gitignore b/.gitignore index b9b720f..5956e82 100644 --- a/.gitignore +++ b/.gitignore @@ -18,8 +18,6 @@ # vendor/ # Go workspace file -go.work -go.work.sum # env file .env* diff --git a/go.work b/go.work new file mode 100644 index 0000000..0b274c8 --- /dev/null +++ b/go.work @@ -0,0 +1,7 @@ +go 1.26.0 + +use ( + . + ./e2e + ./objects +) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 0000000..ed6e33d --- /dev/null +++ b/go.work.sum @@ -0,0 +1,470 @@ +atomicgo.dev/cursor v0.2.0 h1:H6XN5alUJ52FZZUkI7AlJbUc1aW38GWZalpYRPpoPOw= +atomicgo.dev/cursor v0.2.0/go.mod h1:Lr4ZJB3U7DfPPOkbH7/6TOtJ4vFGHlgj1nc+n900IpU= +atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8= +atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ= +atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs= +atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU= +cloud.google.com/go/accessapproval v1.13.0 h1:kx3RQSS0VglTngQTPfywgVj+xFgre/vJouGETBlPU1E= +cloud.google.com/go/accessapproval v1.13.0/go.mod h1:7bmInw17bQX+ZPi7YmReC3xKymDrMmxXaUnaI6zQOqI= +cloud.google.com/go/accesscontextmanager v1.15.0 h1:0baVgug9IFV8y5cIoD4iN/m/QtJzk2k+j3MW+zJWCVw= +cloud.google.com/go/accesscontextmanager v1.15.0/go.mod h1:YjW9urferk8i9ALwBF3bmdcogZeQYRn2yWwR8nkhsBc= +cloud.google.com/go/aiplatform v1.126.0 h1:5PxeWpQfkAyN8mtgtVp4H5nP+ntwPLFMjNREVdiolR0= +cloud.google.com/go/aiplatform v1.126.0/go.mod h1:iR3za3evdprLe1XL2pLu0cYVCuTbc87QG0pgvcgiJlE= +cloud.google.com/go/analytics v0.35.0 h1:GuwmzJHIaQRvtko6g4wW1ngQ7rpDvDLZ8M3iP3kiflU= +cloud.google.com/go/analytics v0.35.0/go.mod h1:V9Qef2N0y8GDqQ9FTlmM2XpDEMYonZJRPSUNGZlPCcc= +cloud.google.com/go/apigateway v1.13.0 h1:IENNmFQlMsl58RlvYHf6wxBNQFeP+lAnVNcyxtmByTE= +cloud.google.com/go/apigateway v1.13.0/go.mod h1:pvEpOuuOIw2ev9VCcOyVkDXHHL4lvgMuqIe7XjJ8JoU= +cloud.google.com/go/apigeeconnect v1.12.0 h1:YvZhi0QkHobBRMrN6ri9PUPnnhgEd4VpW0+yo1WI7r0= +cloud.google.com/go/apigeeconnect v1.12.0/go.mod h1:mYJekCKZHc2ia5yZX5lwtexTn9CzsOfb6+sh/2hi42Q= +cloud.google.com/go/apigeeregistry v1.1.0 h1:2zIogaYq7dF4E1dAGC4MUnasRuSmohsMl7wVuJ963/4= +cloud.google.com/go/apigeeregistry v1.1.0/go.mod h1:4ZFhQlxMuyfDMz9ORDSV8FPZtf2yPQkKjigsFtrrE4Y= +cloud.google.com/go/appengine v1.15.0 h1:0MiBM2KGk1WAhh489WoELQ7IeW2qw6HTE6nZIlrOw5U= +cloud.google.com/go/appengine v1.15.0/go.mod h1:/8gGZsOX5GDjOo4mAWk8IV59p2991dxTbEtKIlhDjzU= +cloud.google.com/go/area120 v0.15.0 h1:9v6HeQsBpfRqoZtNeLI4V8SQ7Jcmt6s2gmgbjl0fVFc= +cloud.google.com/go/area120 v0.15.0/go.mod h1:jD1fw9W4xxIZMY68g7PpbCPleoeGddFs5jPcdhfg3+Y= +cloud.google.com/go/artifactregistry v1.26.0 h1:iq5kkdY2FJY8RkkNghUelb3EOcqyy7dVhfDGVk/Xw9g= +cloud.google.com/go/artifactregistry v1.26.0/go.mod h1:c5FPi5GtDBP+OAr5kKhCBNQDT9ZgAyobXQjekx93VWs= +cloud.google.com/go/asset v1.28.0 h1:M6YE1exBuZQhTi8wfIKrwjKK6b2ySfHKgrqXjPZA4EM= +cloud.google.com/go/asset v1.28.0/go.mod h1:Pnvjhay8/FgodOH9uJC8OkfJfRtSnNIIU4WSxg5JfJw= +cloud.google.com/go/assuredworkloads v1.19.0 h1:XyNQ9SBX0eUkZYCZhC8dOY6ZFeSF6/OIjHn5dQs78Jo= +cloud.google.com/go/assuredworkloads v1.19.0/go.mod h1:/UGGtFCMokM3sGJ4FxjfmLvvFpPa5I/Oz68mwk4Su+0= +cloud.google.com/go/automl v1.21.0 h1:brKXr4i9AMVfgbCKOYSu4rnIoz1ILIgcIAy3nerzEGc= +cloud.google.com/go/automl v1.21.0/go.mod h1:MNbhUevuECzM3jqSOM7hmOedOdRJkm8xbbXW44SU15U= +cloud.google.com/go/baremetalsolution v1.10.0 h1:6MaZilXzGZJ3mGqbSCdDezs9yrze6kcmBABsRUZ/aYI= +cloud.google.com/go/baremetalsolution v1.10.0/go.mod h1:xhhT9VQiKPFd2fUs4oeDSRrxV0sb0PGeVmuZoUE2cBA= +cloud.google.com/go/batch v1.20.0 h1:nTL5x9HA1yVdPIdeqcbLjNnV9iYDYvMNmP6AnZ+qbHU= +cloud.google.com/go/batch v1.20.0/go.mod h1:ABT/5QqsIDsONa+n/8C7XYPjwh/kjOEPXukcRTaMsCg= +cloud.google.com/go/beyondcorp v1.8.0 h1:rsyle6zjxce2s57BpULw5u170xS+KSjFENSY7C0MZbs= +cloud.google.com/go/beyondcorp v1.8.0/go.mod h1:aVxzwamO8H4GXWQHowBAmL0KYNfYpW4E6Do2wfP0RYs= +cloud.google.com/go/bigquery v1.79.0 h1:+tW6oRuP/4dOgdl/p32on0+3OktVQtU+veOMxhjoOhE= +cloud.google.com/go/bigquery v1.79.0/go.mod h1:QTt5tgZxqqvZs3dOZKpvriGqy+CdvY9LyetirFZRPOE= +cloud.google.com/go/bigtable v1.47.0 h1:NGLgDSr/i79BTGCjxH/maPKxyvl5q8/SsBsyLK52kdI= +cloud.google.com/go/bigtable v1.47.0/go.mod h1:GUM6PdkG3rrDse9kugqvX5+ktwo3ldfLtLi1VFn5Wj4= +cloud.google.com/go/billing v1.26.0 h1:6RRjbRd6iZKZFb7/MgRvmXKq/Ism02ckkZLJazj4CQ0= +cloud.google.com/go/billing v1.26.0/go.mod h1:axqDO1uHegh7u5qngkTfqN1djAeLGsWAFAblERgmgEk= +cloud.google.com/go/binaryauthorization v1.16.0 h1:f1iNUaHWP9XPdh6grftTlIRuJSBytud4j51l1Jce94E= +cloud.google.com/go/binaryauthorization v1.16.0/go.mod h1:E+iC5Avu4pdItdzGiSGHnh6TfQrl+KmPxDDg/T/VuHs= +cloud.google.com/go/certificatemanager v1.15.0 h1:PwljEZlI3lZgoFUG0pnIBNOwrQUaAP+gS0ZahTwUZBA= +cloud.google.com/go/certificatemanager v1.15.0/go.mod h1:8dfGG2/TbUpCNqsCF/TIMOGV0OVvU6nhkZWTU4MmCXU= +cloud.google.com/go/channel v1.27.0 h1:fqDZDzVw61c4iIUe7rQ+cBYlBdSZ3yqnzw5WrZTPjqs= +cloud.google.com/go/channel v1.27.0/go.mod h1:9ekufBLXuQ6j1oyqtDSIp29qWU5EwCi8WUi9qkLn3MA= +cloud.google.com/go/cloudbuild v1.32.0 h1:tlF+KSIJJ0kaKkxACpk9htZ1euDpyOqPuO1aHR4j3oI= +cloud.google.com/go/cloudbuild v1.32.0/go.mod h1:mYgcM8CMaPmAnO7GxSQ9ADAxVRwS+1b7s6WVkt29OXY= +cloud.google.com/go/clouddms v1.14.0 h1:oaktwVyUeKTuzjpde3xbpmc9bAQPQq6WJWOBr7K6YEA= +cloud.google.com/go/clouddms v1.14.0/go.mod h1:qSwET2Q27cJ4wCDsPsbkagXqQqkWfOy+gU3RjMsT/c8= +cloud.google.com/go/cloudtasks v1.18.0 h1:KzT7hfix/9/xAf20tNPIxwX59XGpRF0Lun2t8LHOj9E= +cloud.google.com/go/cloudtasks v1.18.0/go.mod h1:3KeCxwtGEyaySL7CR3lMmEa2I4mq1ynXdgmfNiO4RYE= +cloud.google.com/go/compute v1.62.0 h1:tJ7lKJ8YEVa6vZX03Jc8o1YePbjKDOQhDw1BscMZ1bs= +cloud.google.com/go/compute v1.62.0/go.mod h1:Xm6PbsLgBpAg4va77ljbBdpMjzuU+uPp5Ze2dnZq7lw= +cloud.google.com/go/contactcenterinsights v1.23.0 h1:EtrRf8T7ajR+dR8nseDxwKy3o1VaWP1GFAu7lgMmT9A= +cloud.google.com/go/contactcenterinsights v1.23.0/go.mod h1:uB/kygbfYH/gWEq3NEgq3QRI7/MvpjFyX81ajcW5YAI= +cloud.google.com/go/container v1.51.0 h1:KdeWHqwlOvABdhkhSTSb4QFySMOuXnc3UUoQMmq9lic= +cloud.google.com/go/container v1.51.0/go.mod h1:EvqoT2eXfxLweXXUlhAMGR0sOAB00XPzEjoL01esSDs= +cloud.google.com/go/containeranalysis v0.19.0 h1:89ZhFvJHWzX9/jUJy/IEVbtSa2hv4+NtsvwWMEsUYnY= +cloud.google.com/go/containeranalysis v0.19.0/go.mod h1:Zq0XHzUIa0oTa7H6aSR8HWqeJnoRI9syUcYJzfozjZQ= +cloud.google.com/go/datacatalog v1.33.0 h1:8V80PpoAGdOOr2QhBrp4wZ66MDCbATdAB/fmVmo5rlU= +cloud.google.com/go/datacatalog v1.33.0/go.mod h1:/EMN04S73fZcPdtNg86VYLDrhi2HheMehQtMCS86Klk= +cloud.google.com/go/dataflow v0.16.0 h1:BchGCAl9QIZ/pyTGokv5V3daxBTrXsOmrU4ARXoTNd4= +cloud.google.com/go/dataflow v0.16.0/go.mod h1:BWhSrIGmsMfuYj3J+nJ2Tw7tplRR6r28kvRiqCD3WlQ= +cloud.google.com/go/dataform v1.2.0 h1:eAIsWhr6AbUIX+wOdfctO6hAr9UoQnTR2qQW739XH1I= +cloud.google.com/go/dataform v1.2.0/go.mod h1:Lhkjd6L04/nBqsEo7S9Tx7D+Vm0pDDDZuKczewAJuX0= +cloud.google.com/go/datafusion v1.14.0 h1:56rjOW8xFBnTOLgTEWdU09KhVC2/X+y/ukzux7zYqBE= +cloud.google.com/go/datafusion v1.14.0/go.mod h1:2z+uDUKkLPacNNos5lW1Jf1IRDoFyeE+glJ4hmxF2Uc= +cloud.google.com/go/datalabeling v0.15.0 h1:9TL2kwlO/ODD1fne3uvgTLa53SttwrQi92Bn7KWnKhg= +cloud.google.com/go/datalabeling v0.15.0/go.mod h1:H8WSRKD9XYCDXDlZE3bPgvV7UYI0F05e+ufKev2AFc8= +cloud.google.com/go/dataplex v1.36.0 h1:r4O6RHEkwHr41ItsTUym0ZgH1iDf48R+W8fI7KoMNl8= +cloud.google.com/go/dataplex v1.36.0/go.mod h1:ftgNMXBt+wJ4wPVNvYJ3UY3VTZtKS/i/uFEQppaEbKk= +cloud.google.com/go/dataproc/v2 v2.25.0 h1:IF1wxkXvcgoTP92NVqX6QNDf2SdKNZFPa2IWeSKP310= +cloud.google.com/go/dataproc/v2 v2.25.0/go.mod h1:hkiM6kzc8CwLGoquMN1oghyhuI1fE0girmChH4h9W7w= +cloud.google.com/go/dataqna v0.13.0 h1:VR1y/NuN+RvPkmfmlowbmk60BLINR4MgZAusVFIGQjU= +cloud.google.com/go/dataqna v0.13.0/go.mod h1:XiVVFTOEJLBSvm3ILbyjXngGQYpjb/66MSksqz/56fs= +cloud.google.com/go/datastore v1.25.0 h1:zUjMnCLCcRZVDSdQIXsbnNCl1SVRNw5Jm0J77gPaPKs= +cloud.google.com/go/datastore v1.25.0/go.mod h1:jvJVNe+S2nHVIndV1H/B4s9K3MLsTMqOKlxSrzHTxB4= +cloud.google.com/go/datastream v1.21.0 h1:hrWGASBj85xs6wpnx9ifdSBYiRLhKDgQrA1HjeNYBhc= +cloud.google.com/go/datastream v1.21.0/go.mod h1:z9AlkQGdXqkeyO5HE+D6sYbOkLJYB4BCZpXFPX/1Vpo= +cloud.google.com/go/deploy v1.33.0 h1:2PdZ8kbIztLVRMrIN/wHbalZeZZikQlxBptBFK+eEtM= +cloud.google.com/go/deploy v1.33.0/go.mod h1:QdF3plD8D5gV2RmkTXBB6cHrq490WlpFr1SChdOJO2Y= +cloud.google.com/go/dialogflow v1.84.0 h1:BymJ6nPotDcnphK2/D0VHMAfakjJlIV5GqxLHQg/Tos= +cloud.google.com/go/dialogflow v1.84.0/go.mod h1:OU8Lj1aw5Vr2hl9ifW+vsKnc2b4iJH+41U7nZ4whg3U= +cloud.google.com/go/dlp v1.34.0 h1:SI7KSLOtAfhEAw3af8NxpKHGHbJ9BkoM7805TMaU6m4= +cloud.google.com/go/dlp v1.34.0/go.mod h1:+haQd/n0QTv5BK7wZnCk2qctd5sfKL50jjh9E6N0d/Q= +cloud.google.com/go/documentai v1.49.0 h1:WLZOaWvK4NH7xMLxANsDgIdXugeqmVyYRCbVN1m87DQ= +cloud.google.com/go/documentai v1.49.0/go.mod h1:VyQA+SxPnCPlVLSJ5UcFx+LQm8JCzK7uUXdkOaAHvG8= +cloud.google.com/go/domains v0.16.0 h1:h+lcYPxyEulj5SYuH3+OwOsLZb4DTW11QX6nlxxXQ1I= +cloud.google.com/go/domains v0.16.0/go.mod h1:O5AhaEyUAgZC2X4M10nSu3dQt2cJLtbjhtrNrdeSPF8= +cloud.google.com/go/edgecontainer v1.10.0 h1:4WOjcIZRCB4ynxH9Zs1tteIcm2LD58VUCLLmIwLYUAc= +cloud.google.com/go/edgecontainer v1.10.0/go.mod h1:g4xb11IzVWa9peXNTlnNguKP8uJVvMK4zZeDlGS2Wus= +cloud.google.com/go/errorreporting v0.9.0 h1:LlE2SVIbz0k+OSeNTksk34inr3Fy62JMhHUvNaS8f7c= +cloud.google.com/go/errorreporting v0.9.0/go.mod h1:V7ojx7z76JITDZNGyDNkIIa9nNEkQzF6Yj+VHl2YF84= +cloud.google.com/go/essentialcontacts v1.12.0 h1:+AtGn+hYLdr62sMqP0ZGMRzp9A4t+MSWe8eZoD/ho+M= +cloud.google.com/go/essentialcontacts v1.12.0/go.mod h1:W8fTL17jP6vmsPHQaCT5rOjWGohEssuqDUroxnjST0A= +cloud.google.com/go/eventarc v1.25.0 h1:+XAJNEmxJIPZLMtzLYdUZN61Kmqx62hWLry4AsWaTwo= +cloud.google.com/go/eventarc v1.25.0/go.mod h1:ncY2NKHKiX+sUjIfxVozrivvmJQ4HWo2znxms7AxlP8= +cloud.google.com/go/filestore v1.16.0 h1:2GXy5jvq2oUX6HnCpx3GjzSqlcMz8K9gkHaF9HPcYNE= +cloud.google.com/go/filestore v1.16.0/go.mod h1:szr35omqptDEuXgBbJ8PdVdYM3lf/Md96kNufWr1tVs= +cloud.google.com/go/firestore v1.24.0 h1:x0Z3hrgjYgo2wI9whuBRQcNc2hYwzZDQy/7pkUXbXcs= +cloud.google.com/go/firestore v1.24.0/go.mod h1:5aojyjN4olKUnBZDCRWwM+NsdrrCX3t1qfyERZGOonM= +cloud.google.com/go/functions v1.25.0 h1:ndUtLkam3XF9b0t2zVACH9D/EgFBISbVuQFqRz/X58k= +cloud.google.com/go/functions v1.25.0/go.mod h1:b/tqakoKeAkj9RspEjqswWf5299Lkz9C/742QUD3OEk= +cloud.google.com/go/gkebackup v1.14.0 h1:li3BtGRis1QYrkLo8+Iq2wf5WbP9v3sz9VoUw8WqgaA= +cloud.google.com/go/gkebackup v1.14.0/go.mod h1:kaD4l/s0ONcb3L9iHC8PzG1XkC5ggPwA/KAl6yAyQGs= +cloud.google.com/go/gkeconnect v1.0.0 h1:IDuAD/w0Xph8k/31vdt6pBkDdRx55lVXz7gW8ABOfCs= +cloud.google.com/go/gkeconnect v1.0.0/go.mod h1:5iWSBQzMIRLwUHUWVhxxcNK45ZPE8ntyBgE0MkavlqQ= +cloud.google.com/go/gkehub v0.22.0 h1:gHKPoWQuWpd9dXLoC58dxwBvSYTNc5/fdYYcs3JUn1s= +cloud.google.com/go/gkehub v0.22.0/go.mod h1:WiXX1w9ZHwKZVUDwL//YQfjfWS7yE0I/ym3smZn9iwE= +cloud.google.com/go/gkemulticloud v1.12.0 h1:3LPj8ro7bPdZ0BlQJwby25G32rRezGLRdhNM6U3AxGs= +cloud.google.com/go/gkemulticloud v1.12.0/go.mod h1:vLNCxGah7pPIoNSX4Yx+hb8klqA0lzzXTWBSut9KzRo= +cloud.google.com/go/gsuiteaddons v1.12.0 h1:kz9DyBk84wZCda2Rdha1MOIf7/9x6R+N+LuH9B3zYFs= +cloud.google.com/go/gsuiteaddons v1.12.0/go.mod h1:rm/XT7wmwOFGn7jmWtVV65QmZCakzTbHLSojIC4Hskg= +cloud.google.com/go/iap v1.17.0 h1:BDAVJy+juq7cMRumIx9toc4pt1K7zXoZdAI3lDD6D3g= +cloud.google.com/go/iap v1.17.0/go.mod h1:b+r+yjrss2WmAEzNrQQjlEdD5E9B8c47mOF7XnqT+z0= +cloud.google.com/go/ids v1.11.0 h1:394LkKIavhv/+oQVMtbXe9WQqCeY9wSFcHktqC9ZkLE= +cloud.google.com/go/ids v1.11.0/go.mod h1:+drdvU0pQ4x5uYiWCv364VOeIpTN/PETBrdR51D4Tjk= +cloud.google.com/go/iot v1.13.0 h1:pyt1EuMFpbV/2BlmYlXr+HBnUNSPKlP9dVK/dpFTu1U= +cloud.google.com/go/iot v1.13.0/go.mod h1:62W4n2fe/Ct66NWJEfCB5suZ3XsL5Atx+MxFjScr+9s= +cloud.google.com/go/kms v1.32.0 h1:s+rEluaaZKhLVjrIWG7uNBsnWbiitElzNzFGyp6+nIg= +cloud.google.com/go/kms v1.32.0/go.mod h1:CSGvW6GnMQbY+1nOHcIzhMtHSbExXlOmCKjWtYVjcpA= +cloud.google.com/go/language v1.18.0 h1:q58bL7rmxvw6Q6VHt+wjFsAE1Tj/JkuAEIR4+84rx9U= +cloud.google.com/go/language v1.18.0/go.mod h1:xSeiVB4UiA9wYmFy2GWjf1Mb1K3uR1Yi/80qoqTxH04= +cloud.google.com/go/lifesciences v0.16.0 h1:T+7N3oVHK8a7BhX6HJLrBW8xEbMV9iY8Mb+aOl7r6+A= +cloud.google.com/go/lifesciences v0.16.0/go.mod h1:axEwGa3A63+vCXIis+0Zkseu8KecqtNoSn7x0zyjJfM= +cloud.google.com/go/managedidentities v1.13.0 h1:ZzWkg3LSrIi9OdzCK3bxutexvw1dDDg9+tOMitbkOXw= +cloud.google.com/go/managedidentities v1.13.0/go.mod h1:lUYH5r6QEJTHqjgga0WFeiieqJ0iRwEuQSk20O41Vj0= +cloud.google.com/go/maps v1.37.0 h1:YoTRWohpNKeKxdQynorwHH69NZuUM06divZtqGPYjUA= +cloud.google.com/go/maps v1.37.0/go.mod h1:oalKFBmf2eHmdr3OvfEiiBlOakNlVitYYEPcM3TTUB4= +cloud.google.com/go/mediatranslation v0.13.0 h1:qqMRAK0mhc9M+lM6pb791oh3DYUbwLcIy58jOyXtadE= +cloud.google.com/go/mediatranslation v0.13.0/go.mod h1:kjZrowuigFr+Bf1HM1TCtp1a3E3kfG1ovPK5VEuaNAQ= +cloud.google.com/go/memcache v1.17.0 h1:l/co1jsmVLkNyl8uMqj1u1HIwd3AhCvGdHbEAIPb6jc= +cloud.google.com/go/memcache v1.17.0/go.mod h1:QQpFWgJvrFaQ6DgmitHejdbkLg8SJfHg5BzltKEWSt0= +cloud.google.com/go/metastore v1.20.0 h1:kz0XBkP531MB1IR9jTY6/DB33UFP5Df1vFpPNgW1iPk= +cloud.google.com/go/metastore v1.20.0/go.mod h1:/bhZoizjM5iOrqWJeAFDw7c16C783wEftqofnJgKKYI= +cloud.google.com/go/networkconnectivity v1.27.0 h1:ieYJjbUn2L9ZSzEB9+qKgtoD3/l1/qAnId/Z6KE4ZBo= +cloud.google.com/go/networkconnectivity v1.27.0/go.mod h1:pCnczH2W/cnLSlnsnN+VzBoXlM81ZoUGuuacFBGThyw= +cloud.google.com/go/networkmanagement v1.30.0 h1:LqYm91vSQk3NM6fHjthseRh4xrDvhXIWQBsQY/Ksus8= +cloud.google.com/go/networkmanagement v1.30.0/go.mod h1:3SBf5T7jyGzw5jqJWE7TUDRhIl2E029jggbeoFEgt5E= +cloud.google.com/go/networksecurity v0.19.0 h1:PzJnbVd0sS2+xDR9hnk31NAyqFt9qB8ohLn7XIsKIBA= +cloud.google.com/go/networksecurity v0.19.0/go.mod h1:VWDFX+stDgzZYDsCX1Wy/JO9Tlw7g/V1UHbiORVgqq0= +cloud.google.com/go/notebooks v1.18.0 h1:BgxdQcoRSSjiyszWNOTu3fmXG/6Ks4W2sHgjx0BAke0= +cloud.google.com/go/notebooks v1.18.0/go.mod h1:fXU6A3TJ2YobFy6fxOr4tKZZ8QgTjdJAqDIykOB85Gk= +cloud.google.com/go/optimization v1.12.0 h1:Cd4DgujNhEMcjfaVLGqbjfDwVDnmS1Bc93cx9TE8ZYw= +cloud.google.com/go/optimization v1.12.0/go.mod h1:28gzCUmeCLcT4vctGEo71QF4b60TYkKQo5y8Gs2KPq8= +cloud.google.com/go/orchestration v1.17.0 h1:dqf2HzqUi4CaaDiNfv6jVoZjz9lF0HWjyIEgccxPHMk= +cloud.google.com/go/orchestration v1.17.0/go.mod h1:Lf/Czqh4Jfy3IFpvDkKWjfjkYFI+tj6nAjq5ihivrq4= +cloud.google.com/go/orgpolicy v1.20.0 h1:kpVcE/OsC5aAzHCsAiuQSg3+s6ILzgPTuPZyS7n7ejA= +cloud.google.com/go/orgpolicy v1.20.0/go.mod h1:9LHqEGx5P5dhansdKTNIEXpM+QbebAIOs66+HUID4aQ= +cloud.google.com/go/osconfig v1.22.0 h1:r5lzneR9GNixJ96lZ6oIfJK9Cd5sKpnzZJe1LcGZf4w= +cloud.google.com/go/osconfig v1.22.0/go.mod h1:bUL0FaSR2ahPcFRRYnd6a0LyUzsQYIdUpBq8Tmxg8fE= +cloud.google.com/go/oslogin v1.18.0 h1:OkccORMcY2XEHwN5+AP0XH/SCUNBl39GGjaQuSCVcIw= +cloud.google.com/go/oslogin v1.18.0/go.mod h1:3Oa36T3781Mv+yCSVYlfasi7auHjfPFqvNOd1q92umc= +cloud.google.com/go/phishingprotection v0.13.0 h1:6WJF1z3Ie8fZAiCRpeSGB81h6s86XWS2SBC9iQUZxX8= +cloud.google.com/go/phishingprotection v0.13.0/go.mod h1:2gyYqwNjePPEocXDkDve3EuJPaRqN/E7fp28K3arR0k= +cloud.google.com/go/policytroubleshooter v1.16.0 h1:V3CcqzY6FhY2c2idRGZv0Ef4mFinLnXnfDFtAhxyTfM= +cloud.google.com/go/policytroubleshooter v1.16.0/go.mod h1:FZg3IW3exF6wc9eO/iBYijsGqiiCzc9mjZhsxgATXYA= +cloud.google.com/go/privatecatalog v0.16.0 h1:SBhcVNVhgEGT1edcfSFY/KSiy2xSQTSQUpY12f10rmw= +cloud.google.com/go/privatecatalog v0.16.0/go.mod h1:Dq1bSHRRaDqFr7Rb7UntXVjh1reeY6YdzYicL0EPTrM= +cloud.google.com/go/pubsub v1.51.0 h1:XOaCejsqX7EEtUdQz+WPag66wWsUUGliyCOfGPKfo90= +cloud.google.com/go/pubsub v1.51.0/go.mod h1:NERXf11sd82UV3VnflcUj8POIyQUXT/QwrKlxD8di/I= +cloud.google.com/go/pubsub/v2 v2.6.0 h1:8pjR0id+GTB+krKx5G6AGJoYrHog58w2Q89PCOrfM64= +cloud.google.com/go/pubsub/v2 v2.6.0/go.mod h1:4anqvV/w8Pcgu2tO0qr2XgsF3GXHowzryfQ5gOnVmWY= +cloud.google.com/go/pubsublite v1.10.0 h1:kqNk9e0gt0/Eu6lChjDlgu4B8k/399FZFpAU4LNq4Co= +cloud.google.com/go/pubsublite v1.10.0/go.mod h1:o9NVNBY4m8LubZqRCJtBdxpjP8DAsYizsxC6Z1vI7Dk= +cloud.google.com/go/recaptchaenterprise/v2 v2.26.0 h1:9qDHoQtUIZ8FDN5uFAfnNz8jyyXdpAJf6rMN4ZMmMcU= +cloud.google.com/go/recaptchaenterprise/v2 v2.26.0/go.mod h1:+ntF70/j7qBa6G/pwmYA0mkBcDeTCXV6WDqUL7GObfs= +cloud.google.com/go/recommendationengine v0.15.0 h1:G0tmQXp67YDaduvde7AX+cIdTtDCo2PBMeqv8SBOpnw= +cloud.google.com/go/recommendationengine v0.15.0/go.mod h1:Yx45rCF3A5fLSeXxSkXOCTXSBDBogrQnR7kUTJHwYxw= +cloud.google.com/go/recommender v1.19.0 h1:fJ6oO/7Ta/yzfRHcuJUln9iCeo6FDb5yIi2L7eLldzs= +cloud.google.com/go/recommender v1.19.0/go.mod h1:LRh+1HJjLx2kDE3S65AIlG/lvwA0llEFWYPD/QtgoaU= +cloud.google.com/go/redis v1.24.0 h1:lV6xFUF7t3fm2ZRFjB4IVNkznG9EVwetYVJYwzmW8pc= +cloud.google.com/go/redis v1.24.0/go.mod h1:ebtw9WLFKswecHO2ifNykuteNJNwoPqMCHz4UI11kF4= +cloud.google.com/go/resourcemanager v1.16.0 h1:aDf2RuuQ1Z+kbHHDzHyJNMHPGRFK+cMYfZIhWRaeRX0= +cloud.google.com/go/resourcemanager v1.16.0/go.mod h1:Hn4HPkLRnTuiUhFEFJg736Brt7BwlS84xYU06sc3STc= +cloud.google.com/go/resourcesettings v1.8.3 h1:13HOFU7v4cEvIHXSAQbinF4wp2Baybbq7q9FMctg1Ek= +cloud.google.com/go/resourcesettings v1.8.3/go.mod h1:BzgfXFHIWOOmHe6ZV9+r3OWfpHJgnqXy8jqwx4zTMLw= +cloud.google.com/go/retail v1.32.0 h1:u27GLYlnNsQrqzcvSlpZK0L6Ig8k3PEuXGkUdNFCOn4= +cloud.google.com/go/retail v1.32.0/go.mod h1:t9w9mBarD59BnFHTST2LoiCP5608ZlEfniHLgA6OoH0= +cloud.google.com/go/run v1.22.0 h1:U56fxJWdrT+yjo4S/Vrtw5m69NdNL11Cyv9jX2JOi1s= +cloud.google.com/go/run v1.22.0/go.mod h1:Wo0aTNrqfftGmbxPPraeOxSUDUZ2c7IVNg2dk8Qm1Bs= +cloud.google.com/go/scheduler v1.16.0 h1:EPdChptxnvCasdMixuu58247qhKMO1iAlrVaQhvuRyE= +cloud.google.com/go/scheduler v1.16.0/go.mod h1:0hsZg0MZJADyke1lutI0FHAYJR8Dtm8oIivXkmpACkA= +cloud.google.com/go/secretmanager v1.20.0 h1:GjE3NoyFXo7ipRPy26PMmg4oRX1Ra8fswH45r16rWV0= +cloud.google.com/go/secretmanager v1.20.0/go.mod h1:9OmSuOeiiUicANglrbdKWSnT3gYkRcXuUQDk7dDW0zU= +cloud.google.com/go/security v1.26.0 h1:xp/htSPHpyqo225Ju6PjNS9jxfcOEQMxnF229Oze62g= +cloud.google.com/go/security v1.26.0/go.mod h1:nd0i5OHXtJduMt0n6UnEojy7fiTfnfj/PSDeD7LAD+c= +cloud.google.com/go/securitycenter v1.45.0 h1:k5uzLtTSxnh4fwG+8nIcuwDjEE3J+6xe2wcZ72cys7s= +cloud.google.com/go/securitycenter v1.45.0/go.mod h1:7mAlzsCsKlEVmciAFORl431laDGpoKGFkSQndAzFs30= +cloud.google.com/go/servicedirectory v1.17.0 h1:yrohWkwM8t5JfEFCmmlyksKnpMpvxM8XbRpwg+yIo64= +cloud.google.com/go/servicedirectory v1.17.0/go.mod h1:CtgjXS1idj3s9Q6tB68021Rzk8Q6decV6+ldXC1BoBk= +cloud.google.com/go/shell v1.13.0 h1:vJ/g4BXCwBRMmUxHoNjdbE0DRh5Aysw/+vrZ/aPl0yc= +cloud.google.com/go/shell v1.13.0/go.mod h1:9WWf3xHQUElP5fL/lB9IJ/MMMnN2W/T86cBp+pXFFWo= +cloud.google.com/go/spanner v1.91.0 h1:XwXfcZ0kc1NT9Uu2IsThFiWtYptB+WgLn/KZEZcyzRg= +cloud.google.com/go/spanner v1.91.0/go.mod h1:8NB5a7qgwIhGD19Ly+vkpKffPL78vIG9RcrgsuREha0= +cloud.google.com/go/speech v1.36.0 h1:ZcmRKcY02vkF3o/Eaa9yI3i1baNFTF4ctyDZVRfnNS0= +cloud.google.com/go/speech v1.36.0/go.mod h1:tiSA8MiX49o1ngq5Ww2JFTvfjKxtAuBKY/UIH6coCPg= +cloud.google.com/go/storagetransfer v1.19.0 h1:jh+3SegMUulEhuJbkADCv8lmfRC8/pCpxja4oex2Ns8= +cloud.google.com/go/storagetransfer v1.19.0/go.mod h1:sy4ImXynHkm9CKmbILtmzLN36PHh7JOhUTpqXf5SvMs= +cloud.google.com/go/talent v1.14.0 h1:dUNSwgBUFlDEMHJpmntEvDeLkPQAoEzBrvDbNfS/UeQ= +cloud.google.com/go/talent v1.14.0/go.mod h1:jieYQngp1YqRtqV2t92w3LTrjuLV05kMM4BZMUUneaw= +cloud.google.com/go/texttospeech v1.22.0 h1:xGqQv2LB4nttaBNMepCm19EQ0Sk0YdjzPvklDU+LYsI= +cloud.google.com/go/texttospeech v1.22.0/go.mod h1:bAksATiWPKaw8r8wVgANa4GkVdsyFE4y9ulRzKyuJec= +cloud.google.com/go/tpu v1.14.0 h1:u2f8Lhezy6piu0BB3+4pTLw9xyOeU5n8PCgWjbjCitk= +cloud.google.com/go/tpu v1.14.0/go.mod h1:1pggTTG5npfxea6vYjyl60Fg09VgbM7efBgVjnFZjpo= +cloud.google.com/go/translate v1.18.0 h1:2dxWfGMG7y5vobxD2pPyngnfPPIk4MYdNB9QK5sEbuc= +cloud.google.com/go/translate v1.18.0/go.mod h1:aRVIE+P+7fngk8HwwFAgis5QA7wphGpKrFpdNoWtGCM= +cloud.google.com/go/video v1.33.0 h1:zPA41kNscRrJeRo/xD73TjNCFAnRDnP9ViNPuWARVeg= +cloud.google.com/go/video v1.33.0/go.mod h1:hEx8TNpQT6kdjMVsywePvT8BCb63Ee3F/R0GRa9wnzo= +cloud.google.com/go/videointelligence v1.17.0 h1:cd+s4jLMS59xiYLT6eQ6PVKP8R0PskgifXDa7YwMUUU= +cloud.google.com/go/videointelligence v1.17.0/go.mod h1:Phxz7AQpvXoOvz+KrrOZEJRo4CDgYXMDVDqhCtdF1jc= +cloud.google.com/go/vision/v2 v2.15.0 h1:aTR1vj4++WtS9HD6YdGuoaYygMTJ873WaoV9sYjlQCc= +cloud.google.com/go/vision/v2 v2.15.0/go.mod h1:DUdjdFkXqPvEoPC4WDYFvYCn0LlAZ4vVz29A0bXvW90= +cloud.google.com/go/vmmigration v1.16.0 h1:OPGMxx73owRHMve5B4L6W+0KQUwlunyE9kfbYzFHqug= +cloud.google.com/go/vmmigration v1.16.0/go.mod h1:ILrSjXnHMpdamkkAU8fjMKKMsH27B6FLC5kv/6TkLy0= +cloud.google.com/go/vmwareengine v1.9.0 h1:vdSutihjzH5iYEFlN+1x+gDm4xzj6Md2yMzXADKI5cQ= +cloud.google.com/go/vmwareengine v1.9.0/go.mod h1:zXXuUaIpvDhsV6sR+JdQfcQ4V5+pDarrp7FW7nOdS2I= +cloud.google.com/go/vpcaccess v1.14.0 h1:fFa1i67AjC/3MuG5GPlvx36sIMeffpsIs3kI1C7S3Jk= +cloud.google.com/go/vpcaccess v1.14.0/go.mod h1:MxbVgr+2fpIFIEIdSmgnb8ykNWRPVtslpmWijp7an68= +cloud.google.com/go/webrisk v1.17.0 h1:t2WMlBo3xXxH6k5kTSkyIYUl24AHgmBI76F/5OqAPRY= +cloud.google.com/go/webrisk v1.17.0/go.mod h1:ypwCZ+G/SXyUZ+x3ppxn1hu+6tDifGNd/OpwPtCdJHI= +cloud.google.com/go/websecurityscanner v1.12.0 h1:iV+hAXeEo8kKtFWDZWPp9Z0fsziZnuOt4nCCYp68RP0= +cloud.google.com/go/websecurityscanner v1.12.0/go.mod h1:cZSc9HqoFdccL1mqZtPIInOd4R8PBGwI20wdnrz6AO8= +cloud.google.com/go/workflows v1.20.0 h1:qROpn1zRDdeIjUP9w9O3MHXJEg8v/pzVMicAcw9vLvQ= +cloud.google.com/go/workflows v1.20.0/go.mod h1:TC9yx7VpjGdBBeKM8FG2EMtms5Q9nyTqI+2uV9bDNs4= +github.com/ThalesGroup/crypto11 v1.5.0 h1:fV+gZtXl36t19Xw7bbbpWRsEbzLB9Qxjk/YQLTRk0YQ= +github.com/ThalesGroup/crypto11 v1.5.0/go.mod h1:sHbXFYNbNLe231R/gmWlE4MXh8dn8n0EqfD+harPBLA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/checkpoint-restore/go-criu/v6 v6.3.0 h1:mIdrSO2cPNWQY1truPg6uHLXyKHk3Z5Odx4wjKOASzA= +github.com/checkpoint-restore/go-criu/v6 v6.3.0/go.mod h1:rrRTN/uSwY2X+BPRl/gkulo9gsKOSAeVp9/K2tv7xZI= +github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= +github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg= +github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= +github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/qqsc= +github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= +github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= +github.com/cyphar/filepath-securejoin v0.3.5 h1:L81NHjquoQmcPgXcttUS9qTSR/+bXry6pbSINQGpjj4= +github.com/cyphar/filepath-securejoin v0.3.5/go.mod h1:edhVd3c6OXKjUmSrVa/tGJRS9joFTxlslFCAyaxigkE= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/frostbyte73/core v0.1.1 h1:ChhJOR7bAKOCPbA+lqDLE2cGKlCG5JXsDvvQr4YaJIA= +github.com/frostbyte73/core v0.1.1/go.mod h1:mhfOtR+xWAvwXiwor7jnqPMnu4fxbv1F2MwZ0BEpzZo= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/gammazero/deque v1.2.1 h1:9fnQVFCCZ9/NOc7ccTNqzoKd1tCWOqeI05/lPqFPMGQ= +github.com/gammazero/deque v1.2.1/go.mod h1:5nSFkzVm+afG9+gy0VIowlqVAW4N8zNcMne+CMQVD2g= +github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY= +github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= +github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-pkcs11 v0.3.0 h1:PVRnTgtArZ3QQqTGtbtjtnIkzl2iY2kt24yqbrf7td8= +github.com/google/go-pkcs11 v0.3.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= +github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA= +github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/hamba/avro/v2 v2.31.0 h1:wv3nmua7lCEIwWsb6vqsTS3pXktTxcKg5eoyNu0VhrU= +github.com/hamba/avro/v2 v2.31.0/go.mod h1:t6lJYAGE5Mswfn17zjtyQsssRQgnqO6TXLBCHHWRqrw= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= +github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= +github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4= +github.com/lithammer/fuzzysearch v1.1.8/go.mod h1:IdqeyBClc3FFqSzYq/MXESsS4S0FsZ5ajtkr5xPLts4= +github.com/lithammer/shortuuid/v4 v4.3.0 h1:XRr80OYPOlqxgnCv2/iuElkl/ZtXtPtLKk2AvpXGdWA= +github.com/lithammer/shortuuid/v4 v4.3.0/go.mod h1:D5noHZ2oFw/YaKCfGy0YxyE7M0wMbezmMjPdhyEFe6Y= +github.com/lyft/protoc-gen-star/v2 v2.0.4 h1:JDlNKttNIRd68AAIychs0AqEpO8/I/WYi01OQ7Raw6Q= +github.com/lyft/protoc-gen-star/v2 v2.0.4/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= +github.com/mackerelio/go-osstat v0.2.8 h1:I2duicTaCGWoM53XwAwA9OIe1inu0xnVs8/pqOWWVr4= +github.com/mackerelio/go-osstat v0.2.8/go.mod h1:SyS3XxKdoSKJnTGTkN5Yrh6VUQVuAURACfE6y+2DN4k= +github.com/magefile/mage v1.17.2 h1:fyXVu1eadI8Ap1HCCNgEhJ5McIWiYhLR8uol64ZZc40= +github.com/magefile/mage v1.17.2/go.mod h1:Yj51kqllmsgFpvvSzgrZPK9WtluG3kUhFaBUVLo4feA= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.20 h1:WcT52H91ZUAwy8+HUkdM3THM6gXqXuLJi9O3rjcQQaQ= +github.com/mattn/go-runewidth v0.0.20/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/maxbrunsfeld/counterfeiter/v6 v6.12.2 h1:V23nK2R2B63g2GhygF9zVGpnigmhvoZoH8d0hrZwMGY= +github.com/maxbrunsfeld/counterfeiter/v6 v6.12.2/go.mod h1:Mr897yU9FmyKaQDPtRlVKibrjz40XXyOHUfyZBPSyZU= +github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= +github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= +github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mrunalp/fileutils v0.5.1 h1:F+S7ZlNKnrwHfSwdlgNSkKo67ReVf8o9fel6C3dkm/Q= +github.com/mrunalp/fileutils v0.5.1/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/nyaruka/phonenumbers v1.8.1 h1:2K9YMQuv1dCGqjjzB1DwmdCe89khT4KPBQb2CxAMMlU= +github.com/nyaruka/phonenumbers v1.8.1/go.mod h1:fsKPJ70O9JetEA4ggnJadYTFWwtGPvu/lETTXNXq6Cs= +github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= +github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= +github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/petermattis/goid v0.0.0-20260820044319-269ab09b5261 h1:lcWAnrqr2nNfDiArwFNHCE4787Mw2tCdVSOXCru0/0E= +github.com/petermattis/goid v0.0.0-20260820044319-269ab09b5261/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/pion/datachannel v1.6.2 h1:7EXQ8TH3vTouBUdRWYbcX2edSx9Yj6k5zl5P+qyxEPc= +github.com/pion/datachannel v1.6.2/go.mod h1:pzbdAZvyGtXbcHM1hBbsFaOTf40lZizU/dNlvVOak6E= +github.com/pion/dtls/v3 v3.1.5 h1:9xJtVsHwMYeSjPp5Hh1FTis4DchnQWtnOa5o+6ygqfc= +github.com/pion/dtls/v3 v3.1.5/go.mod h1:gz1K4jg6c+fq86oQMH4pilpCEOEPwmEr2jY+VcF/mkU= +github.com/pion/ice/v4 v4.4.0 h1:wvHDDqimaC38Y7MVpD46Y63p246ChvXd87VKoLYS5b4= +github.com/pion/ice/v4 v4.4.0/go.mod h1:obAyD+J+Hzs7QA7Y8YXHp5uIn6gb7z87pKedXZkrcFU= +github.com/pion/interceptor v0.1.47 h1:yw8t5pJ2f8t78NgU+8EmxhaqYLXS7uFCC/tAGOaSDBo= +github.com/pion/interceptor v0.1.47/go.mod h1:7yoRBzaIDETPC6cIN8Zj9EyGqHv1ImOpcTFPha6MuOM= +github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= +github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= +github.com/pion/mdns/v2 v2.1.0 h1:3IJ9+Xio6tWYjhN6WwuY142P/1jA0D5ERaIqawg/fOY= +github.com/pion/mdns/v2 v2.1.0/go.mod h1:pcez23GdynwcfRU1977qKU0mDxSeucttSHbCSfFOd9A= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtcp v1.2.17 h1:PxiT6L79yPZKtXIsXdG1eakBl6dtBj4x+4oVEL0DlSw= +github.com/pion/rtcp v1.2.17/go.mod h1:7kBpuBJaWwax4hzc/pgexY8vkOpvh8atgYDbaKZq0iU= +github.com/pion/rtp v1.10.5 h1:ip0HhO/wYZqQ4bKS+R99KnZh/GRCmIT0jDXikub7vlE= +github.com/pion/rtp v1.10.5/go.mod h1:Au8fc6cEByy8RLTwKTQTEeQqDB/SJDxwL4mZuxYA5Pk= +github.com/pion/sctp v1.11.1 h1:O4dIFyURw1KTST7w+gtD4gLeYXkhPa0xXLHMMoe/OSA= +github.com/pion/sctp v1.11.1/go.mod h1:7KFmTwLcoYgJs/Z+99nJvsWL0qDpuyloSI0RbAqlrz0= +github.com/pion/sdp/v3 v3.0.19 h1:1VMKs3gIkTQV5M3hNKfTAPrDXSNrYtOlmOD8+mSZUGQ= +github.com/pion/sdp/v3 v3.0.19/go.mod h1:dE5WOSlzXrtiE/iuZqe9n+AcEbOjtAd3k5m5NtlV/qU= +github.com/pion/srtp/v3 v3.0.12 h1:U7V17bckl7sI4mb3sepiojByDuBY0wNCqQE+6IlQBbc= +github.com/pion/srtp/v3 v3.0.12/go.mod h1:EeZOi/sd6glM1EXapg051gdNWO9yWT1YSsgQ4SlJkns= +github.com/pion/stun/v3 v3.1.6 h1:WnhsD0eHCiwCfKNkVx0VJJwr2Y3eV4Ueih3KJ+dfZy8= +github.com/pion/stun/v3 v3.1.6/go.mod h1:zRUghXSQU32Lx5orJsz3uYMkIihweXb3mu5gIns02fs= +github.com/pion/transport/v4 v4.0.2 h1:ifYlPqNwsy6aKQ9y8yzxXlHae5431ZrH2avkD/Rn6Tk= +github.com/pion/transport/v4 v4.0.2/go.mod h1:06hFI+jCFcok2X2MekVufNZ/uzNZXivGBPfviSVcjgM= +github.com/pion/turn/v5 v5.0.12 h1:6+b69ivQQXSlyfkp2AKripqD2k3W32qXK8QzCzpJWPI= +github.com/pion/turn/v5 v5.0.12/go.mod h1:CQACsRDJtjQ+6RSrGHrS2PCIerLwbW3uqXRqOvtjAFg= +github.com/pion/webrtc/v4 v4.2.18 h1:smA/3g6Gy4RohM0VIZ5KKY/12TQbxv3XFgpUMyb2EUI= +github.com/pion/webrtc/v4 v4.2.18/go.mod h1:vmzi6s+rvhoIuT94DPqivB+0xJXs9rG4QRD+4MgBtlY= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= +github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/pterm/pterm v0.12.83 h1:ie+YmGmA727VuhxBlyGr74Ks+7McV6kT99IB8EU80aA= +github.com/pterm/pterm v0.12.83/go.mod h1:xlgc6bFWyJIMtmLJvGim+L7jhSReilOlOnodeIYe4Tk= +github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= +github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= +github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2rWM6M9YjLpY= +github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= +github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/substrait-io/substrait v0.87.0 h1:40rP4LejyK6SNQlWz7NX6kQELf8cmScWMBGruWhN4io= +github.com/substrait-io/substrait v0.87.0/go.mod h1:MPFNw6sToJgpD5Z2rj0rQrdP/Oq8HG7Z2t3CAEHtkHw= +github.com/substrait-io/substrait-go/v8 v8.1.1 h1:XL7CqVVOQkXP9FRSIm2ylJCjqRASg1yBRcn5nRCRwss= +github.com/substrait-io/substrait-go/v8 v8.1.1/go.mod h1:6GLz9k21udB64g4lLKq8632TKfQCRAVfhuU3NSXtZWY= +github.com/substrait-io/substrait-protobuf/go v0.85.0 h1:zk6MtNWLtDSl8a7qCZRFH0+EIIXVrrd/hsgYK/SQTgM= +github.com/substrait-io/substrait-protobuf/go v0.85.0/go.mod h1:hn+Szm1NmZZc91FwWK9EXD/lmuGBSRTJ5IvHhlG1YnQ= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/thales-e-security/pool v0.0.2 h1:RAPs4q2EbWsTit6tpzuvTFlgFRJ3S8Evf5gtvVDbmPg= +github.com/thales-e-security/pool v0.0.2/go.mod h1:qtpMm2+thHtqhLzTwgDBj/OuNnMpupY8mv0Phz0gjhU= +github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= +github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= +github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= +github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 h1:HjU6IWBiAgRIdAJ9/y1rwCn+UELEmwV+VsTLzj/W4sE= +golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6/go.mod h1:Eqhaxk/wZsWEH8CRxLwj6xzEJbz7k1EFGqx7nyCoabE= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= +golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20260921155816-b14227669459 h1:YeHSHMle+uLS75+Z+FuOA7bH9wibPWVkMRWD+9Aaz7I= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20260921155816-b14227669459/go.mod h1:RoCpRfcA27uTJ0TZb3Vyad8eLVakUKdLdV2yMdlm2+w= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.2 h1:rgSNvqscFZ1JgV/4wH5GOsZFSFkR2Eua9As3KIr2LlM= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.2/go.mod h1:iMEtFwDlAhjDU9L5mY6U1XLwlIId/G3h+QcBHDIvrJ8= +google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6 h1:ExN12ndbJ608cboPYflpTny6mXSzPrDLh0iTaVrRrds= +google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6/go.mod h1:6ytKWczdvnpnO+m+JiG9NjEDzR1FJfsnmJdG7B8QVZ8= +modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= +modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= +modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= From eb0a5dc60241a040380e5c36678ebebfb7adc969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sun, 18 Oct 2026 10:31:10 -0700 Subject: [PATCH 24/29] .gitignore: drop the empty go.work section --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index 5956e82..db11270 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,6 @@ # Dependency directories (remove the comment below to include it) # vendor/ -# Go workspace file - # env file .env* From 6705390d5c631c93c0ef541f6ea73186a1ebdc96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sun, 18 Oct 2026 10:37:58 -0700 Subject: [PATCH 25/29] CI: a stale objects version prints the command that sets the next one --- .github/workflows/test.yaml | 6 +++++- README.md | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3f56768..f8e6d24 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -44,7 +44,11 @@ jobs: exit 1 fi if git rev-parse -q --verify "refs/tags/objects/$v" >/dev/null && ! git diff --quiet "refs/tags/objects/$v" HEAD -- objects; then - echo "::error::objects/ changed since objects/$v: raise github.com/livekit/storage/objects in go.mod to the next version" + # The next minor after the latest release: before v1, any change + # may break a caller. + latest=$(git tag -l 'objects/v*' --sort=-v:refname | head -1) + next=$(echo "${latest#objects/}" | awk -F. '{print $1 "." $2+1 ".0"}') + echo "::error::objects/ changed since objects/$v. Run: go mod edit -require=github.com/livekit/storage/objects@$next" exit 1 fi diff --git a/README.md b/README.md index 761fb3f..bba948a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Object storage on S3 and S3-compatible stores, GCS, Azure Blob, OCI Object Stora - `Storage` is `Bucket` plus `UploadFile` and `DownloadFile`. `DownloadFile` replaces the local file only once the whole object is read. - `PutOptions{ContentType, IfAbsent}` on every write. `IfAbsent` refuses to replace an object, atomically where the store honors the condition. - Errors: a missing object is `ErrNotFound`, a write `IfAbsent` refuses is `ErrExists` (`errors.Is`); `ErrorWithStatusCode` carries the store's HTTP status and `DeleteError` the paths a delete left (`errors.As`). -- `storage/objects` is a module of its own with no requirements, so a fake or a library can implement or accept the interfaces without inheriting any SDK or its versions. A PR that changes `objects/` raises the `objects` version in `go.mod` (CI says so when it is missing); `go.work` builds the repo against `objects/` until then, and the version is tagged `objects/vX.Y.Z` on main when the PR merges. +- `storage/objects` is a module of its own with no requirements, so a fake or a library can implement or accept the interfaces without inheriting any SDK or its versions. A PR that changes `objects/` raises the `objects` version in `go.mod`; CI fails without it and prints the command. Nobody tags: the version is tagged `objects/vX.Y.Z` on main when the PR merges, and `go.work` builds the repo against `objects/` until then. ## Behavior From d35ec08277740d5dbc438503ce2ad0d9d6993f6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sun, 18 Oct 2026 10:39:50 -0700 Subject: [PATCH 26/29] objects: the next version is committed to a PR that changes it; README says why objects is a module objects-version.yaml raises the objects version in go.mod when a PR changes objects/ under a released one, then starts the checks for the new head, so whoever changes the interfaces does nothing more. A fork still gets the command from the test workflow. --- .github/workflows/objects-version.yaml | 53 ++++++++++++++++++++++++++ README.md | 21 +++++++++- e2e/go.mod | 2 +- 3 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/objects-version.yaml diff --git a/.github/workflows/objects-version.yaml b/.github/workflows/objects-version.yaml new file mode 100644 index 0000000..00f2899 --- /dev/null +++ b/.github/workflows/objects-version.yaml @@ -0,0 +1,53 @@ +name: objects version + +permissions: + contents: write + actions: write + +on: + pull_request: + paths: ["objects/**"] + +jobs: + # Raises the objects version go.mod requires when a PR changes objects/ + # under one already released, so whoever changes the interfaces does + # nothing more. A fork's branch cannot be pushed to: there, the test + # workflow's check prints the command instead. + bump: + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.head_ref }} + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: "go.mod" + cache: false + + - name: Raise the objects version + env: + GH_TOKEN: ${{ github.token }} + BRANCH: ${{ github.head_ref }} + run: | + v=$(awk '$1 == "github.com/livekit/storage/objects" {print $2}' go.mod) + # A version not yet released, or released with these interfaces, stands. + if ! git rev-parse -q --verify "refs/tags/objects/$v" >/dev/null || git diff --quiet "refs/tags/objects/$v" HEAD -- objects; then + exit 0 + fi + # The next minor after the latest release: before v1, any change may + # break a caller. + latest=$(git tag -l 'objects/v*' --sort=-v:refname | head -1) + next=$(echo "${latest#objects/}" | awk -F. '{print $1 "." $2+1 ".0"}') + go mod edit -require=github.com/livekit/storage/objects@$next + (cd e2e && go mod edit -require=github.com/livekit/storage/objects@$next) + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -am "objects: $next" + git push origin "HEAD:$BRANCH" + # A push with this token starts no workflow, so the checks for the + # new head are started here. + gh workflow run test.yaml --ref "$BRANCH" diff --git a/README.md b/README.md index bba948a..fd31849 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Object storage on S3 and S3-compatible stores, GCS, Azure Blob, OCI Object Stora - `Storage` is `Bucket` plus `UploadFile` and `DownloadFile`. `DownloadFile` replaces the local file only once the whole object is read. - `PutOptions{ContentType, IfAbsent}` on every write. `IfAbsent` refuses to replace an object, atomically where the store honors the condition. - Errors: a missing object is `ErrNotFound`, a write `IfAbsent` refuses is `ErrExists` (`errors.Is`); `ErrorWithStatusCode` carries the store's HTTP status and `DeleteError` the paths a delete left (`errors.As`). -- `storage/objects` is a module of its own with no requirements, so a fake or a library can implement or accept the interfaces without inheriting any SDK or its versions. A PR that changes `objects/` raises the `objects` version in `go.mod`; CI fails without it and prints the command. Nobody tags: the version is tagged `objects/vX.Y.Z` on main when the PR merges, and `go.work` builds the repo against `objects/` until then. +- `storage/objects` holds the interfaces in a module of its own, with no requirements (see Modules). ## Behavior @@ -17,6 +17,25 @@ Object storage on S3 and S3-compatible stores, GCS, Azure Blob, OCI Object Stora - **Proxy**: `ProxyConfig` (S3, GCS) carries its credentials for http and https targets, honors `NO_PROXY`, and never proxies the cloud metadata services. - **Local writes** go in place, as a file is written, so mounted volumes and file watchers see one write; `IfAbsent` writes a temporary file (`.upload-*`, never listed) and links it into place. +## Modules + +| module | holds | requires | +|---|---|---| +| `github.com/livekit/storage` | the backends, `New`, the configs | every provider SDK, `livekit/protocol` | +| `github.com/livekit/storage/objects` | `Bucket`, `Storage`, `PutOptions`, the errors | nothing | +| `github.com/livekit/storage/e2e` | the end-to-end suites | `dockertest` | + +Why `objects` is its own module: + +- A module's requirements take part in the version selection of every module that requires it, whichever packages it imports. In the root module, the interfaces would carry every SDK's and `livekit/protocol`'s versions (grpc, otel) into each fake or library that implements or accepts them. +- With no requirements, implementing them costs no dependency: gosim's `fakes/objstore` `Bucket` is an `objects.Bucket`. + +How its versions work: + +- `go.mod` names the `objects` version the root requires; `go.work` builds the repo against `objects/` in the tree. +- A PR that changes `objects/` under a released version gets the next minor version committed to it by `objects-version.yaml`. From a fork, CI fails and prints the command. +- On merge, `objects-release.yaml` tags `objects/vX.Y.Z` on main. Squash merges drop a branch's commits, so only a version tagged on main stays resolvable. + ## Tests The suites live in the `e2e` module, so this module's users do not inherit Docker's dependencies. diff --git a/e2e/go.mod b/e2e/go.mod index 8ee21cb..8e25098 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -79,7 +79,7 @@ require ( github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/livekit/protocol v1.52.1 // indirect - github.com/livekit/storage/objects v0.0.0-20261018042631-42ef441ca5be // indirect + github.com/livekit/storage/objects v0.1.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/user v0.3.0 // indirect github.com/moby/term v0.5.0 // indirect From 0a8d10265cfd2dabd6338592fa0e03fa8474da07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sun, 18 Oct 2026 10:51:51 -0700 Subject: [PATCH 27/29] objects: the bump pushes as the writer app, release reruns by hand, notes from the audit objects-version.yaml pushes with the LiveKit writer app's token, as a GITHUB_TOKEN push leaves the PR's checks waiting for approval, and commits as the app so the change is attributed; it runs once per PR at a time. objects-release.yaml can be rerun by hand and runs one at a time. go.mod lists objects as a direct requirement, go.work.sum is regenerated, and the README says what a bump branch needs and that test must be required with branches up to date. --- .github/workflows/objects-release.yaml | 7 + .github/workflows/objects-version.yaml | 31 +- README.md | 6 +- go.mod | 2 +- go.work.sum | 469 ------------------------- 5 files changed, 33 insertions(+), 482 deletions(-) diff --git a/.github/workflows/objects-release.yaml b/.github/workflows/objects-release.yaml index 7e811e4..b1835a6 100644 --- a/.github/workflows/objects-release.yaml +++ b/.github/workflows/objects-release.yaml @@ -6,6 +6,13 @@ permissions: on: push: branches: [main] + # A failed run is retried by hand; GitHub emails the failure to whoever + # merged. + workflow_dispatch: + +concurrency: + group: objects-release + cancel-in-progress: false jobs: # Tags the objects version go.mod requires on this commit, so main never diff --git a/.github/workflows/objects-version.yaml b/.github/workflows/objects-version.yaml index 00f2899..c82a0af 100644 --- a/.github/workflows/objects-version.yaml +++ b/.github/workflows/objects-version.yaml @@ -1,26 +1,38 @@ name: objects version permissions: - contents: write - actions: write + contents: read on: pull_request: paths: ["objects/**"] +concurrency: + group: objects-version-${{ github.event.pull_request.number }} + cancel-in-progress: false + jobs: # Raises the objects version go.mod requires when a PR changes objects/ # under one already released, so whoever changes the interfaces does - # nothing more. A fork's branch cannot be pushed to: there, the test - # workflow's check prints the command instead. + # nothing more. It pushes with the LiveKit writer app's token: a push with + # GITHUB_TOKEN would leave the PR's checks waiting for approval. A fork's + # branch cannot be pushed to: there, the test workflow's check prints the + # command instead. bump: if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest steps: + - id: app + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 + with: + app-id: ${{ vars.REPO_WRITER_GH_APP_ID }} + private-key: ${{ secrets.REPO_WRITER_GH_APP_PRIVATE_KEY }} + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.head_ref }} fetch-depth: 0 + token: ${{ steps.app.outputs.token }} - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -30,8 +42,9 @@ jobs: - name: Raise the objects version env: - GH_TOKEN: ${{ github.token }} BRANCH: ${{ github.head_ref }} + GH_TOKEN: ${{ steps.app.outputs.token }} + APP: ${{ steps.app.outputs.app-slug }} run: | v=$(awk '$1 == "github.com/livekit/storage/objects" {print $2}' go.mod) # A version not yet released, or released with these interfaces, stands. @@ -44,10 +57,8 @@ jobs: next=$(echo "${latest#objects/}" | awk -F. '{print $1 "." $2+1 ".0"}') go mod edit -require=github.com/livekit/storage/objects@$next (cd e2e && go mod edit -require=github.com/livekit/storage/objects@$next) - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + # The app's own identity, so the commit is attributed to it. + git config user.name "$APP[bot]" + git config user.email "$(gh api "/users/$APP%5Bbot%5D" --jq .id)+$APP[bot]@users.noreply.github.com" git commit -am "objects: $next" git push origin "HEAD:$BRANCH" - # A push with this token starts no workflow, so the checks for the - # new head are started here. - gh workflow run test.yaml --ref "$BRANCH" diff --git a/README.md b/README.md index fd31849..8baac6a 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,10 @@ Why `objects` is its own module: How its versions work: - `go.mod` names the `objects` version the root requires; `go.work` builds the repo against `objects/` in the tree. -- A PR that changes `objects/` under a released version gets the next minor version committed to it by `objects-version.yaml`. From a fork, CI fails and prints the command. -- On merge, `objects-release.yaml` tags `objects/vX.Y.Z` on main. Squash merges drop a branch's commits, so only a version tagged on main stays resolvable. +- A PR that changes `objects/` under a released version gets the next minor version committed to it by `objects-version.yaml`; pull that commit before pushing again. From a fork, CI fails and prints the command. +- Until the version is tagged, `go mod tidy` in the root fails on it (unknown revision); `go.work` builds the repo meanwhile. +- On merge, `objects-release.yaml` tags `objects/vX.Y.Z` on main; a failed run can be rerun by hand. Squash merges drop a branch's commits, so only a version tagged on main is sure to stay resolvable. +- `test` must be a required check on main with branches up to date: two PRs bumped to the same version cannot both merge then. ## Tests diff --git a/go.mod b/go.mod index 6ce2835..dba78e6 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/aws/smithy-go v1.28.1 github.com/googleapis/gax-go/v2 v2.26.2 github.com/livekit/protocol v1.52.1 + github.com/livekit/storage/objects v0.1.0 github.com/oracle/oci-go-sdk/v65 v65.126.1 github.com/stretchr/testify v1.12.1 golang.org/x/net v0.59.0 @@ -66,7 +67,6 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.22 // indirect github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect - github.com/livekit/storage/objects v0.1.0 github.com/pierrec/lz4/v4 v4.1.28 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/puzpuzpuz/xsync/v4 v4.5.0 // indirect diff --git a/go.work.sum b/go.work.sum index ed6e33d..410d71e 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,470 +1 @@ -atomicgo.dev/cursor v0.2.0 h1:H6XN5alUJ52FZZUkI7AlJbUc1aW38GWZalpYRPpoPOw= -atomicgo.dev/cursor v0.2.0/go.mod h1:Lr4ZJB3U7DfPPOkbH7/6TOtJ4vFGHlgj1nc+n900IpU= -atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8= -atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ= -atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs= -atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU= -cloud.google.com/go/accessapproval v1.13.0 h1:kx3RQSS0VglTngQTPfywgVj+xFgre/vJouGETBlPU1E= -cloud.google.com/go/accessapproval v1.13.0/go.mod h1:7bmInw17bQX+ZPi7YmReC3xKymDrMmxXaUnaI6zQOqI= -cloud.google.com/go/accesscontextmanager v1.15.0 h1:0baVgug9IFV8y5cIoD4iN/m/QtJzk2k+j3MW+zJWCVw= -cloud.google.com/go/accesscontextmanager v1.15.0/go.mod h1:YjW9urferk8i9ALwBF3bmdcogZeQYRn2yWwR8nkhsBc= -cloud.google.com/go/aiplatform v1.126.0 h1:5PxeWpQfkAyN8mtgtVp4H5nP+ntwPLFMjNREVdiolR0= -cloud.google.com/go/aiplatform v1.126.0/go.mod h1:iR3za3evdprLe1XL2pLu0cYVCuTbc87QG0pgvcgiJlE= -cloud.google.com/go/analytics v0.35.0 h1:GuwmzJHIaQRvtko6g4wW1ngQ7rpDvDLZ8M3iP3kiflU= -cloud.google.com/go/analytics v0.35.0/go.mod h1:V9Qef2N0y8GDqQ9FTlmM2XpDEMYonZJRPSUNGZlPCcc= -cloud.google.com/go/apigateway v1.13.0 h1:IENNmFQlMsl58RlvYHf6wxBNQFeP+lAnVNcyxtmByTE= -cloud.google.com/go/apigateway v1.13.0/go.mod h1:pvEpOuuOIw2ev9VCcOyVkDXHHL4lvgMuqIe7XjJ8JoU= -cloud.google.com/go/apigeeconnect v1.12.0 h1:YvZhi0QkHobBRMrN6ri9PUPnnhgEd4VpW0+yo1WI7r0= -cloud.google.com/go/apigeeconnect v1.12.0/go.mod h1:mYJekCKZHc2ia5yZX5lwtexTn9CzsOfb6+sh/2hi42Q= -cloud.google.com/go/apigeeregistry v1.1.0 h1:2zIogaYq7dF4E1dAGC4MUnasRuSmohsMl7wVuJ963/4= -cloud.google.com/go/apigeeregistry v1.1.0/go.mod h1:4ZFhQlxMuyfDMz9ORDSV8FPZtf2yPQkKjigsFtrrE4Y= -cloud.google.com/go/appengine v1.15.0 h1:0MiBM2KGk1WAhh489WoELQ7IeW2qw6HTE6nZIlrOw5U= -cloud.google.com/go/appengine v1.15.0/go.mod h1:/8gGZsOX5GDjOo4mAWk8IV59p2991dxTbEtKIlhDjzU= -cloud.google.com/go/area120 v0.15.0 h1:9v6HeQsBpfRqoZtNeLI4V8SQ7Jcmt6s2gmgbjl0fVFc= -cloud.google.com/go/area120 v0.15.0/go.mod h1:jD1fw9W4xxIZMY68g7PpbCPleoeGddFs5jPcdhfg3+Y= -cloud.google.com/go/artifactregistry v1.26.0 h1:iq5kkdY2FJY8RkkNghUelb3EOcqyy7dVhfDGVk/Xw9g= -cloud.google.com/go/artifactregistry v1.26.0/go.mod h1:c5FPi5GtDBP+OAr5kKhCBNQDT9ZgAyobXQjekx93VWs= -cloud.google.com/go/asset v1.28.0 h1:M6YE1exBuZQhTi8wfIKrwjKK6b2ySfHKgrqXjPZA4EM= -cloud.google.com/go/asset v1.28.0/go.mod h1:Pnvjhay8/FgodOH9uJC8OkfJfRtSnNIIU4WSxg5JfJw= -cloud.google.com/go/assuredworkloads v1.19.0 h1:XyNQ9SBX0eUkZYCZhC8dOY6ZFeSF6/OIjHn5dQs78Jo= -cloud.google.com/go/assuredworkloads v1.19.0/go.mod h1:/UGGtFCMokM3sGJ4FxjfmLvvFpPa5I/Oz68mwk4Su+0= -cloud.google.com/go/automl v1.21.0 h1:brKXr4i9AMVfgbCKOYSu4rnIoz1ILIgcIAy3nerzEGc= -cloud.google.com/go/automl v1.21.0/go.mod h1:MNbhUevuECzM3jqSOM7hmOedOdRJkm8xbbXW44SU15U= -cloud.google.com/go/baremetalsolution v1.10.0 h1:6MaZilXzGZJ3mGqbSCdDezs9yrze6kcmBABsRUZ/aYI= -cloud.google.com/go/baremetalsolution v1.10.0/go.mod h1:xhhT9VQiKPFd2fUs4oeDSRrxV0sb0PGeVmuZoUE2cBA= -cloud.google.com/go/batch v1.20.0 h1:nTL5x9HA1yVdPIdeqcbLjNnV9iYDYvMNmP6AnZ+qbHU= -cloud.google.com/go/batch v1.20.0/go.mod h1:ABT/5QqsIDsONa+n/8C7XYPjwh/kjOEPXukcRTaMsCg= -cloud.google.com/go/beyondcorp v1.8.0 h1:rsyle6zjxce2s57BpULw5u170xS+KSjFENSY7C0MZbs= -cloud.google.com/go/beyondcorp v1.8.0/go.mod h1:aVxzwamO8H4GXWQHowBAmL0KYNfYpW4E6Do2wfP0RYs= -cloud.google.com/go/bigquery v1.79.0 h1:+tW6oRuP/4dOgdl/p32on0+3OktVQtU+veOMxhjoOhE= -cloud.google.com/go/bigquery v1.79.0/go.mod h1:QTt5tgZxqqvZs3dOZKpvriGqy+CdvY9LyetirFZRPOE= -cloud.google.com/go/bigtable v1.47.0 h1:NGLgDSr/i79BTGCjxH/maPKxyvl5q8/SsBsyLK52kdI= -cloud.google.com/go/bigtable v1.47.0/go.mod h1:GUM6PdkG3rrDse9kugqvX5+ktwo3ldfLtLi1VFn5Wj4= -cloud.google.com/go/billing v1.26.0 h1:6RRjbRd6iZKZFb7/MgRvmXKq/Ism02ckkZLJazj4CQ0= -cloud.google.com/go/billing v1.26.0/go.mod h1:axqDO1uHegh7u5qngkTfqN1djAeLGsWAFAblERgmgEk= -cloud.google.com/go/binaryauthorization v1.16.0 h1:f1iNUaHWP9XPdh6grftTlIRuJSBytud4j51l1Jce94E= -cloud.google.com/go/binaryauthorization v1.16.0/go.mod h1:E+iC5Avu4pdItdzGiSGHnh6TfQrl+KmPxDDg/T/VuHs= -cloud.google.com/go/certificatemanager v1.15.0 h1:PwljEZlI3lZgoFUG0pnIBNOwrQUaAP+gS0ZahTwUZBA= -cloud.google.com/go/certificatemanager v1.15.0/go.mod h1:8dfGG2/TbUpCNqsCF/TIMOGV0OVvU6nhkZWTU4MmCXU= -cloud.google.com/go/channel v1.27.0 h1:fqDZDzVw61c4iIUe7rQ+cBYlBdSZ3yqnzw5WrZTPjqs= -cloud.google.com/go/channel v1.27.0/go.mod h1:9ekufBLXuQ6j1oyqtDSIp29qWU5EwCi8WUi9qkLn3MA= -cloud.google.com/go/cloudbuild v1.32.0 h1:tlF+KSIJJ0kaKkxACpk9htZ1euDpyOqPuO1aHR4j3oI= -cloud.google.com/go/cloudbuild v1.32.0/go.mod h1:mYgcM8CMaPmAnO7GxSQ9ADAxVRwS+1b7s6WVkt29OXY= -cloud.google.com/go/clouddms v1.14.0 h1:oaktwVyUeKTuzjpde3xbpmc9bAQPQq6WJWOBr7K6YEA= -cloud.google.com/go/clouddms v1.14.0/go.mod h1:qSwET2Q27cJ4wCDsPsbkagXqQqkWfOy+gU3RjMsT/c8= -cloud.google.com/go/cloudtasks v1.18.0 h1:KzT7hfix/9/xAf20tNPIxwX59XGpRF0Lun2t8LHOj9E= -cloud.google.com/go/cloudtasks v1.18.0/go.mod h1:3KeCxwtGEyaySL7CR3lMmEa2I4mq1ynXdgmfNiO4RYE= cloud.google.com/go/compute v1.62.0 h1:tJ7lKJ8YEVa6vZX03Jc8o1YePbjKDOQhDw1BscMZ1bs= -cloud.google.com/go/compute v1.62.0/go.mod h1:Xm6PbsLgBpAg4va77ljbBdpMjzuU+uPp5Ze2dnZq7lw= -cloud.google.com/go/contactcenterinsights v1.23.0 h1:EtrRf8T7ajR+dR8nseDxwKy3o1VaWP1GFAu7lgMmT9A= -cloud.google.com/go/contactcenterinsights v1.23.0/go.mod h1:uB/kygbfYH/gWEq3NEgq3QRI7/MvpjFyX81ajcW5YAI= -cloud.google.com/go/container v1.51.0 h1:KdeWHqwlOvABdhkhSTSb4QFySMOuXnc3UUoQMmq9lic= -cloud.google.com/go/container v1.51.0/go.mod h1:EvqoT2eXfxLweXXUlhAMGR0sOAB00XPzEjoL01esSDs= -cloud.google.com/go/containeranalysis v0.19.0 h1:89ZhFvJHWzX9/jUJy/IEVbtSa2hv4+NtsvwWMEsUYnY= -cloud.google.com/go/containeranalysis v0.19.0/go.mod h1:Zq0XHzUIa0oTa7H6aSR8HWqeJnoRI9syUcYJzfozjZQ= -cloud.google.com/go/datacatalog v1.33.0 h1:8V80PpoAGdOOr2QhBrp4wZ66MDCbATdAB/fmVmo5rlU= -cloud.google.com/go/datacatalog v1.33.0/go.mod h1:/EMN04S73fZcPdtNg86VYLDrhi2HheMehQtMCS86Klk= -cloud.google.com/go/dataflow v0.16.0 h1:BchGCAl9QIZ/pyTGokv5V3daxBTrXsOmrU4ARXoTNd4= -cloud.google.com/go/dataflow v0.16.0/go.mod h1:BWhSrIGmsMfuYj3J+nJ2Tw7tplRR6r28kvRiqCD3WlQ= -cloud.google.com/go/dataform v1.2.0 h1:eAIsWhr6AbUIX+wOdfctO6hAr9UoQnTR2qQW739XH1I= -cloud.google.com/go/dataform v1.2.0/go.mod h1:Lhkjd6L04/nBqsEo7S9Tx7D+Vm0pDDDZuKczewAJuX0= -cloud.google.com/go/datafusion v1.14.0 h1:56rjOW8xFBnTOLgTEWdU09KhVC2/X+y/ukzux7zYqBE= -cloud.google.com/go/datafusion v1.14.0/go.mod h1:2z+uDUKkLPacNNos5lW1Jf1IRDoFyeE+glJ4hmxF2Uc= -cloud.google.com/go/datalabeling v0.15.0 h1:9TL2kwlO/ODD1fne3uvgTLa53SttwrQi92Bn7KWnKhg= -cloud.google.com/go/datalabeling v0.15.0/go.mod h1:H8WSRKD9XYCDXDlZE3bPgvV7UYI0F05e+ufKev2AFc8= -cloud.google.com/go/dataplex v1.36.0 h1:r4O6RHEkwHr41ItsTUym0ZgH1iDf48R+W8fI7KoMNl8= -cloud.google.com/go/dataplex v1.36.0/go.mod h1:ftgNMXBt+wJ4wPVNvYJ3UY3VTZtKS/i/uFEQppaEbKk= -cloud.google.com/go/dataproc/v2 v2.25.0 h1:IF1wxkXvcgoTP92NVqX6QNDf2SdKNZFPa2IWeSKP310= -cloud.google.com/go/dataproc/v2 v2.25.0/go.mod h1:hkiM6kzc8CwLGoquMN1oghyhuI1fE0girmChH4h9W7w= -cloud.google.com/go/dataqna v0.13.0 h1:VR1y/NuN+RvPkmfmlowbmk60BLINR4MgZAusVFIGQjU= -cloud.google.com/go/dataqna v0.13.0/go.mod h1:XiVVFTOEJLBSvm3ILbyjXngGQYpjb/66MSksqz/56fs= -cloud.google.com/go/datastore v1.25.0 h1:zUjMnCLCcRZVDSdQIXsbnNCl1SVRNw5Jm0J77gPaPKs= -cloud.google.com/go/datastore v1.25.0/go.mod h1:jvJVNe+S2nHVIndV1H/B4s9K3MLsTMqOKlxSrzHTxB4= -cloud.google.com/go/datastream v1.21.0 h1:hrWGASBj85xs6wpnx9ifdSBYiRLhKDgQrA1HjeNYBhc= -cloud.google.com/go/datastream v1.21.0/go.mod h1:z9AlkQGdXqkeyO5HE+D6sYbOkLJYB4BCZpXFPX/1Vpo= -cloud.google.com/go/deploy v1.33.0 h1:2PdZ8kbIztLVRMrIN/wHbalZeZZikQlxBptBFK+eEtM= -cloud.google.com/go/deploy v1.33.0/go.mod h1:QdF3plD8D5gV2RmkTXBB6cHrq490WlpFr1SChdOJO2Y= -cloud.google.com/go/dialogflow v1.84.0 h1:BymJ6nPotDcnphK2/D0VHMAfakjJlIV5GqxLHQg/Tos= -cloud.google.com/go/dialogflow v1.84.0/go.mod h1:OU8Lj1aw5Vr2hl9ifW+vsKnc2b4iJH+41U7nZ4whg3U= -cloud.google.com/go/dlp v1.34.0 h1:SI7KSLOtAfhEAw3af8NxpKHGHbJ9BkoM7805TMaU6m4= -cloud.google.com/go/dlp v1.34.0/go.mod h1:+haQd/n0QTv5BK7wZnCk2qctd5sfKL50jjh9E6N0d/Q= -cloud.google.com/go/documentai v1.49.0 h1:WLZOaWvK4NH7xMLxANsDgIdXugeqmVyYRCbVN1m87DQ= -cloud.google.com/go/documentai v1.49.0/go.mod h1:VyQA+SxPnCPlVLSJ5UcFx+LQm8JCzK7uUXdkOaAHvG8= -cloud.google.com/go/domains v0.16.0 h1:h+lcYPxyEulj5SYuH3+OwOsLZb4DTW11QX6nlxxXQ1I= -cloud.google.com/go/domains v0.16.0/go.mod h1:O5AhaEyUAgZC2X4M10nSu3dQt2cJLtbjhtrNrdeSPF8= -cloud.google.com/go/edgecontainer v1.10.0 h1:4WOjcIZRCB4ynxH9Zs1tteIcm2LD58VUCLLmIwLYUAc= -cloud.google.com/go/edgecontainer v1.10.0/go.mod h1:g4xb11IzVWa9peXNTlnNguKP8uJVvMK4zZeDlGS2Wus= -cloud.google.com/go/errorreporting v0.9.0 h1:LlE2SVIbz0k+OSeNTksk34inr3Fy62JMhHUvNaS8f7c= -cloud.google.com/go/errorreporting v0.9.0/go.mod h1:V7ojx7z76JITDZNGyDNkIIa9nNEkQzF6Yj+VHl2YF84= -cloud.google.com/go/essentialcontacts v1.12.0 h1:+AtGn+hYLdr62sMqP0ZGMRzp9A4t+MSWe8eZoD/ho+M= -cloud.google.com/go/essentialcontacts v1.12.0/go.mod h1:W8fTL17jP6vmsPHQaCT5rOjWGohEssuqDUroxnjST0A= -cloud.google.com/go/eventarc v1.25.0 h1:+XAJNEmxJIPZLMtzLYdUZN61Kmqx62hWLry4AsWaTwo= -cloud.google.com/go/eventarc v1.25.0/go.mod h1:ncY2NKHKiX+sUjIfxVozrivvmJQ4HWo2znxms7AxlP8= -cloud.google.com/go/filestore v1.16.0 h1:2GXy5jvq2oUX6HnCpx3GjzSqlcMz8K9gkHaF9HPcYNE= -cloud.google.com/go/filestore v1.16.0/go.mod h1:szr35omqptDEuXgBbJ8PdVdYM3lf/Md96kNufWr1tVs= -cloud.google.com/go/firestore v1.24.0 h1:x0Z3hrgjYgo2wI9whuBRQcNc2hYwzZDQy/7pkUXbXcs= -cloud.google.com/go/firestore v1.24.0/go.mod h1:5aojyjN4olKUnBZDCRWwM+NsdrrCX3t1qfyERZGOonM= -cloud.google.com/go/functions v1.25.0 h1:ndUtLkam3XF9b0t2zVACH9D/EgFBISbVuQFqRz/X58k= -cloud.google.com/go/functions v1.25.0/go.mod h1:b/tqakoKeAkj9RspEjqswWf5299Lkz9C/742QUD3OEk= -cloud.google.com/go/gkebackup v1.14.0 h1:li3BtGRis1QYrkLo8+Iq2wf5WbP9v3sz9VoUw8WqgaA= -cloud.google.com/go/gkebackup v1.14.0/go.mod h1:kaD4l/s0ONcb3L9iHC8PzG1XkC5ggPwA/KAl6yAyQGs= -cloud.google.com/go/gkeconnect v1.0.0 h1:IDuAD/w0Xph8k/31vdt6pBkDdRx55lVXz7gW8ABOfCs= -cloud.google.com/go/gkeconnect v1.0.0/go.mod h1:5iWSBQzMIRLwUHUWVhxxcNK45ZPE8ntyBgE0MkavlqQ= -cloud.google.com/go/gkehub v0.22.0 h1:gHKPoWQuWpd9dXLoC58dxwBvSYTNc5/fdYYcs3JUn1s= -cloud.google.com/go/gkehub v0.22.0/go.mod h1:WiXX1w9ZHwKZVUDwL//YQfjfWS7yE0I/ym3smZn9iwE= -cloud.google.com/go/gkemulticloud v1.12.0 h1:3LPj8ro7bPdZ0BlQJwby25G32rRezGLRdhNM6U3AxGs= -cloud.google.com/go/gkemulticloud v1.12.0/go.mod h1:vLNCxGah7pPIoNSX4Yx+hb8klqA0lzzXTWBSut9KzRo= -cloud.google.com/go/gsuiteaddons v1.12.0 h1:kz9DyBk84wZCda2Rdha1MOIf7/9x6R+N+LuH9B3zYFs= -cloud.google.com/go/gsuiteaddons v1.12.0/go.mod h1:rm/XT7wmwOFGn7jmWtVV65QmZCakzTbHLSojIC4Hskg= -cloud.google.com/go/iap v1.17.0 h1:BDAVJy+juq7cMRumIx9toc4pt1K7zXoZdAI3lDD6D3g= -cloud.google.com/go/iap v1.17.0/go.mod h1:b+r+yjrss2WmAEzNrQQjlEdD5E9B8c47mOF7XnqT+z0= -cloud.google.com/go/ids v1.11.0 h1:394LkKIavhv/+oQVMtbXe9WQqCeY9wSFcHktqC9ZkLE= -cloud.google.com/go/ids v1.11.0/go.mod h1:+drdvU0pQ4x5uYiWCv364VOeIpTN/PETBrdR51D4Tjk= -cloud.google.com/go/iot v1.13.0 h1:pyt1EuMFpbV/2BlmYlXr+HBnUNSPKlP9dVK/dpFTu1U= -cloud.google.com/go/iot v1.13.0/go.mod h1:62W4n2fe/Ct66NWJEfCB5suZ3XsL5Atx+MxFjScr+9s= -cloud.google.com/go/kms v1.32.0 h1:s+rEluaaZKhLVjrIWG7uNBsnWbiitElzNzFGyp6+nIg= -cloud.google.com/go/kms v1.32.0/go.mod h1:CSGvW6GnMQbY+1nOHcIzhMtHSbExXlOmCKjWtYVjcpA= -cloud.google.com/go/language v1.18.0 h1:q58bL7rmxvw6Q6VHt+wjFsAE1Tj/JkuAEIR4+84rx9U= -cloud.google.com/go/language v1.18.0/go.mod h1:xSeiVB4UiA9wYmFy2GWjf1Mb1K3uR1Yi/80qoqTxH04= -cloud.google.com/go/lifesciences v0.16.0 h1:T+7N3oVHK8a7BhX6HJLrBW8xEbMV9iY8Mb+aOl7r6+A= -cloud.google.com/go/lifesciences v0.16.0/go.mod h1:axEwGa3A63+vCXIis+0Zkseu8KecqtNoSn7x0zyjJfM= -cloud.google.com/go/managedidentities v1.13.0 h1:ZzWkg3LSrIi9OdzCK3bxutexvw1dDDg9+tOMitbkOXw= -cloud.google.com/go/managedidentities v1.13.0/go.mod h1:lUYH5r6QEJTHqjgga0WFeiieqJ0iRwEuQSk20O41Vj0= -cloud.google.com/go/maps v1.37.0 h1:YoTRWohpNKeKxdQynorwHH69NZuUM06divZtqGPYjUA= -cloud.google.com/go/maps v1.37.0/go.mod h1:oalKFBmf2eHmdr3OvfEiiBlOakNlVitYYEPcM3TTUB4= -cloud.google.com/go/mediatranslation v0.13.0 h1:qqMRAK0mhc9M+lM6pb791oh3DYUbwLcIy58jOyXtadE= -cloud.google.com/go/mediatranslation v0.13.0/go.mod h1:kjZrowuigFr+Bf1HM1TCtp1a3E3kfG1ovPK5VEuaNAQ= -cloud.google.com/go/memcache v1.17.0 h1:l/co1jsmVLkNyl8uMqj1u1HIwd3AhCvGdHbEAIPb6jc= -cloud.google.com/go/memcache v1.17.0/go.mod h1:QQpFWgJvrFaQ6DgmitHejdbkLg8SJfHg5BzltKEWSt0= -cloud.google.com/go/metastore v1.20.0 h1:kz0XBkP531MB1IR9jTY6/DB33UFP5Df1vFpPNgW1iPk= -cloud.google.com/go/metastore v1.20.0/go.mod h1:/bhZoizjM5iOrqWJeAFDw7c16C783wEftqofnJgKKYI= -cloud.google.com/go/networkconnectivity v1.27.0 h1:ieYJjbUn2L9ZSzEB9+qKgtoD3/l1/qAnId/Z6KE4ZBo= -cloud.google.com/go/networkconnectivity v1.27.0/go.mod h1:pCnczH2W/cnLSlnsnN+VzBoXlM81ZoUGuuacFBGThyw= -cloud.google.com/go/networkmanagement v1.30.0 h1:LqYm91vSQk3NM6fHjthseRh4xrDvhXIWQBsQY/Ksus8= -cloud.google.com/go/networkmanagement v1.30.0/go.mod h1:3SBf5T7jyGzw5jqJWE7TUDRhIl2E029jggbeoFEgt5E= -cloud.google.com/go/networksecurity v0.19.0 h1:PzJnbVd0sS2+xDR9hnk31NAyqFt9qB8ohLn7XIsKIBA= -cloud.google.com/go/networksecurity v0.19.0/go.mod h1:VWDFX+stDgzZYDsCX1Wy/JO9Tlw7g/V1UHbiORVgqq0= -cloud.google.com/go/notebooks v1.18.0 h1:BgxdQcoRSSjiyszWNOTu3fmXG/6Ks4W2sHgjx0BAke0= -cloud.google.com/go/notebooks v1.18.0/go.mod h1:fXU6A3TJ2YobFy6fxOr4tKZZ8QgTjdJAqDIykOB85Gk= -cloud.google.com/go/optimization v1.12.0 h1:Cd4DgujNhEMcjfaVLGqbjfDwVDnmS1Bc93cx9TE8ZYw= -cloud.google.com/go/optimization v1.12.0/go.mod h1:28gzCUmeCLcT4vctGEo71QF4b60TYkKQo5y8Gs2KPq8= -cloud.google.com/go/orchestration v1.17.0 h1:dqf2HzqUi4CaaDiNfv6jVoZjz9lF0HWjyIEgccxPHMk= -cloud.google.com/go/orchestration v1.17.0/go.mod h1:Lf/Czqh4Jfy3IFpvDkKWjfjkYFI+tj6nAjq5ihivrq4= -cloud.google.com/go/orgpolicy v1.20.0 h1:kpVcE/OsC5aAzHCsAiuQSg3+s6ILzgPTuPZyS7n7ejA= -cloud.google.com/go/orgpolicy v1.20.0/go.mod h1:9LHqEGx5P5dhansdKTNIEXpM+QbebAIOs66+HUID4aQ= -cloud.google.com/go/osconfig v1.22.0 h1:r5lzneR9GNixJ96lZ6oIfJK9Cd5sKpnzZJe1LcGZf4w= -cloud.google.com/go/osconfig v1.22.0/go.mod h1:bUL0FaSR2ahPcFRRYnd6a0LyUzsQYIdUpBq8Tmxg8fE= -cloud.google.com/go/oslogin v1.18.0 h1:OkccORMcY2XEHwN5+AP0XH/SCUNBl39GGjaQuSCVcIw= -cloud.google.com/go/oslogin v1.18.0/go.mod h1:3Oa36T3781Mv+yCSVYlfasi7auHjfPFqvNOd1q92umc= -cloud.google.com/go/phishingprotection v0.13.0 h1:6WJF1z3Ie8fZAiCRpeSGB81h6s86XWS2SBC9iQUZxX8= -cloud.google.com/go/phishingprotection v0.13.0/go.mod h1:2gyYqwNjePPEocXDkDve3EuJPaRqN/E7fp28K3arR0k= -cloud.google.com/go/policytroubleshooter v1.16.0 h1:V3CcqzY6FhY2c2idRGZv0Ef4mFinLnXnfDFtAhxyTfM= -cloud.google.com/go/policytroubleshooter v1.16.0/go.mod h1:FZg3IW3exF6wc9eO/iBYijsGqiiCzc9mjZhsxgATXYA= -cloud.google.com/go/privatecatalog v0.16.0 h1:SBhcVNVhgEGT1edcfSFY/KSiy2xSQTSQUpY12f10rmw= -cloud.google.com/go/privatecatalog v0.16.0/go.mod h1:Dq1bSHRRaDqFr7Rb7UntXVjh1reeY6YdzYicL0EPTrM= -cloud.google.com/go/pubsub v1.51.0 h1:XOaCejsqX7EEtUdQz+WPag66wWsUUGliyCOfGPKfo90= -cloud.google.com/go/pubsub v1.51.0/go.mod h1:NERXf11sd82UV3VnflcUj8POIyQUXT/QwrKlxD8di/I= -cloud.google.com/go/pubsub/v2 v2.6.0 h1:8pjR0id+GTB+krKx5G6AGJoYrHog58w2Q89PCOrfM64= -cloud.google.com/go/pubsub/v2 v2.6.0/go.mod h1:4anqvV/w8Pcgu2tO0qr2XgsF3GXHowzryfQ5gOnVmWY= -cloud.google.com/go/pubsublite v1.10.0 h1:kqNk9e0gt0/Eu6lChjDlgu4B8k/399FZFpAU4LNq4Co= -cloud.google.com/go/pubsublite v1.10.0/go.mod h1:o9NVNBY4m8LubZqRCJtBdxpjP8DAsYizsxC6Z1vI7Dk= -cloud.google.com/go/recaptchaenterprise/v2 v2.26.0 h1:9qDHoQtUIZ8FDN5uFAfnNz8jyyXdpAJf6rMN4ZMmMcU= -cloud.google.com/go/recaptchaenterprise/v2 v2.26.0/go.mod h1:+ntF70/j7qBa6G/pwmYA0mkBcDeTCXV6WDqUL7GObfs= -cloud.google.com/go/recommendationengine v0.15.0 h1:G0tmQXp67YDaduvde7AX+cIdTtDCo2PBMeqv8SBOpnw= -cloud.google.com/go/recommendationengine v0.15.0/go.mod h1:Yx45rCF3A5fLSeXxSkXOCTXSBDBogrQnR7kUTJHwYxw= -cloud.google.com/go/recommender v1.19.0 h1:fJ6oO/7Ta/yzfRHcuJUln9iCeo6FDb5yIi2L7eLldzs= -cloud.google.com/go/recommender v1.19.0/go.mod h1:LRh+1HJjLx2kDE3S65AIlG/lvwA0llEFWYPD/QtgoaU= -cloud.google.com/go/redis v1.24.0 h1:lV6xFUF7t3fm2ZRFjB4IVNkznG9EVwetYVJYwzmW8pc= -cloud.google.com/go/redis v1.24.0/go.mod h1:ebtw9WLFKswecHO2ifNykuteNJNwoPqMCHz4UI11kF4= -cloud.google.com/go/resourcemanager v1.16.0 h1:aDf2RuuQ1Z+kbHHDzHyJNMHPGRFK+cMYfZIhWRaeRX0= -cloud.google.com/go/resourcemanager v1.16.0/go.mod h1:Hn4HPkLRnTuiUhFEFJg736Brt7BwlS84xYU06sc3STc= -cloud.google.com/go/resourcesettings v1.8.3 h1:13HOFU7v4cEvIHXSAQbinF4wp2Baybbq7q9FMctg1Ek= -cloud.google.com/go/resourcesettings v1.8.3/go.mod h1:BzgfXFHIWOOmHe6ZV9+r3OWfpHJgnqXy8jqwx4zTMLw= -cloud.google.com/go/retail v1.32.0 h1:u27GLYlnNsQrqzcvSlpZK0L6Ig8k3PEuXGkUdNFCOn4= -cloud.google.com/go/retail v1.32.0/go.mod h1:t9w9mBarD59BnFHTST2LoiCP5608ZlEfniHLgA6OoH0= -cloud.google.com/go/run v1.22.0 h1:U56fxJWdrT+yjo4S/Vrtw5m69NdNL11Cyv9jX2JOi1s= -cloud.google.com/go/run v1.22.0/go.mod h1:Wo0aTNrqfftGmbxPPraeOxSUDUZ2c7IVNg2dk8Qm1Bs= -cloud.google.com/go/scheduler v1.16.0 h1:EPdChptxnvCasdMixuu58247qhKMO1iAlrVaQhvuRyE= -cloud.google.com/go/scheduler v1.16.0/go.mod h1:0hsZg0MZJADyke1lutI0FHAYJR8Dtm8oIivXkmpACkA= -cloud.google.com/go/secretmanager v1.20.0 h1:GjE3NoyFXo7ipRPy26PMmg4oRX1Ra8fswH45r16rWV0= -cloud.google.com/go/secretmanager v1.20.0/go.mod h1:9OmSuOeiiUicANglrbdKWSnT3gYkRcXuUQDk7dDW0zU= -cloud.google.com/go/security v1.26.0 h1:xp/htSPHpyqo225Ju6PjNS9jxfcOEQMxnF229Oze62g= -cloud.google.com/go/security v1.26.0/go.mod h1:nd0i5OHXtJduMt0n6UnEojy7fiTfnfj/PSDeD7LAD+c= -cloud.google.com/go/securitycenter v1.45.0 h1:k5uzLtTSxnh4fwG+8nIcuwDjEE3J+6xe2wcZ72cys7s= -cloud.google.com/go/securitycenter v1.45.0/go.mod h1:7mAlzsCsKlEVmciAFORl431laDGpoKGFkSQndAzFs30= -cloud.google.com/go/servicedirectory v1.17.0 h1:yrohWkwM8t5JfEFCmmlyksKnpMpvxM8XbRpwg+yIo64= -cloud.google.com/go/servicedirectory v1.17.0/go.mod h1:CtgjXS1idj3s9Q6tB68021Rzk8Q6decV6+ldXC1BoBk= -cloud.google.com/go/shell v1.13.0 h1:vJ/g4BXCwBRMmUxHoNjdbE0DRh5Aysw/+vrZ/aPl0yc= -cloud.google.com/go/shell v1.13.0/go.mod h1:9WWf3xHQUElP5fL/lB9IJ/MMMnN2W/T86cBp+pXFFWo= -cloud.google.com/go/spanner v1.91.0 h1:XwXfcZ0kc1NT9Uu2IsThFiWtYptB+WgLn/KZEZcyzRg= -cloud.google.com/go/spanner v1.91.0/go.mod h1:8NB5a7qgwIhGD19Ly+vkpKffPL78vIG9RcrgsuREha0= -cloud.google.com/go/speech v1.36.0 h1:ZcmRKcY02vkF3o/Eaa9yI3i1baNFTF4ctyDZVRfnNS0= -cloud.google.com/go/speech v1.36.0/go.mod h1:tiSA8MiX49o1ngq5Ww2JFTvfjKxtAuBKY/UIH6coCPg= -cloud.google.com/go/storagetransfer v1.19.0 h1:jh+3SegMUulEhuJbkADCv8lmfRC8/pCpxja4oex2Ns8= -cloud.google.com/go/storagetransfer v1.19.0/go.mod h1:sy4ImXynHkm9CKmbILtmzLN36PHh7JOhUTpqXf5SvMs= -cloud.google.com/go/talent v1.14.0 h1:dUNSwgBUFlDEMHJpmntEvDeLkPQAoEzBrvDbNfS/UeQ= -cloud.google.com/go/talent v1.14.0/go.mod h1:jieYQngp1YqRtqV2t92w3LTrjuLV05kMM4BZMUUneaw= -cloud.google.com/go/texttospeech v1.22.0 h1:xGqQv2LB4nttaBNMepCm19EQ0Sk0YdjzPvklDU+LYsI= -cloud.google.com/go/texttospeech v1.22.0/go.mod h1:bAksATiWPKaw8r8wVgANa4GkVdsyFE4y9ulRzKyuJec= -cloud.google.com/go/tpu v1.14.0 h1:u2f8Lhezy6piu0BB3+4pTLw9xyOeU5n8PCgWjbjCitk= -cloud.google.com/go/tpu v1.14.0/go.mod h1:1pggTTG5npfxea6vYjyl60Fg09VgbM7efBgVjnFZjpo= -cloud.google.com/go/translate v1.18.0 h1:2dxWfGMG7y5vobxD2pPyngnfPPIk4MYdNB9QK5sEbuc= -cloud.google.com/go/translate v1.18.0/go.mod h1:aRVIE+P+7fngk8HwwFAgis5QA7wphGpKrFpdNoWtGCM= -cloud.google.com/go/video v1.33.0 h1:zPA41kNscRrJeRo/xD73TjNCFAnRDnP9ViNPuWARVeg= -cloud.google.com/go/video v1.33.0/go.mod h1:hEx8TNpQT6kdjMVsywePvT8BCb63Ee3F/R0GRa9wnzo= -cloud.google.com/go/videointelligence v1.17.0 h1:cd+s4jLMS59xiYLT6eQ6PVKP8R0PskgifXDa7YwMUUU= -cloud.google.com/go/videointelligence v1.17.0/go.mod h1:Phxz7AQpvXoOvz+KrrOZEJRo4CDgYXMDVDqhCtdF1jc= -cloud.google.com/go/vision/v2 v2.15.0 h1:aTR1vj4++WtS9HD6YdGuoaYygMTJ873WaoV9sYjlQCc= -cloud.google.com/go/vision/v2 v2.15.0/go.mod h1:DUdjdFkXqPvEoPC4WDYFvYCn0LlAZ4vVz29A0bXvW90= -cloud.google.com/go/vmmigration v1.16.0 h1:OPGMxx73owRHMve5B4L6W+0KQUwlunyE9kfbYzFHqug= -cloud.google.com/go/vmmigration v1.16.0/go.mod h1:ILrSjXnHMpdamkkAU8fjMKKMsH27B6FLC5kv/6TkLy0= -cloud.google.com/go/vmwareengine v1.9.0 h1:vdSutihjzH5iYEFlN+1x+gDm4xzj6Md2yMzXADKI5cQ= -cloud.google.com/go/vmwareengine v1.9.0/go.mod h1:zXXuUaIpvDhsV6sR+JdQfcQ4V5+pDarrp7FW7nOdS2I= -cloud.google.com/go/vpcaccess v1.14.0 h1:fFa1i67AjC/3MuG5GPlvx36sIMeffpsIs3kI1C7S3Jk= -cloud.google.com/go/vpcaccess v1.14.0/go.mod h1:MxbVgr+2fpIFIEIdSmgnb8ykNWRPVtslpmWijp7an68= -cloud.google.com/go/webrisk v1.17.0 h1:t2WMlBo3xXxH6k5kTSkyIYUl24AHgmBI76F/5OqAPRY= -cloud.google.com/go/webrisk v1.17.0/go.mod h1:ypwCZ+G/SXyUZ+x3ppxn1hu+6tDifGNd/OpwPtCdJHI= -cloud.google.com/go/websecurityscanner v1.12.0 h1:iV+hAXeEo8kKtFWDZWPp9Z0fsziZnuOt4nCCYp68RP0= -cloud.google.com/go/websecurityscanner v1.12.0/go.mod h1:cZSc9HqoFdccL1mqZtPIInOd4R8PBGwI20wdnrz6AO8= -cloud.google.com/go/workflows v1.20.0 h1:qROpn1zRDdeIjUP9w9O3MHXJEg8v/pzVMicAcw9vLvQ= -cloud.google.com/go/workflows v1.20.0/go.mod h1:TC9yx7VpjGdBBeKM8FG2EMtms5Q9nyTqI+2uV9bDNs4= -github.com/ThalesGroup/crypto11 v1.5.0 h1:fV+gZtXl36t19Xw7bbbpWRsEbzLB9Qxjk/YQLTRk0YQ= -github.com/ThalesGroup/crypto11 v1.5.0/go.mod h1:sHbXFYNbNLe231R/gmWlE4MXh8dn8n0EqfD+harPBLA= -github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= -github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= -github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= -github.com/checkpoint-restore/go-criu/v6 v6.3.0 h1:mIdrSO2cPNWQY1truPg6uHLXyKHk3Z5Odx4wjKOASzA= -github.com/checkpoint-restore/go-criu/v6 v6.3.0/go.mod h1:rrRTN/uSwY2X+BPRl/gkulo9gsKOSAeVp9/K2tv7xZI= -github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= -github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= -github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= -github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= -github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg= -github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= -github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/qqsc= -github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= -github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= -github.com/cyphar/filepath-securejoin v0.3.5 h1:L81NHjquoQmcPgXcttUS9qTSR/+bXry6pbSINQGpjj4= -github.com/cyphar/filepath-securejoin v0.3.5/go.mod h1:edhVd3c6OXKjUmSrVa/tGJRS9joFTxlslFCAyaxigkE= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/frostbyte73/core v0.1.1 h1:ChhJOR7bAKOCPbA+lqDLE2cGKlCG5JXsDvvQr4YaJIA= -github.com/frostbyte73/core v0.1.1/go.mod h1:mhfOtR+xWAvwXiwor7jnqPMnu4fxbv1F2MwZ0BEpzZo= -github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= -github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= -github.com/gammazero/deque v1.2.1 h1:9fnQVFCCZ9/NOc7ccTNqzoKd1tCWOqeI05/lPqFPMGQ= -github.com/gammazero/deque v1.2.1/go.mod h1:5nSFkzVm+afG9+gy0VIowlqVAW4N8zNcMne+CMQVD2g= -github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY= -github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= -github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= -github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= -github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-pkcs11 v0.3.0 h1:PVRnTgtArZ3QQqTGtbtjtnIkzl2iY2kt24yqbrf7td8= -github.com/google/go-pkcs11 v0.3.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= -github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA= -github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= -github.com/hamba/avro/v2 v2.31.0 h1:wv3nmua7lCEIwWsb6vqsTS3pXktTxcKg5eoyNu0VhrU= -github.com/hamba/avro/v2 v2.31.0/go.mod h1:t6lJYAGE5Mswfn17zjtyQsssRQgnqO6TXLBCHHWRqrw= -github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= -github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= -github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= -github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= -github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= -github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= -github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= -github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4= -github.com/lithammer/fuzzysearch v1.1.8/go.mod h1:IdqeyBClc3FFqSzYq/MXESsS4S0FsZ5ajtkr5xPLts4= -github.com/lithammer/shortuuid/v4 v4.3.0 h1:XRr80OYPOlqxgnCv2/iuElkl/ZtXtPtLKk2AvpXGdWA= -github.com/lithammer/shortuuid/v4 v4.3.0/go.mod h1:D5noHZ2oFw/YaKCfGy0YxyE7M0wMbezmMjPdhyEFe6Y= -github.com/lyft/protoc-gen-star/v2 v2.0.4 h1:JDlNKttNIRd68AAIychs0AqEpO8/I/WYi01OQ7Raw6Q= -github.com/lyft/protoc-gen-star/v2 v2.0.4/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= -github.com/mackerelio/go-osstat v0.2.8 h1:I2duicTaCGWoM53XwAwA9OIe1inu0xnVs8/pqOWWVr4= -github.com/mackerelio/go-osstat v0.2.8/go.mod h1:SyS3XxKdoSKJnTGTkN5Yrh6VUQVuAURACfE6y+2DN4k= -github.com/magefile/mage v1.17.2 h1:fyXVu1eadI8Ap1HCCNgEhJ5McIWiYhLR8uol64ZZc40= -github.com/magefile/mage v1.17.2/go.mod h1:Yj51kqllmsgFpvvSzgrZPK9WtluG3kUhFaBUVLo4feA= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.20 h1:WcT52H91ZUAwy8+HUkdM3THM6gXqXuLJi9O3rjcQQaQ= -github.com/mattn/go-runewidth v0.0.20/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= -github.com/maxbrunsfeld/counterfeiter/v6 v6.12.2 h1:V23nK2R2B63g2GhygF9zVGpnigmhvoZoH8d0hrZwMGY= -github.com/maxbrunsfeld/counterfeiter/v6 v6.12.2/go.mod h1:Mr897yU9FmyKaQDPtRlVKibrjz40XXyOHUfyZBPSyZU= -github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= -github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= -github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= -github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= -github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mrunalp/fileutils v0.5.1 h1:F+S7ZlNKnrwHfSwdlgNSkKo67ReVf8o9fel6C3dkm/Q= -github.com/mrunalp/fileutils v0.5.1/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= -github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/nyaruka/phonenumbers v1.8.1 h1:2K9YMQuv1dCGqjjzB1DwmdCe89khT4KPBQb2CxAMMlU= -github.com/nyaruka/phonenumbers v1.8.1/go.mod h1:fsKPJ70O9JetEA4ggnJadYTFWwtGPvu/lETTXNXq6Cs= -github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= -github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= -github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= -github.com/petermattis/goid v0.0.0-20260820044319-269ab09b5261 h1:lcWAnrqr2nNfDiArwFNHCE4787Mw2tCdVSOXCru0/0E= -github.com/petermattis/goid v0.0.0-20260820044319-269ab09b5261/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= -github.com/pion/datachannel v1.6.2 h1:7EXQ8TH3vTouBUdRWYbcX2edSx9Yj6k5zl5P+qyxEPc= -github.com/pion/datachannel v1.6.2/go.mod h1:pzbdAZvyGtXbcHM1hBbsFaOTf40lZizU/dNlvVOak6E= -github.com/pion/dtls/v3 v3.1.5 h1:9xJtVsHwMYeSjPp5Hh1FTis4DchnQWtnOa5o+6ygqfc= -github.com/pion/dtls/v3 v3.1.5/go.mod h1:gz1K4jg6c+fq86oQMH4pilpCEOEPwmEr2jY+VcF/mkU= -github.com/pion/ice/v4 v4.4.0 h1:wvHDDqimaC38Y7MVpD46Y63p246ChvXd87VKoLYS5b4= -github.com/pion/ice/v4 v4.4.0/go.mod h1:obAyD+J+Hzs7QA7Y8YXHp5uIn6gb7z87pKedXZkrcFU= -github.com/pion/interceptor v0.1.47 h1:yw8t5pJ2f8t78NgU+8EmxhaqYLXS7uFCC/tAGOaSDBo= -github.com/pion/interceptor v0.1.47/go.mod h1:7yoRBzaIDETPC6cIN8Zj9EyGqHv1ImOpcTFPha6MuOM= -github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= -github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= -github.com/pion/mdns/v2 v2.1.0 h1:3IJ9+Xio6tWYjhN6WwuY142P/1jA0D5ERaIqawg/fOY= -github.com/pion/mdns/v2 v2.1.0/go.mod h1:pcez23GdynwcfRU1977qKU0mDxSeucttSHbCSfFOd9A= -github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= -github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= -github.com/pion/rtcp v1.2.17 h1:PxiT6L79yPZKtXIsXdG1eakBl6dtBj4x+4oVEL0DlSw= -github.com/pion/rtcp v1.2.17/go.mod h1:7kBpuBJaWwax4hzc/pgexY8vkOpvh8atgYDbaKZq0iU= -github.com/pion/rtp v1.10.5 h1:ip0HhO/wYZqQ4bKS+R99KnZh/GRCmIT0jDXikub7vlE= -github.com/pion/rtp v1.10.5/go.mod h1:Au8fc6cEByy8RLTwKTQTEeQqDB/SJDxwL4mZuxYA5Pk= -github.com/pion/sctp v1.11.1 h1:O4dIFyURw1KTST7w+gtD4gLeYXkhPa0xXLHMMoe/OSA= -github.com/pion/sctp v1.11.1/go.mod h1:7KFmTwLcoYgJs/Z+99nJvsWL0qDpuyloSI0RbAqlrz0= -github.com/pion/sdp/v3 v3.0.19 h1:1VMKs3gIkTQV5M3hNKfTAPrDXSNrYtOlmOD8+mSZUGQ= -github.com/pion/sdp/v3 v3.0.19/go.mod h1:dE5WOSlzXrtiE/iuZqe9n+AcEbOjtAd3k5m5NtlV/qU= -github.com/pion/srtp/v3 v3.0.12 h1:U7V17bckl7sI4mb3sepiojByDuBY0wNCqQE+6IlQBbc= -github.com/pion/srtp/v3 v3.0.12/go.mod h1:EeZOi/sd6glM1EXapg051gdNWO9yWT1YSsgQ4SlJkns= -github.com/pion/stun/v3 v3.1.6 h1:WnhsD0eHCiwCfKNkVx0VJJwr2Y3eV4Ueih3KJ+dfZy8= -github.com/pion/stun/v3 v3.1.6/go.mod h1:zRUghXSQU32Lx5orJsz3uYMkIihweXb3mu5gIns02fs= -github.com/pion/transport/v4 v4.0.2 h1:ifYlPqNwsy6aKQ9y8yzxXlHae5431ZrH2avkD/Rn6Tk= -github.com/pion/transport/v4 v4.0.2/go.mod h1:06hFI+jCFcok2X2MekVufNZ/uzNZXivGBPfviSVcjgM= -github.com/pion/turn/v5 v5.0.12 h1:6+b69ivQQXSlyfkp2AKripqD2k3W32qXK8QzCzpJWPI= -github.com/pion/turn/v5 v5.0.12/go.mod h1:CQACsRDJtjQ+6RSrGHrS2PCIerLwbW3uqXRqOvtjAFg= -github.com/pion/webrtc/v4 v4.2.18 h1:smA/3g6Gy4RohM0VIZ5KKY/12TQbxv3XFgpUMyb2EUI= -github.com/pion/webrtc/v4 v4.2.18/go.mod h1:vmzi6s+rvhoIuT94DPqivB+0xJXs9rG4QRD+4MgBtlY= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= -github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= -github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= -github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= -github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= -github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= -github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= -github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= -github.com/pterm/pterm v0.12.83 h1:ie+YmGmA727VuhxBlyGr74Ks+7McV6kT99IB8EU80aA= -github.com/pterm/pterm v0.12.83/go.mod h1:xlgc6bFWyJIMtmLJvGim+L7jhSReilOlOnodeIYe4Tk= -github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= -github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= -github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2rWM6M9YjLpY= -github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= -github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= -github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= -github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= -github.com/substrait-io/substrait v0.87.0 h1:40rP4LejyK6SNQlWz7NX6kQELf8cmScWMBGruWhN4io= -github.com/substrait-io/substrait v0.87.0/go.mod h1:MPFNw6sToJgpD5Z2rj0rQrdP/Oq8HG7Z2t3CAEHtkHw= -github.com/substrait-io/substrait-go/v8 v8.1.1 h1:XL7CqVVOQkXP9FRSIm2ylJCjqRASg1yBRcn5nRCRwss= -github.com/substrait-io/substrait-go/v8 v8.1.1/go.mod h1:6GLz9k21udB64g4lLKq8632TKfQCRAVfhuU3NSXtZWY= -github.com/substrait-io/substrait-protobuf/go v0.85.0 h1:zk6MtNWLtDSl8a7qCZRFH0+EIIXVrrd/hsgYK/SQTgM= -github.com/substrait-io/substrait-protobuf/go v0.85.0/go.mod h1:hn+Szm1NmZZc91FwWK9EXD/lmuGBSRTJ5IvHhlG1YnQ= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/thales-e-security/pool v0.0.2 h1:RAPs4q2EbWsTit6tpzuvTFlgFRJ3S8Evf5gtvVDbmPg= -github.com/thales-e-security/pool v0.0.2/go.mod h1:qtpMm2+thHtqhLzTwgDBj/OuNnMpupY8mv0Phz0gjhU= -github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo= -github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= -github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= -github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= -github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= -github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= -github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM= -github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= -github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= -go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= -go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= -golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= -golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= -golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 h1:HjU6IWBiAgRIdAJ9/y1rwCn+UELEmwV+VsTLzj/W4sE= -golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6/go.mod h1:Eqhaxk/wZsWEH8CRxLwj6xzEJbz7k1EFGqx7nyCoabE= -golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= -golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= -golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= -golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= -golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= -golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= -golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= -golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20260921155816-b14227669459 h1:YeHSHMle+uLS75+Z+FuOA7bH9wibPWVkMRWD+9Aaz7I= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20260921155816-b14227669459/go.mod h1:RoCpRfcA27uTJ0TZb3Vyad8eLVakUKdLdV2yMdlm2+w= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.2 h1:rgSNvqscFZ1JgV/4wH5GOsZFSFkR2Eua9As3KIr2LlM= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.2/go.mod h1:iMEtFwDlAhjDU9L5mY6U1XLwlIId/G3h+QcBHDIvrJ8= -google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6 h1:ExN12ndbJ608cboPYflpTny6mXSzPrDLh0iTaVrRrds= -google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6/go.mod h1:6ytKWczdvnpnO+m+JiG9NjEDzR1FJfsnmJdG7B8QVZ8= -modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= -modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= -modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= -modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= -modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= -modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= From f7e611dd58b105d9acbf9fa70b765d1da3da773a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sun, 18 Oct 2026 10:55:03 -0700 Subject: [PATCH 28/29] objects version: the app's token is asked for only when a version needs committing --- .github/workflows/objects-version.yaml | 49 +++++++++++++++----------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/.github/workflows/objects-version.yaml b/.github/workflows/objects-version.yaml index c82a0af..704d03b 100644 --- a/.github/workflows/objects-version.yaml +++ b/.github/workflows/objects-version.yaml @@ -22,43 +22,52 @@ jobs: if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.head_ref }} + fetch-depth: 0 + persist-credentials: false + + # A version not yet released, or released with these interfaces, stands. + # Otherwise the next minor after the latest release: before v1, any + # change may break a caller. + - name: Next objects version + id: next + run: | + v=$(awk '$1 == "github.com/livekit/storage/objects" {print $2}' go.mod) + if ! git rev-parse -q --verify "refs/tags/objects/$v" >/dev/null || git diff --quiet "refs/tags/objects/$v" HEAD -- objects; then + exit 0 + fi + latest=$(git tag -l 'objects/v*' --sort=-v:refname | head -1) + echo "version=$(echo "${latest#objects/}" | awk -F. '{print $1 "." $2+1 ".0"}')" >> "$GITHUB_OUTPUT" + - id: app + if: steps.next.outputs.version != '' uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 with: app-id: ${{ vars.REPO_WRITER_GH_APP_ID }} private-key: ${{ secrets.REPO_WRITER_GH_APP_PRIVATE_KEY }} - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.head_ref }} - fetch-depth: 0 - token: ${{ steps.app.outputs.token }} - - name: Set up Go + if: steps.next.outputs.version != '' uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: "go.mod" cache: false - - name: Raise the objects version + - name: Commit the version + if: steps.next.outputs.version != '' env: + NEXT: ${{ steps.next.outputs.version }} BRANCH: ${{ github.head_ref }} + REPO: ${{ github.repository }} GH_TOKEN: ${{ steps.app.outputs.token }} APP: ${{ steps.app.outputs.app-slug }} run: | - v=$(awk '$1 == "github.com/livekit/storage/objects" {print $2}' go.mod) - # A version not yet released, or released with these interfaces, stands. - if ! git rev-parse -q --verify "refs/tags/objects/$v" >/dev/null || git diff --quiet "refs/tags/objects/$v" HEAD -- objects; then - exit 0 - fi - # The next minor after the latest release: before v1, any change may - # break a caller. - latest=$(git tag -l 'objects/v*' --sort=-v:refname | head -1) - next=$(echo "${latest#objects/}" | awk -F. '{print $1 "." $2+1 ".0"}') - go mod edit -require=github.com/livekit/storage/objects@$next - (cd e2e && go mod edit -require=github.com/livekit/storage/objects@$next) + go mod edit -require=github.com/livekit/storage/objects@$NEXT + (cd e2e && go mod edit -require=github.com/livekit/storage/objects@$NEXT) # The app's own identity, so the commit is attributed to it. git config user.name "$APP[bot]" git config user.email "$(gh api "/users/$APP%5Bbot%5D" --jq .id)+$APP[bot]@users.noreply.github.com" - git commit -am "objects: $next" - git push origin "HEAD:$BRANCH" + git commit -am "objects: $NEXT" + git push "https://x-access-token:$GH_TOKEN@github.com/$REPO.git" "HEAD:$BRANCH" From 0e1a334ad3b3a1beba5d0817b8616b2677458978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Monnom?= Date: Sun, 18 Oct 2026 11:06:20 -0700 Subject: [PATCH 29/29] objects version: commits with the workflow's token and starts the checks for that commit, which count for the PR --- .github/workflows/objects-version.yaml | 32 ++++++++++---------------- README.md | 2 +- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/.github/workflows/objects-version.yaml b/.github/workflows/objects-version.yaml index 704d03b..e10b568 100644 --- a/.github/workflows/objects-version.yaml +++ b/.github/workflows/objects-version.yaml @@ -1,7 +1,8 @@ name: objects version permissions: - contents: read + contents: write + actions: write on: pull_request: @@ -14,10 +15,8 @@ concurrency: jobs: # Raises the objects version go.mod requires when a PR changes objects/ # under one already released, so whoever changes the interfaces does - # nothing more. It pushes with the LiveKit writer app's token: a push with - # GITHUB_TOKEN would leave the PR's checks waiting for approval. A fork's - # branch cannot be pushed to: there, the test workflow's check prints the - # command instead. + # nothing more. A fork's branch cannot be pushed to: there, the test + # workflow's check prints the command instead. bump: if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest @@ -26,7 +25,6 @@ jobs: with: ref: ${{ github.head_ref }} fetch-depth: 0 - persist-credentials: false # A version not yet released, or released with these interfaces, stands. # Otherwise the next minor after the latest release: before v1, any @@ -41,13 +39,6 @@ jobs: latest=$(git tag -l 'objects/v*' --sort=-v:refname | head -1) echo "version=$(echo "${latest#objects/}" | awk -F. '{print $1 "." $2+1 ".0"}')" >> "$GITHUB_OUTPUT" - - id: app - if: steps.next.outputs.version != '' - uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 - with: - app-id: ${{ vars.REPO_WRITER_GH_APP_ID }} - private-key: ${{ secrets.REPO_WRITER_GH_APP_PRIVATE_KEY }} - - name: Set up Go if: steps.next.outputs.version != '' uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -55,19 +46,20 @@ jobs: go-version-file: "go.mod" cache: false + # The push is this workflow's token's, which leaves the PR's own runs + # for the new commit waiting for approval; the checks are started here + # instead, and count for that commit. - name: Commit the version if: steps.next.outputs.version != '' env: NEXT: ${{ steps.next.outputs.version }} BRANCH: ${{ github.head_ref }} - REPO: ${{ github.repository }} - GH_TOKEN: ${{ steps.app.outputs.token }} - APP: ${{ steps.app.outputs.app-slug }} + GH_TOKEN: ${{ github.token }} run: | go mod edit -require=github.com/livekit/storage/objects@$NEXT (cd e2e && go mod edit -require=github.com/livekit/storage/objects@$NEXT) - # The app's own identity, so the commit is attributed to it. - git config user.name "$APP[bot]" - git config user.email "$(gh api "/users/$APP%5Bbot%5D" --jq .id)+$APP[bot]@users.noreply.github.com" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git commit -am "objects: $NEXT" - git push "https://x-access-token:$GH_TOKEN@github.com/$REPO.git" "HEAD:$BRANCH" + git push origin "HEAD:$BRANCH" + gh workflow run test.yaml --ref "$BRANCH" diff --git a/README.md b/README.md index 8baac6a..4d85384 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Why `objects` is its own module: How its versions work: - `go.mod` names the `objects` version the root requires; `go.work` builds the repo against `objects/` in the tree. -- A PR that changes `objects/` under a released version gets the next minor version committed to it by `objects-version.yaml`; pull that commit before pushing again. From a fork, CI fails and prints the command. +- A PR that changes `objects/` under a released version gets the next minor version committed to it by `objects-version.yaml`, which also runs the checks on that commit; the PR's own runs for it wait for an approval nobody needs to give. Pull that commit before pushing again. From a fork, CI fails and prints the command. - Until the version is tagged, `go mod tidy` in the root fails on it (unknown revision); `go.work` builds the repo meanwhile. - On merge, `objects-release.yaml` tags `objects/vX.Y.Z` on main; a failed run can be rerun by hand. Squash merges drop a branch's commits, so only a version tagged on main is sure to stay resolvable. - `test` must be a required check on main with branches up to date: two PRs bumped to the same version cannot both merge then.