diff --git a/.github/workflows/objects-release.yaml b/.github/workflows/objects-release.yaml new file mode 100644 index 0000000..b1835a6 --- /dev/null +++ b/.github/workflows/objects-release.yaml @@ -0,0 +1,36 @@ +name: objects release + +permissions: + contents: write + +on: + push: + branches: [main] + # A failed run is retried by hand; GitHub emails the failure to whoever + # merged. + workflow_dispatch: + +concurrency: + group: objects-release + cancel-in-progress: false + +jobs: + # Tags the objects version go.mod requires on this commit, so main never + # requires one that does not exist. An existing tag is left as it is: the + # test workflow kept objects/ from changing under it. + tag: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Tag objects + run: | + tag="objects/$(awk '$1 == "github.com/livekit/storage/objects" {print $2}' go.mod)" + if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then + git diff --quiet "refs/tags/$tag" HEAD -- objects || { echo "::error::objects/ differs from $tag"; exit 1; } + exit 0 + fi + git tag "$tag" HEAD + git push origin "$tag" diff --git a/.github/workflows/objects-version.yaml b/.github/workflows/objects-version.yaml new file mode 100644 index 0000000..e10b568 --- /dev/null +++ b/.github/workflows/objects-version.yaml @@ -0,0 +1,65 @@ +name: objects version + +permissions: + contents: write + actions: write + +on: + pull_request: + paths: ["objects/**"] + +concurrency: + group: objects-version-${{ github.event.pull_request.number }} + cancel-in-progress: false + +jobs: + # Raises the objects version go.mod requires when a PR changes objects/ + # under one already released, so whoever changes the interfaces does + # nothing more. A fork's branch cannot be pushed to: there, the test + # workflow's check prints the command instead. + bump: + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.head_ref }} + fetch-depth: 0 + + # A version not yet released, or released with these interfaces, stands. + # Otherwise the next minor after the latest release: before v1, any + # change may break a caller. + - name: Next objects version + id: next + run: | + v=$(awk '$1 == "github.com/livekit/storage/objects" {print $2}' go.mod) + if ! git rev-parse -q --verify "refs/tags/objects/$v" >/dev/null || git diff --quiet "refs/tags/objects/$v" HEAD -- objects; then + exit 0 + fi + latest=$(git tag -l 'objects/v*' --sort=-v:refname | head -1) + echo "version=$(echo "${latest#objects/}" | awk -F. '{print $1 "." $2+1 ".0"}')" >> "$GITHUB_OUTPUT" + + - name: Set up Go + if: steps.next.outputs.version != '' + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: "go.mod" + cache: false + + # The push is this workflow's token's, which leaves the PR's own runs + # for the new commit waiting for approval; the checks are started here + # instead, and count for that commit. + - name: Commit the version + if: steps.next.outputs.version != '' + env: + NEXT: ${{ steps.next.outputs.version }} + BRANCH: ${{ github.head_ref }} + GH_TOKEN: ${{ github.token }} + run: | + go mod edit -require=github.com/livekit/storage/objects@$NEXT + (cd e2e && go mod edit -require=github.com/livekit/storage/objects@$NEXT) + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -am "objects: $NEXT" + git push origin "HEAD:$BRANCH" + gh workflow run test.yaml --ref "$BRANCH" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f2b7641..f8e6d24 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,31 +15,95 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: | - ~/go/pkg/mod - ~/go/bin - ~/.cache - key: livekit-storage - + fetch-depth: 0 # the objects/ tags + # setup-go caches the module and build caches, keyed on both modules. - name: Set up Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: "go.mod" + cache-dependency-path: | + go.sum + e2e/go.sum - name: Set up gotestfmt run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@9eae5abc81d6d08f73268741ef4a8b97f29b60d8 # v2.4.1 - name: Download Go modules - run: go mod download + run: go mod download && (cd e2e && go mod download) + + # objects/ is released by tag: objects-release.yaml tags the version + # go.mod requires on main after the merge, and go.work builds this tree + # against objects/ until then. A tag never moves, so once it exists, a + # change to objects/ needs the next version. + - name: objects version names these interfaces + run: | + v=$(awk '$1 == "github.com/livekit/storage/objects" {print $2}' go.mod) + if ! [[ "$v" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::go.mod requires github.com/livekit/storage/objects at $v: name a release, such as v0.2.0" + exit 1 + fi + if git rev-parse -q --verify "refs/tags/objects/$v" >/dev/null && ! git diff --quiet "refs/tags/objects/$v" HEAD -- objects; then + # The next minor after the latest release: before v1, any change + # may break a caller. + latest=$(git tag -l 'objects/v*' --sort=-v:refname | head -1) + next=$(echo "${latest#objects/}" | awk -F. '{print $1 "." $2+1 ".0"}') + echo "::error::objects/ changed since objects/$v. Run: go mod edit -require=github.com/livekit/storage/objects@$next" + exit 1 + fi - name: Lint uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 with: version: v2.14.0 + - name: Lint e2e + uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 + with: + version: v2.14.0 + working-directory: e2e + + # The e2e module holds the suites: here they run on the local backend. The + # real-account tests skip, as CI passes them no credentials. - name: Test run: | set -euo pipefail go test -race -json -v ./... 2>&1 | gotestfmt + (cd objects && go vet ./...) + cd e2e && go test -race -json -v -skip '^Test(Floci|MinIO)' ./... 2>&1 | gotestfmt + + # Every backend end to end against its floci emulator (e2e/floci_test.go), + # and S3 against MinIO (e2e/minio_test.go), whose conditional writes are + # atomic where floci's are not. One check each; TestWorkflowRunsEveryEmulatorTest + # fails when a test is missing here. + e2e: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test: [FlociS3, FlociAzure, FlociGCP, FlociOCI, MinIO] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # setup-go caches the module and build caches, keyed on both modules. + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: "go.mod" + cache-dependency-path: | + go.sum + e2e/go.sum + + - name: Set up gotestfmt + run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@9eae5abc81d6d08f73268741ef4a8b97f29b60d8 # v2.4.1 + + - name: Install MinIO + if: matrix.test == 'MinIO' + run: | + go install github.com/minio/minio@v0.0.0-20260212201848-7aac2a2c5b7c + echo "MINIO_BIN=$(go env GOPATH)/bin/minio" >> "$GITHUB_ENV" + + - name: Test + working-directory: e2e + run: | + set -euo pipefail + go test -race -json -v -run '^Test${{ matrix.test }}$' ./... 2>&1 | gotestfmt diff --git a/.gitignore b/.gitignore index b9b720f..db11270 100644 --- a/.gitignore +++ b/.gitignore @@ -17,10 +17,6 @@ # Dependency directories (remove the comment below to include it) # vendor/ -# Go workspace file -go.work -go.work.sum - # env file .env* diff --git a/README.md b/README.md index ed031d9..4d85384 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,47 @@ # Storage -A GORM-style storage wrapper with unified API for different cloud storage backends +Object storage on S3 and S3-compatible stores, GCS, Azure Blob, OCI Object Storage, AliOSS and a local directory, behind one interface. `storage.New(conf)` returns a `Storage`; every call takes a context. + +- `Bucket` (`storage/objects`): `PutObject`, `UploadObject` (streams), `GetObject`, `GetObjectRange`, `DownloadObject` (to any `io.Writer`), `ObjectExists`, `ListObjects` (byte order), `DeleteObjects`, `PresignedURL`. +- `Storage` is `Bucket` plus `UploadFile` and `DownloadFile`. `DownloadFile` replaces the local file only once the whole object is read. +- `PutOptions{ContentType, IfAbsent}` on every write. `IfAbsent` refuses to replace an object, atomically where the store honors the condition. +- Errors: a missing object is `ErrNotFound`, a write `IfAbsent` refuses is `ErrExists` (`errors.Is`); `ErrorWithStatusCode` carries the store's HTTP status and `DeleteError` the paths a delete left (`errors.As`). +- `storage/objects` holds the interfaces in a module of its own, with no requirements (see Modules). + +## Behavior + +- **S3 semantics** where stores differ. `DeleteObjects` of a missing path succeeds, as S3's delete does. It tries every path, and when some are not deleted fails with a `*DeleteError` whose `Failed` names each one with its error (`errors.As`); every other path is deleted, and a retry is safe. +- **Keys** are the store's keys as given. The local directory, as a filesystem, drops a leading `/`, cleans `//` and `.`, refuses keys outside it, and cannot hold both `a` and `a/b`. +- **Timeouts**: a connection that moves no bytes either way for 5 minutes is cut; a slow but live transfer never is. The context bounds the call. +- **Retries**: every store retries throttling, 5xx and network errors with backoff, including AliOSS, whose SDK does not. A stream given to `UploadObject` is sent once on AliOSS. +- **Proxy**: `ProxyConfig` (S3, GCS) carries its credentials for http and https targets, honors `NO_PROXY`, and never proxies the cloud metadata services. +- **Local writes** go in place, as a file is written, so mounted volumes and file watchers see one write; `IfAbsent` writes a temporary file (`.upload-*`, never listed) and links it into place. + +## Modules + +| module | holds | requires | +|---|---|---| +| `github.com/livekit/storage` | the backends, `New`, the configs | every provider SDK, `livekit/protocol` | +| `github.com/livekit/storage/objects` | `Bucket`, `Storage`, `PutOptions`, the errors | nothing | +| `github.com/livekit/storage/e2e` | the end-to-end suites | `dockertest` | + +Why `objects` is its own module: + +- A module's requirements take part in the version selection of every module that requires it, whichever packages it imports. In the root module, the interfaces would carry every SDK's and `livekit/protocol`'s versions (grpc, otel) into each fake or library that implements or accepts them. +- With no requirements, implementing them costs no dependency: gosim's `fakes/objstore` `Bucket` is an `objects.Bucket`. + +How its versions work: + +- `go.mod` names the `objects` version the root requires; `go.work` builds the repo against `objects/` in the tree. +- A PR that changes `objects/` under a released version gets the next minor version committed to it by `objects-version.yaml`, which also runs the checks on that commit; the PR's own runs for it wait for an approval nobody needs to give. Pull that commit before pushing again. From a fork, CI fails and prints the command. +- Until the version is tagged, `go mod tidy` in the root fails on it (unknown revision); `go.work` builds the repo meanwhile. +- On merge, `objects-release.yaml` tags `objects/vX.Y.Z` on main; a failed run can be rerun by hand. Squash merges drop a branch's commits, so only a version tagged on main is sure to stay resolvable. +- `test` must be a required check on main with branches up to date: two PRs bumped to the same version cannot both merge then. + +## Tests + +The suites live in the `e2e` module, so this module's users do not inherit Docker's dependencies. + +- `go test ./...` here runs the unit tests. +- `cd e2e && go test ./...` runs every backend except AliOSS end to end: against [floci](https://github.com/floci-io) emulators in Docker (S3, Azure Blob, GCS, OCI Object Storage), against MinIO for S3's conditional writes (`MINIO_BIN` or `minio` on `PATH`), and on the local directory. Without Docker or MinIO those tests skip, except when `CI` is set. +- The provider tests (`TestS3`, `TestGCP`, ...) reach real accounts when their environment variables are set; AliOSS has no emulator and is covered only that way. diff --git a/alioss.go b/alioss.go index e420edf..41dbbdd 100644 --- a/alioss.go +++ b/alioss.go @@ -16,11 +16,18 @@ package storage import ( "bytes" + "context" "errors" + "fmt" "io" + "math/rand/v2" + "net" + "net/http" "net/url" "os" + "slices" "strings" + "syscall" "time" "github.com/aliyun/aliyun-oss-go-sdk/oss" @@ -69,25 +76,16 @@ func NewAliOSS(conf *AliOSSConfig) (Storage, error) { }, nil } -func (s *aliOSSStorage) UploadData(data []byte, storagePath, contentType string) (string, int64, error) { - reader := bytes.NewBuffer(data) - if err := s.bucket.PutObject(storagePath, reader, oss.ContentType(contentType)); err != nil { - return "", 0, wrapAliOSSError(err) - } - - return s.location(storagePath), int64(len(data)), nil -} - -func (s *aliOSSStorage) UploadFile(filepath, storagePath, contentType string) (string, int64, error) { +func (s *aliOSSStorage) UploadFile(ctx context.Context, storagePath, filepath string, opts PutOptions) (string, int64, error) { info, err := os.Stat(filepath) if err != nil { return "", 0, err } - - if err = s.bucket.PutObjectFromFile(storagePath, filepath, oss.ContentType(contentType)); err != nil { - return "", 0, wrapAliOSSError(err) + if err = aliOSSRetry(ctx, func() error { + return s.bucket.PutObjectFromFile(storagePath, filepath, aliOSSWriteOptions(ctx, opts)...) + }); err != nil { + return "", 0, wrapAliOSSObjectError(err) } - return s.location(storagePath), info.Size(), nil } @@ -98,68 +96,215 @@ func (s *aliOSSStorage) location(storagePath string) string { return u.String() } -func (s *aliOSSStorage) ListObjects(prefix string) ([]string, error) { - var objects []string - marker := oss.Marker("") - for { - lor, err := s.bucket.ListObjects(oss.Prefix(prefix), marker) - if err != nil { - return nil, wrapAliOSSError(err) - } +func (s *aliOSSStorage) DownloadFile(ctx context.Context, storagePath, filepath string) (int64, error) { + return downloadFile(filepath, func(f *os.File) (int64, error) { + return s.download(ctx, storagePath, f) + }) +} - for _, object := range lor.Objects { - objects = append(objects, object.Key) - } +func (s *aliOSSStorage) PresignedURL(_ context.Context, storagePath string, expiration time.Duration) (string, error) { + u, err := s.bucket.SignURL(storagePath, oss.HTTPGet, int64(expiration.Seconds())) + if err != nil { + return "", wrapAliOSSError(err) + } + return u, nil +} - if !lor.IsTruncated { - break - } - marker = oss.Marker(lor.NextMarker) +func (s *aliOSSStorage) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + return aliOSSRetry(ctx, func() error { + return s.UploadObject(ctx, storagePath, bytes.NewReader(data), opts) + }) +} + +// UploadObject streams body in one PutObject, chunked when its length is +// unknown, so it holds no part in memory and caps an object at 5 GB. It is +// sent once, as the body is read as it goes. IfAbsent sets +// x-oss-forbid-overwrite, atomic here, but OSS ignores it on a bucket with +// versioning enabled or suspended. +func (s *aliOSSStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + return wrapAliOSSObjectError(s.bucket.PutObject(storagePath, body, aliOSSWriteOptions(ctx, opts)...)) +} + +// aliOSSWriteOptions leaves out an empty content type, which the SDK then +// takes from the key's extension. +func aliOSSWriteOptions(ctx context.Context, opts PutOptions) []oss.Option { + options := []oss.Option{oss.WithContext(ctx)} + if opts.ContentType != "" { + options = append(options, oss.ContentType(opts.ContentType)) } + if opts.IfAbsent { + options = append(options, oss.ForbidOverWrite(true)) + } + return options +} - return objects, nil +func (s *aliOSSStorage) GetObject(ctx context.Context, storagePath string) ([]byte, error) { + return getObject(ctx, s, storagePath) } -func (s *aliOSSStorage) DownloadData(storagePath string) ([]byte, error) { - reader, err := s.bucket.GetObject(storagePath) +// GetObjectRange asks for standard range behavior: by default OSS answers an +// unsatisfiable range with the whole object. +func (s *aliOSSStorage) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + var buf bytes.Buffer + _, err := s.download(ctx, storagePath, &buf, + oss.Range(offset, offset+length-1), oss.RangeBehavior("standard")) + var svcErr oss.ServiceError + if errors.As(err, &svcErr) && svcErr.StatusCode == http.StatusRequestedRangeNotSatisfiable { + return []byte{}, nil + } if err != nil { - return nil, wrapAliOSSError(err) + return nil, err } - defer reader.Close() + return buf.Bytes(), nil +} + +func (s *aliOSSStorage) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { + return s.download(ctx, storagePath, w) +} - data, err := io.ReadAll(reader) +// download copies the object to w and, for a whole object, checks the CRC64 +// OSS sends against what arrived, as the SDK's GetObjectToFile does. +func (s *aliOSSStorage) download(ctx context.Context, storagePath string, w io.Writer, options ...oss.Option) (int64, error) { + var result *oss.GetObjectResult + err := aliOSSRetry(ctx, func() (err error) { + result, err = s.bucket.DoGetObject(&oss.GetObjectRequest{ObjectKey: storagePath}, append(options, oss.WithContext(ctx))) + return err + }) + if err != nil { + return 0, wrapAliOSSObjectError(err) + } + defer result.Response.Close() + n, err := io.Copy(w, result.Response.Body) if err != nil { - return nil, wrapAliOSSError(err) + return n, wrapAliOSSObjectError(err) + } + if result.ClientCRC != nil { + result.Response.ClientCRC = result.ClientCRC.Sum64() + err = oss.CheckCRC(result.Response, "GetObject") } - return data, nil + return n, err } -func (s *aliOSSStorage) DownloadFile(filepath, storagePath string) (int64, error) { - if err := s.bucket.GetObjectToFile(storagePath, filepath); err != nil { - return 0, wrapAliOSSError(err) - } +func (s *aliOSSStorage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + var exists bool + err := aliOSSRetry(ctx, func() (err error) { + exists, err = s.bucket.IsObjectExist(storagePath, oss.WithContext(ctx)) + return err + }) + return exists, wrapAliOSSObjectError(err) +} - info, err := os.Stat(filepath) - if err != nil { - return 0, err +func (s *aliOSSStorage) ListObjects(ctx context.Context, prefix string) ([]string, error) { + var keys []string + options := []oss.Option{oss.Prefix(prefix), oss.MaxKeys(1000), oss.WithContext(ctx)} + for { + var res oss.ListObjectsResultV2 + err := aliOSSRetry(ctx, func() (err error) { + res, err = s.bucket.ListObjectsV2(options...) + return err + }) + if err != nil { + return nil, wrapAliOSSObjectError(err) + } + for _, obj := range res.Objects { + keys = append(keys, obj.Key) + } + if !res.IsTruncated || res.NextContinuationToken == "" { + slices.Sort(keys) + return keys, nil + } + options = []oss.Option{oss.Prefix(prefix), oss.MaxKeys(1000), oss.ContinuationToken(res.NextContinuationToken), oss.WithContext(ctx)} } +} - return info.Size(), nil +// DeleteObjects sends batches of 1000, the most one request takes. OSS reports +// no per-key errors, only the keys it deleted, so a key it leaves out fails the +// call unless it holds nothing. +func (s *aliOSSStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { + failed := map[string]error{} + for i := 0; i < len(storagePaths); i += 1000 { + batch := storagePaths[i:min(i+1000, len(storagePaths))] + var res oss.DeleteObjectsResult + err := aliOSSRetry(ctx, func() (err error) { + res, err = s.bucket.DeleteObjects(batch, oss.WithContext(ctx)) + return err + }) + if err != nil { + for _, k := range batch { + failed[k] = wrapAliOSSObjectError(err) + } + continue + } + deleted := make(map[string]bool, len(res.DeletedObjects)) + for _, k := range res.DeletedObjects { + deleted[k] = true + } + for _, k := range batch { + if deleted[k] { + continue + } + exists, err := s.ObjectExists(ctx, k) + if err == nil && exists { + err = errors.New("delete not confirmed") + } + if err != nil { + failed[k] = err + } + } + } + return deleteError(failed) } -func (s *aliOSSStorage) GeneratePresignedUrl(storagePath string, expiration time.Duration) (string, error) { - u, err := s.bucket.SignURL(storagePath, oss.HTTPGet, int64(expiration.Seconds())) - if err != nil { - return "", wrapAliOSSError(err) +// aliOSSAttempts is how many times a call is sent: the v1 SDK never retries. +const aliOSSAttempts = 3 + +// aliOSSRetry runs call again after a 5xx, a 429 or a network error, with +// backoff, until it succeeds or aliOSSAttempts is spent. Only calls whose body, +// if any, is sent anew each time are retried. +func aliOSSRetry(ctx context.Context, call func() error) error { + var err error + for attempt := range aliOSSAttempts { + if err = call(); err == nil || ctx.Err() != nil || !aliOSSRetryable(err) { + return err + } + backoff := 200 * time.Millisecond << attempt + select { + case <-ctx.Done(): + return err + case <-time.After(backoff/2 + rand.N(backoff)): + } } - return u, nil + return err } -func (s *aliOSSStorage) DeleteObject(storagePath string) error { - return wrapAliOSSError(s.bucket.DeleteObject(storagePath)) +// aliOSSRetryable is a failure another attempt can get past: OSS throttling or +// a 5xx, or a connection that broke. A 4xx, as for a denied or bad request, +// would fail again. +func aliOSSRetryable(err error) bool { + var svcErr oss.ServiceError + if errors.As(err, &svcErr) { + return svcErr.StatusCode >= 500 || svcErr.StatusCode == http.StatusTooManyRequests + } + var netErr net.Error + return errors.As(err, &netErr) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.ECONNRESET) } -func (s *aliOSSStorage) DeleteObjects(storagePaths []string) error { - _, err := s.bucket.DeleteObjects(storagePaths) +func wrapAliOSSObjectError(err error) error { + if err == nil { + return nil + } + var svcErr oss.ServiceError + if errors.As(err, &svcErr) { + switch { + // A HEAD answers a missing key with a bare 404. + case svcErr.Code == "NoSuchKey" || (svcErr.StatusCode == http.StatusNotFound && svcErr.Code == ""): + return fmt.Errorf("%w: %w", ErrNotFound, wrapAliOSSError(err)) + case svcErr.Code == "FileAlreadyExists": + return fmt.Errorf("%w: %w", ErrExists, wrapAliOSSError(err)) + } + } return wrapAliOSSError(err) } diff --git a/azure.go b/azure.go index 375ec8e..05a018b 100644 --- a/azure.go +++ b/azure.go @@ -15,18 +15,25 @@ package storage import ( + "bytes" "context" "errors" "fmt" + "io" + "net/http" "net/url" "os" "path" + "slices" + "strings" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" ) @@ -51,192 +58,308 @@ const azureParallelism = 16 type azureBLOBStorage struct { conf *AzureConfig client *azblob.Client - serviceUrl string + cred *azblob.SharedKeyCredential + serviceURL string + options policy.ClientOptions } +// NewAzure signs requests with the account key, or with TokenCredential +// where no key is set, as with a managed or workload identity. func NewAzure(conf *AzureConfig) (Storage, error) { - cred, err := azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey) - if err != nil { - return nil, err + serviceURL := azureServiceURL(conf) + if _, err := url.Parse(serviceURL); err != nil { + return nil, fmt.Errorf("azure endpoint: %w", err) + } + s := &azureBLOBStorage{ + conf: conf, + serviceURL: serviceURL, + // azcore's own transport has no idle timeout, and keeps fewer idle + // connections than an upload sends blocks at once. + options: policy.ClientOptions{Transport: &http.Client{Transport: transportWithTimeouts(http.DefaultTransport)}}, } - serviceUrl := fmt.Sprintf("https://%s.blob.core.windows.net/", conf.AccountName) - client, err := azblob.NewClientWithSharedKeyCredential(serviceUrl, cred, nil) - if err != nil { + var err error + if conf.AccountKey == "" && conf.TokenCredential != nil { + s.client, err = azblob.NewClient(serviceURL, conf.TokenCredential, &azblob.ClientOptions{ClientOptions: s.options}) + return s, err + } + if s.cred, err = azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey); err != nil { return nil, err } - - return &azureBLOBStorage{ - conf: conf, - client: client, - serviceUrl: serviceUrl, - }, nil + s.client, err = azblob.NewClientWithSharedKeyCredential(serviceURL, s.cred, &azblob.ClientOptions{ClientOptions: s.options}) + return s, err } -func (s *azureBLOBStorage) location(storagePath string) string { - return (&url.URL{ - Scheme: "https", - Host: fmt.Sprintf("%s.blob.core.windows.net", s.conf.AccountName), - Path: path.Join(s.conf.ContainerName, storagePath), - }).String() +func azureServiceURL(conf *AzureConfig) string { + if conf.Endpoint != "" { + return conf.Endpoint + } + return fmt.Sprintf("https://%s.blob.core.windows.net/", conf.AccountName) } -func (s *azureBLOBStorage) UploadData(data []byte, storagePath, contentType string) (string, int64, error) { - _, err := s.client.UploadBuffer(context.Background(), s.conf.ContainerName, storagePath, data, &azblob.UploadBufferOptions{ - HTTPHeaders: &blob.HTTPHeaders{BlobContentType: &contentType}, - BlockSize: azureBlockSize, - Concurrency: azureParallelism, - }) - if err != nil { - return "", 0, wrapAzureError(err) - } - return s.location(storagePath), int64(len(data)), nil +// blobURL is storagePath under the service URL, path-style below an +// emulator's account segment. NewAzure has checked the URL parses. +func (s *azureBLOBStorage) blobURL(storagePath string) *url.URL { + u, _ := url.Parse(azureServiceURL(s.conf)) + u.Path = path.Join(u.Path, s.conf.ContainerName, storagePath) + return u } -func (s *azureBLOBStorage) UploadFile(filepath, storagePath, contentType string) (string, int64, error) { +func (s *azureBLOBStorage) UploadFile(ctx context.Context, storagePath, filepath string, opts PutOptions) (string, int64, error) { file, err := os.Open(filepath) if err != nil { return "", 0, err } - defer func() { - _ = file.Close() - }() + defer file.Close() stat, err := file.Stat() if err != nil { return "", 0, err } - - _, err = s.client.UploadFile(context.Background(), s.conf.ContainerName, storagePath, file, &azblob.UploadFileOptions{ - HTTPHeaders: &blob.HTTPHeaders{BlobContentType: &contentType}, - BlockSize: azureBlockSize, - Concurrency: azureParallelism, + headers, conditions := azureWriteOptions(opts) + _, err = s.client.UploadFile(ctx, s.conf.ContainerName, storagePath, file, &azblob.UploadFileOptions{ + HTTPHeaders: headers, + AccessConditions: conditions, + BlockSize: azureBlockSize, + Concurrency: azureParallelism, }) if err != nil { - return "", 0, wrapAzureError(err) + return "", 0, wrapAzureWriteError(err, opts.IfAbsent) } + return s.blobURL(storagePath).String(), stat.Size(), nil +} - return s.location(storagePath), stat.Size(), nil +func (s *azureBLOBStorage) DownloadFile(ctx context.Context, storagePath, filepath string) (int64, error) { + return downloadFile(filepath, func(f *os.File) (int64, error) { + n, err := s.client.DownloadFile(ctx, s.conf.ContainerName, storagePath, f, &azblob.DownloadFileOptions{ + BlockSize: azureBlockSize, + Concurrency: azureParallelism, + }) + return n, wrapAzureObjectError(err) + }) } -func (s *azureBLOBStorage) ListObjects(prefix string) ([]string, error) { - var objects []string +func (s *azureBLOBStorage) presignWithUserDelegation(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { + if s.conf.TokenCredential == nil { + return "", errors.New("OAuth required") + } - pager := s.client.NewListBlobsFlatPager(s.conf.ContainerName, &azblob.ListBlobsFlatOptions{ - Prefix: &prefix, - }) - for pager.More() { - page, err := pager.NextPage(context.Background()) - if err != nil { - return nil, wrapAzureError(err) - } - for _, item := range page.Segment.BlobItems { - if item.Name != nil { - objects = append(objects, *item.Name) - } - } + now := time.Now().UTC() + exp := now.Add(expiration) + + svcClient, err := service.NewClient(s.serviceURL, s.conf.TokenCredential, &service.ClientOptions{ClientOptions: s.options}) + if err != nil { + return "", err + } + + // The key starts in the past and the SAS takes no start time, as Azure + // advises: a host clock ahead of Azure's would make the URL not valid yet. + udc, err := svcClient.GetUserDelegationCredential(ctx, service.KeyInfo{ + Start: to.Ptr(now.Add(-15 * time.Minute).Format(sas.TimeFormat)), + Expiry: to.Ptr(exp.Format(sas.TimeFormat)), + }, nil) + if err != nil { + return "", wrapAzureError(err) + } + + qp, err := sas.BlobSignatureValues{ + Protocol: sas.ProtocolHTTPS, + ExpiryTime: exp, + Permissions: (&sas.BlobPermissions{Read: true}).String(), + ContainerName: s.conf.ContainerName, + BlobName: storagePath, + }.SignWithUserDelegation(udc) + if err != nil { + return "", err } - return objects, nil + loc := s.blobURL(storagePath) + loc.RawQuery = qp.Encode() + return loc.String(), nil } -func (s *azureBLOBStorage) DownloadData(storagePath string) ([]byte, error) { - ctx := context.Background() - blobClient := s.client.ServiceClient().NewContainerClient(s.conf.ContainerName).NewBlobClient(storagePath) +// PutObject sends data up to 256 MiB in one Put Blob, IfAbsent as its +// condition, and larger data in blocks. +func (s *azureBLOBStorage) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + headers, conditions := azureWriteOptions(opts) + _, err := s.client.UploadBuffer(ctx, s.conf.ContainerName, storagePath, data, &azblob.UploadBufferOptions{ + BlockSize: azureBlockSize, + Concurrency: azureParallelism, + HTTPHeaders: headers, + AccessConditions: conditions, + }) + return wrapAzureWriteError(err, opts.IfAbsent) +} - props, err := blobClient.GetProperties(ctx, nil) - if err != nil { - return nil, wrapAzureError(err) +func azureWriteOptions(opts PutOptions) (*blob.HTTPHeaders, *blob.AccessConditions) { + var headers *blob.HTTPHeaders + if opts.ContentType != "" { + headers = &blob.HTTPHeaders{BlobContentType: to.Ptr(opts.ContentType)} } - if props.ContentLength == nil { - return nil, errors.New("azure: missing content length") + var conditions *blob.AccessConditions + if opts.IfAbsent { + conditions = &blob.AccessConditions{ + ModifiedAccessConditions: &blob.ModifiedAccessConditions{IfNoneMatch: to.Ptr(azcore.ETagAny)}, + } } + return headers, conditions +} - buf := make([]byte, *props.ContentLength) - _, err = s.client.DownloadBuffer(ctx, s.conf.ContainerName, storagePath, buf, &azblob.DownloadBufferOptions{ - BlockSize: azureBlockSize, - Concurrency: azureParallelism, +// UploadObject stages blocks and commits them; IfAbsent is a condition on the +// commit, so it is atomic here. The check before staging only spares a refused +// upload its traffic. +func (s *azureBLOBStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + if opts.IfAbsent && presentBeforeUpload(ctx, s, storagePath) { + return ErrExists + } + headers, conditions := azureWriteOptions(opts) + _, err := s.client.UploadStream(ctx, s.conf.ContainerName, storagePath, body, &azblob.UploadStreamOptions{ + BlockSize: azureBlockSize, + Concurrency: azureParallelism, + HTTPHeaders: headers, + AccessConditions: conditions, }) + return wrapAzureWriteError(err, opts.IfAbsent) +} + +func (s *azureBLOBStorage) GetObject(ctx context.Context, storagePath string) ([]byte, error) { + return getObject(ctx, s, storagePath) +} + +func (s *azureBLOBStorage) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + var buf bytes.Buffer + _, err := s.download(ctx, storagePath, &buf, blob.HTTPRange{Offset: offset, Count: length}) + if bloberror.HasCode(err, bloberror.InvalidRange) { + return []byte{}, nil + } if err != nil { - return nil, wrapAzureError(err) + return nil, err } - return buf, nil + return buf.Bytes(), nil +} + +func (s *azureBLOBStorage) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { + return s.download(ctx, storagePath, w, blob.HTTPRange{}) } -func (s *azureBLOBStorage) DownloadFile(filepath, storagePath string) (int64, error) { - file, err := os.Create(filepath) +func (s *azureBLOBStorage) download(ctx context.Context, storagePath string, w io.Writer, r blob.HTTPRange) (int64, error) { + resp, err := s.client.DownloadStream(ctx, s.conf.ContainerName, storagePath, &azblob.DownloadStreamOptions{Range: r}) if err != nil { - return 0, err + return 0, wrapAzureObjectError(err) } - defer file.Close() + body := resp.NewRetryReader(ctx, &blob.RetryReaderOptions{MaxRetries: 3}) + defer body.Close() + n, err := io.Copy(w, body) + return n, wrapAzureObjectError(err) +} - _, err = s.client.DownloadFile(context.Background(), s.conf.ContainerName, storagePath, file, &azblob.DownloadFileOptions{ - BlockSize: azureBlockSize, - Concurrency: azureParallelism, - }) +// ObjectExists reports a directory of an account with a hierarchical namespace +// (ADLS Gen2) as no object, as a store reports a prefix. +func (s *azureBLOBStorage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + blobClient := s.client.ServiceClient().NewContainerClient(s.conf.ContainerName).NewBlobClient(storagePath) + props, err := blobClient.GetProperties(ctx, nil) + if bloberror.HasCode(err, bloberror.BlobNotFound) { + return false, nil + } if err != nil { - return 0, wrapAzureError(err) + return false, wrapAzureObjectError(err) } + return !azureFolder(props.Metadata), nil +} - stat, err := file.Stat() - if err != nil { - return 0, err +// azureFolder reports the metadata a hierarchical namespace gives its +// directories. Metadata keys come back in any case. +func azureFolder(metadata map[string]*string) bool { + for k, v := range metadata { + if strings.EqualFold(k, "hdi_isfolder") && v != nil && strings.EqualFold(*v, "true") { + return true + } } - return stat.Size(), nil + return false } -func (s *azureBLOBStorage) GeneratePresignedUrl(storagePath string, expiration time.Duration) (string, error) { - if s.conf.TokenCredential == nil { - return "", errors.New("OAuth required") +func (s *azureBLOBStorage) ListObjects(ctx context.Context, prefix string) ([]string, error) { + // With metadata, to leave out a hierarchical namespace's directories. + pager := s.client.NewListBlobsFlatPager(s.conf.ContainerName, &azblob.ListBlobsFlatOptions{ + Prefix: &prefix, + Include: azblob.ListBlobsInclude{Metadata: true}, + }) + var keys []string + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, wrapAzureObjectError(err) + } + for _, item := range page.Segment.BlobItems { + if item.Name != nil && !azureFolder(item.Metadata) { + keys = append(keys, *item.Name) + } + } } + slices.Sort(keys) + return keys, nil +} - now := time.Now().UTC() - exp := now.Add(expiration) +// DeleteObjects fails on a blob with snapshots, as on main: deleting someone's +// snapshots along with it is not this call's to decide. +func (s *azureBLOBStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { + return deleteEach(ctx, storagePaths, func(ctx context.Context, p string) error { + _, err := s.client.DeleteBlob(ctx, s.conf.ContainerName, p, nil) + if err != nil && !bloberror.HasCode(err, bloberror.BlobNotFound) { + return wrapAzureObjectError(err) + } + return nil + }) +} - svcClient, err := service.NewClient(s.serviceUrl, s.conf.TokenCredential, nil) - if err != nil { - return "", err +// PresignedURL signs with the account key unless a token credential is +// set, in which case it signs with a user delegation key as Storage does. +func (s *azureBLOBStorage) PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { + if s.conf.TokenCredential != nil { + return s.presignWithUserDelegation(ctx, storagePath, expiration) } - - udc, err := svcClient.GetUserDelegationCredential(context.Background(), service.KeyInfo{ - Start: to.Ptr(now.Format(sas.TimeFormat)), - Expiry: to.Ptr(exp.Format(sas.TimeFormat)), - }, nil) - if err != nil { - return "", wrapAzureError(err) + // HTTPS only, as the user delegation SAS, unless the endpoint itself is + // plain HTTP, as an emulator's. + protocol := sas.ProtocolHTTPS + if strings.HasPrefix(s.serviceURL, "http://") { + protocol = sas.ProtocolHTTPSandHTTP } - qp, err := sas.BlobSignatureValues{ - Protocol: sas.ProtocolHTTPS, - StartTime: now, - ExpiryTime: exp, + Protocol: protocol, + ExpiryTime: time.Now().UTC().Add(expiration), Permissions: (&sas.BlobPermissions{Read: true}).String(), ContainerName: s.conf.ContainerName, BlobName: storagePath, - }.SignWithUserDelegation(udc) + }.SignWithSharedKey(s.cred) if err != nil { return "", err } - - loc := &url.URL{ - Scheme: "https", - Host: fmt.Sprintf("%s.blob.core.windows.net", s.conf.AccountName), - Path: path.Join(s.conf.ContainerName, storagePath), - RawQuery: qp.Encode(), - } - return loc.String(), nil + u := s.blobURL(storagePath) + u.RawQuery = qp.Encode() + return u.String(), nil } -func (s *azureBLOBStorage) DeleteObject(storagePath string) error { - _, err := s.client.DeleteBlob(context.Background(), s.conf.ContainerName, storagePath, nil) +func wrapAzureObjectError(err error) error { + switch { + case err == nil: + return nil + case bloberror.HasCode(err, bloberror.BlobNotFound): + return fmt.Errorf("%w: %w", ErrNotFound, wrapAzureError(err)) + } return wrapAzureError(err) } -func (s *azureBLOBStorage) DeleteObjects(storagePaths []string) error { - for _, p := range storagePaths { - if err := s.DeleteObject(p); err != nil { - return err - } +// wrapAzureWriteError is wrapAzureObjectError with a failed condition as +// ErrExists where IfAbsent set one. A read can fail one too: the retry reader +// pins its retries to the first response's ETag. +func wrapAzureWriteError(err error, ifAbsent bool) error { + // InvalidBlockList is a racing upload whose commit discarded this one's + // staged blocks. + if ifAbsent && bloberror.HasCode(err, bloberror.BlobAlreadyExists, bloberror.ConditionNotMet, bloberror.InvalidBlockList) { + return fmt.Errorf("%w: %w", ErrExists, wrapAzureError(err)) } - return nil + return wrapAzureObjectError(err) } diff --git a/config.go b/config.go index cd10d85..725d54b 100644 --- a/config.go +++ b/config.go @@ -34,7 +34,8 @@ type ( ProxyConfig = config.ProxyConfig ) -func newStorage(conf Config) (Storage, error) { +// New builds the Storage a config describes. +func New(conf Config) (Storage, error) { switch c := conf.(type) { case *AliOSSConfig: return NewAliOSS(c) diff --git a/config/config.go b/config/config.go index 7e90a57..7a2c468 100644 --- a/config/config.go +++ b/config/config.go @@ -41,6 +41,9 @@ type AzureConfig struct { AccountKey string `yaml:"account_key,omitempty"` // (env AZURE_STORAGE_KEY) ContainerName string `yaml:"container_name,omitempty"` TokenCredential azcore.TokenCredential `yaml:"-"` // required for presigned url generation + // Endpoint replaces the service URL https://.blob.core.windows.net/, + // e.g. an emulator's http://localhost:4577//. + Endpoint string `yaml:"endpoint,omitempty"` } func (*AzureConfig) storageConfig() {} @@ -49,6 +52,11 @@ type GCPConfig struct { CredentialsJSON string `yaml:"credentials_json,omitempty"` // (env GOOGLE_APPLICATION_CREDENTIALS) Bucket string `yaml:"bucket,omitempty"` ProxyConfig *ProxyConfig `yaml:"proxy_config,omitempty"` + // Endpoint replaces https://storage.googleapis.com, e.g. a Private Service + // Connect endpoint. An http:// one is taken for an emulator, such as + // http://localhost:4588, and gets no credentials; CredentialsJSON still + // signs URLs. + Endpoint string `yaml:"endpoint,omitempty"` } func (*GCPConfig) storageConfig() {} @@ -82,8 +90,8 @@ type S3Config struct { AccessKey string `yaml:"access_key,omitempty"` Secret string `yaml:"secret,omitempty"` SessionToken string `yaml:"session_token,omitempty"` - AssumeRoleArn string `yaml:"assume_role_arn,omitempty"` // ARN of the role to assume for file upload. Egress will make an AssumeRole API call using the provided access_key and secret to assume that role - AssumeRoleExternalId string `yaml:"assume_role_external_id,omitempty"` // ExternalID to use when assuming role for upload + AssumeRoleARN string `yaml:"assume_role_arn,omitempty"` // ARN of the role to assume for file upload. Egress will make an AssumeRole API call using the provided access_key and secret to assume that role + AssumeRoleExternalID string `yaml:"assume_role_external_id,omitempty"` // ExternalID to use when assuming role for upload Region string `yaml:"region,omitempty"` Endpoint string `yaml:"endpoint,omitempty"` Bucket string `yaml:"bucket,omitempty"` @@ -102,7 +110,7 @@ type S3Config struct { func (*S3Config) storageConfig() {} type ProxyConfig struct { - Url string `yaml:"url,omitempty"` + URL string `yaml:"url,omitempty"` Username string `yaml:"username,omitempty"` Password string `yaml:"password,omitempty"` } diff --git a/contract_test.go b/contract_test.go new file mode 100644 index 0000000..ab1d0f6 --- /dev/null +++ b/contract_test.go @@ -0,0 +1,475 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage_test + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "fmt" + "hash/crc64" + "io" + "math/big" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strconv" + "sync/atomic" + "testing" + "time" + + "github.com/aws/smithy-go" + "github.com/stretchr/testify/require" + + "github.com/livekit/storage" + "github.com/livekit/storage/objects" +) + +// These hold what a caller's environment relies on, which the emulators, +// reached with any credentials over plain HTTP, cannot show. + +// A custom CA bundle, as for a self-hosted store behind a private CA, can only +// be added to the AWS SDK's own HTTP client. +func TestS3CustomCABundle(t *testing.T) { + bundle := filepath.Join(t.TempDir(), "ca.pem") + require.NoError(t, os.WriteFile(bundle, selfSignedCert(t), 0o600)) + t.Setenv("AWS_CA_BUNDLE", bundle) + + _, err := storage.New(&storage.S3Config{ + AccessKey: "key", + Secret: "secret", + Region: "us-east-1", + Endpoint: "https://127.0.0.1:1", + Bucket: "bucket", + }) + require.NoError(t, err) +} + +// Building a GCS client must not change the process's default transport, +// which every http.Get in the importing service uses. +func TestGCPLeavesDefaultTransport(t *testing.T) { + orig := http.DefaultTransport + before := orig.(*http.Transport).Clone() + // The rest of the process reads it meanwhile: under -race, a write races. + done := make(chan struct{}) + t.Cleanup(func() { close(done) }) + go func() { + for { + select { + case <-done: + return + default: + _ = http.DefaultTransport + } + } + }() + + _, err := storage.New(&storage.GCPConfig{ + Bucket: "bucket", + Endpoint: "http://127.0.0.1:1", + ProxyConfig: &storage.ProxyConfig{URL: "http://127.0.0.1:2", Username: "u", Password: "p"}, + }) + require.NoError(t, err) + + require.Same(t, orig, http.DefaultTransport) + after := orig.(*http.Transport) + require.Nil(t, after.ProxyConnectHeader) + require.Equal(t, reflect.ValueOf(before.DialContext).Pointer(), reflect.ValueOf(after.DialContext).Pointer(), "dialer") + require.Equal(t, reflect.ValueOf(before.Proxy).Pointer(), reflect.ValueOf(after.Proxy).Pointer(), "proxy") +} + +// The proxy goes on the transport the client sends with: where it is dropped, +// traffic leaves around the proxy a network requires. +func TestProxyCarriesRequests(t *testing.T) { + for name, conf := range map[string]func(proxy *storage.ProxyConfig) storage.Config{ + "S3": func(proxy *storage.ProxyConfig) storage.Config { + return &storage.S3Config{ + AccessKey: "key", + Secret: "secret", + Region: "us-east-1", + Endpoint: "http://store.invalid", + ForcePathStyle: true, + Bucket: "bucket", + ProxyConfig: proxy, + } + }, + "GCP": func(proxy *storage.ProxyConfig) storage.Config { + return &storage.GCPConfig{Bucket: "bucket", Endpoint: "http://store.invalid", ProxyConfig: proxy} + }, + } { + t.Run(name, func(t *testing.T) { + proxied := make(chan *http.Request, 16) + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proxied <- r + http.Error(w, "no", http.StatusForbidden) + })) + t.Cleanup(proxy.Close) + send := func() { + s, err := storage.New(conf(&storage.ProxyConfig{URL: proxy.URL, Username: "user", Password: "pass"})) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, _ = s.ObjectExists(ctx, "path") + } + + // The credentials reach the proxy for a plain http target too, not + // only on CONNECT. + send() + select { + case r := <-proxied: + require.Equal(t, "store.invalid", r.Host) + require.Equal(t, "Basic dXNlcjpwYXNz", r.Header.Get("Proxy-Authorization")) + default: + t.Fatal("the request went around the proxy") + } + + // NO_PROXY is honored, as it is for the metadata services that hand + // out instance credentials. + t.Setenv("NO_PROXY", "store.invalid") + send() + select { + case <-proxied: + t.Fatal("a NO_PROXY host went through the proxy") + default: + } + }) + } +} + +// Only an http:// endpoint, an emulator's, drops credentials: a private one, +// such as Private Service Connect, is still sent a token. +func TestGCPEndpointKeepsCredentials(t *testing.T) { + for endpoint, authenticated := range map[string]bool{ + "https://127.0.0.1:1": true, + "http://127.0.0.1:1": false, + } { + t.Run(endpoint, func(t *testing.T) { + var tokens atomic.Int32 + tokenServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tokens.Add(1) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"access_token":"token","token_type":"Bearer","expires_in":3600}`) + })) + t.Cleanup(tokenServer.Close) + s, err := storage.New(&storage.GCPConfig{ + CredentialsJSON: serviceAccount(t, tokenServer.URL), + Bucket: "bucket", + Endpoint: endpoint, + }) + require.NoError(t, err) + + // The token is fetched before the request is sent, which then fails; + // GCS retries a refused connection until the context ends. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, _ = s.ObjectExists(ctx, "path") + require.Equal(t, authenticated, tokens.Load() > 0) + }) + } +} + +// Local objects get os.Create's mode, 0666 less the umask: egress records to +// a mounted volume a host user reads. +func TestLocalFileMode(t *testing.T) { + dir := t.TempDir() + s, err := storage.New(&storage.LocalConfig{StorageDir: dir}) + require.NoError(t, err) + ctx := context.Background() + + require.NoError(t, s.PutObject(ctx, "put", []byte("x"), storage.PutOptions{IfAbsent: true})) + file := filepath.Join(t.TempDir(), "file") + require.NoError(t, os.WriteFile(file, []byte("x"), 0o600)) + _, _, err = s.UploadFile(ctx, "file", file, storage.PutOptions{ContentType: "text/plain"}) + require.NoError(t, err) + for _, name := range []string{"put", "file"} { + info, err := os.Stat(filepath.Join(dir, name)) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o666)&^umask(t), info.Mode().Perm(), name) + } +} + +// An S3 bulk delete answers 200 and lists the keys it could not delete. A key +// already gone is deleted; any other is still there and must fail the call. +func TestS3DeleteObjectsKeyErrors(t *testing.T) { + for name, tc := range map[string]struct { + code string + deleted bool + }{ + "already gone": {code: "NoSuchKey", deleted: true}, + "access denied": {code: "AccessDenied"}, + "internal": {code: "InternalError"}, + } { + t.Run(name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, `b%sm`, tc.code) + })) + t.Cleanup(srv.Close) + s, err := storage.New(&storage.S3Config{ + AccessKey: "key", + Secret: "secret", + Region: "us-east-1", + Endpoint: srv.URL, + ForcePathStyle: true, + Bucket: "bucket", + }) + require.NoError(t, err) + + err = s.DeleteObjects(context.Background(), []string{"a", "b"}) + if tc.deleted { + require.NoError(t, err) + return + } + // The caller learns which keys are left, and only those. + var failed *storage.DeleteError + require.ErrorAs(t, err, &failed) + require.Len(t, failed.Failed, 1) + var apiErr smithy.APIError + require.ErrorAs(t, failed.Failed["b"], &apiErr) + require.Equal(t, tc.code, apiErr.ErrorCode()) + }) + } +} + +// Local keys drop a leading "/", in listings too, as egress's paths have one; +// a StorageDir that is a symlink lists what it points to. +func TestLocalKeys(t *testing.T) { + dir := t.TempDir() + link := filepath.Join(t.TempDir(), "link") + require.NoError(t, os.Symlink(dir, link)) + ctx := context.Background() + + for _, storageDir := range []string{dir, link} { + s, err := storage.New(&storage.LocalConfig{StorageDir: storageDir}) + require.NoError(t, err) + require.NoError(t, s.PutObject(ctx, "/rec/a.mp4", []byte("x"), storage.PutOptions{})) + for _, prefix := range []string{"", "/", "rec/", "/rec/", "/rec"} { + keys, err := s.ListObjects(ctx, prefix) + require.NoError(t, err, prefix) + require.Equal(t, []string{"rec/a.mp4"}, keys, prefix) + } + require.NoError(t, s.DeleteObjects(ctx, []string{"/rec/a.mp4"})) + } +} + +// A delete that removes an empty directory while a listing walks it does not +// hide the other keys: an empty listing reads as no objects. +func TestLocalListingDuringDeletes(t *testing.T) { + dir := t.TempDir() + s, err := storage.New(&storage.LocalConfig{StorageDir: dir}) + require.NoError(t, err) + ctx := context.Background() + require.NoError(t, s.PutObject(ctx, "keep", []byte("x"), storage.PutOptions{})) + + done := make(chan struct{}) + churned := make(chan struct{}) + go func() { + defer close(churned) + for { + select { + case <-done: + return + default: + } + // What a delete's cleanup of emptied directories does. + _ = os.MkdirAll(filepath.Join(dir, "d", "e"), 0o755) + _ = os.RemoveAll(filepath.Join(dir, "d")) + } + }() + for range 5000 { + keys, err := s.ListObjects(ctx, "") + require.NoError(t, err) + require.Contains(t, keys, "keep") + } + close(done) + <-churned +} + +// umask is the process's, read the way os.Create applies it. +func umask(t *testing.T) os.FileMode { + f, err := os.Create(filepath.Join(t.TempDir(), "probe")) + require.NoError(t, err) + defer f.Close() + info, err := f.Stat() + require.NoError(t, err) + return 0o666 &^ info.Mode().Perm() +} + +func serviceAccount(t *testing.T, tokenURI string) string { + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + der, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(t, err) + b, err := json.Marshal(map[string]string{ + "type": "service_account", + "client_email": "livekit@test.iam.gserviceaccount.com", + "private_key": string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})), + "token_uri": tokenURI, + }) + require.NoError(t, err) + return string(b) +} + +// A multipart upload that fails when its caller's context ends is aborted +// anyway: its parts would stay billed. +func TestS3AbortsAfterDeadline(t *testing.T) { + aborted := make(chan struct{}, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case r.Method == http.MethodPost && q.Has("uploads"): + fmt.Fprint(w, `bucketkup`) + case r.Method == http.MethodPut && q.Has("partNumber"): + _, _ = io.Copy(io.Discard, r.Body) + time.Sleep(time.Second) + case r.Method == http.MethodDelete && q.Get("uploadId") == "up": + aborted <- struct{}{} + w.WriteHeader(http.StatusNoContent) + } + })) + t.Cleanup(srv.Close) + s, err := storage.New(&storage.S3Config{AccessKey: "key", Secret: "secret", Region: "us-east-1", Endpoint: srv.URL, ForcePathStyle: true, Bucket: "bucket"}) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + require.Error(t, s.UploadObject(ctx, "k", bytes.NewReader(make([]byte, 40<<20)), storage.PutOptions{})) + select { + case <-aborted: + case <-time.After(10 * time.Second): + t.Fatal("the upload was not aborted") + } +} + +// The OSS SDK never retries, so this package does, for calls it can send +// again: a 5xx or 429 is retried, a 403 is not. +func TestAliOSSRetries(t *testing.T) { + var requests atomic.Int32 + failNext := atomic.Int32{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if code := failNext.Swap(0); code != 0 { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(int(code)) + fmt.Fprintf(w, `Failedmr`) + return + } + if r.Method == http.MethodGet { + fmt.Fprint(w, "hello") + } + })) + t.Cleanup(srv.Close) + s, err := storage.New(&storage.AliOSSConfig{AccessKey: "key", Secret: "secret", Endpoint: srv.URL, Bucket: "bucket"}) + require.NoError(t, err) + ctx := context.Background() + + for _, code := range []int32{http.StatusServiceUnavailable, http.StatusTooManyRequests} { + requests.Store(0) + failNext.Store(code) + require.NoError(t, s.PutObject(ctx, "k", []byte("hello"), storage.PutOptions{})) + require.EqualValues(t, 2, requests.Load(), "put after %d", code) + + requests.Store(0) + failNext.Store(code) + got, err := s.GetObject(ctx, "k") + require.NoError(t, err) + require.Equal(t, []byte("hello"), got) + require.EqualValues(t, 2, requests.Load(), "get after %d", code) + } + + requests.Store(0) + failNext.Store(http.StatusForbidden) + err = s.PutObject(ctx, "k", []byte("hello"), storage.PutOptions{}) + require.EqualValues(t, 1, requests.Load(), "a 403 is not retried") + // The status is found with the SDK-free package's type. + var status *objects.ErrorWithStatusCode + require.ErrorAs(t, err, &status) + require.Equal(t, http.StatusForbidden, status.StatusCode) +} + +// AliOSS's DownloadFile keeps the contract: it makes the destination's +// directory, a body cut short or a CRC64 that does not match leaves the +// destination as it was and no temporary file behind. +func TestAliOSSDownloadFile(t *testing.T) { + var mode atomic.Value + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := []byte("hello") + crc := crc64.Checksum(body, crc64.MakeTable(crc64.ECMA)) + switch mode.Load() { + case "short": + w.Header().Set("Content-Length", "10") + _, _ = w.Write(body) + // Ends the response five bytes short. + conn, _, _ := w.(http.Hijacker).Hijack() + _ = conn.Close() + return + case "bad crc": + crc++ + } + w.Header().Set("X-Oss-Hash-Crc64ecma", strconv.FormatUint(crc, 10)) + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + s, err := storage.New(&storage.AliOSSConfig{AccessKey: "key", Secret: "secret", Endpoint: srv.URL, Bucket: "bucket"}) + require.NoError(t, err) + ctx := context.Background() + + dir := t.TempDir() + dst := filepath.Join(dir, "new", "dir", "file") + mode.Store("ok") + n, err := s.DownloadFile(ctx, "k", dst) + require.NoError(t, err) + require.EqualValues(t, 5, n) + got, err := os.ReadFile(dst) + require.NoError(t, err) + require.Equal(t, []byte("hello"), got) + + for _, m := range []string{"short", "bad crc"} { + mode.Store(m) + require.NoError(t, os.WriteFile(dst, []byte("kept"), 0o600)) + _, err = s.DownloadFile(ctx, "k", dst) + require.Error(t, err, m) + got, err := os.ReadFile(dst) + require.NoError(t, err) + require.Equal(t, []byte("kept"), got, m) + entries, err := os.ReadDir(filepath.Dir(dst)) + require.NoError(t, err) + require.Len(t, entries, 1, "%s left a temporary file", m) + } +} + +func selfSignedCert(t *testing.T) []byte { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} diff --git a/e2e/ci_test.go b/e2e/ci_test.go new file mode 100644 index 0000000..80bce8b --- /dev/null +++ b/e2e/ci_test.go @@ -0,0 +1,56 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// Each emulator test is a check of its own in CI, from the e2e job's matrix; +// one left out of it would run nowhere. +func TestWorkflowRunsEveryEmulatorTest(t *testing.T) { + workflow, err := os.ReadFile("../.github/workflows/test.yaml") + require.NoError(t, err) + _, rest, ok := strings.Cut(string(workflow), "test: [") + require.True(t, ok, "no e2e matrix") + list, _, _ := strings.Cut(rest, "]") + var matrix []string + for _, name := range strings.Split(list, ",") { + matrix = append(matrix, "Test"+strings.TrimSpace(name)) + } + + files, err := filepath.Glob("*_test.go") + require.NoError(t, err) + var tests []string + for _, file := range files { + f, err := parser.ParseFile(token.NewFileSet(), file, nil, 0) + require.NoError(t, err) + for _, decl := range f.Decls { + fn, ok := decl.(*ast.FuncDecl) + if ok && (strings.HasPrefix(fn.Name.Name, "TestFloci") || fn.Name.Name == "TestMinIO") { + tests = append(tests, fn.Name.Name) + } + } + } + require.ElementsMatch(t, tests, matrix) +} diff --git a/e2e/floci_test.go b/e2e/floci_test.go new file mode 100644 index 0000000..40defdb --- /dev/null +++ b/e2e/floci_test.go @@ -0,0 +1,363 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + gcs "cloud.google.com/go/storage" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/objectstorage" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" + "github.com/stretchr/testify/require" + "google.golang.org/api/option" + + "github.com/livekit/storage" +) + +// The emulators run with the tests on any machine with Docker, and must in CI. +// Each holds its state in memory and is removed when its test ends. +// Cleanups take a context of their own: the test's is canceled by then. + +const flociBucket = "livekit" + +func TestFlociS3(t *testing.T) { + endpoint := floci(t, "floci/floci:2.3.0@sha256:e0b292df4dc9559b7b1bf4f51bdda4bb8e269276236438cd03cae661260ea8a9", "4566") + conf := &storage.S3Config{ + AccessKey: "test", + Secret: "test", + Region: "us-east-1", + Endpoint: endpoint, + ForcePathStyle: true, + Bucket: flociBucket, + } + admin := s3.New(s3.Options{ + Region: conf.Region, + BaseEndpoint: &conf.Endpoint, + UsePathStyle: true, + Credentials: credentials.NewStaticCredentialsProvider(conf.AccessKey, conf.Secret, ""), + }) + _, err := admin.CreateBucket(t.Context(), &s3.CreateBucketInput{Bucket: aws.String(flociBucket)}) + require.NoError(t, err) + + testProvider(t, endpoint, func(endpoint string) storage.Config { + c := *conf + c.Endpoint = endpoint + return &c + }, objectsSuite{bulk: true, multipartIfAbsentRaces: true, commit: s3Commit}) +} + +// flociAzureKey is Azurite's well-known development key; floci-az accepts any. +const flociAzureKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + +func TestFlociAzure(t *testing.T) { + // An IP host: the SDK parses an account from the path only then. + endpoint := strings.Replace(floci(t, "floci/floci-az:0.15.0@sha256:530635d603223e120adbf9221a97560cffc6991d65fa0c72d0f79d1097974c89", "4577"), "localhost", "127.0.0.1", 1) + // floci-az gives devstoreaccount1 a hierarchical namespace (ADLS Gen2), + // where a path's directories are entries of their own, and other accounts + // a flat one: both are common on Azure. + for name, account := range map[string]string{"hierarchical namespace": "devstoreaccount1", "flat namespace": "flat"} { + t.Run(name, func(t *testing.T) { + conf := &storage.AzureConfig{ + AccountName: account, + AccountKey: flociAzureKey, + ContainerName: flociBucket, + Endpoint: endpoint + "/" + account + "/", + } + cred, err := azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey) + require.NoError(t, err) + admin, err := azblob.NewClientWithSharedKeyCredential(conf.Endpoint, cred, nil) + require.NoError(t, err) + _, err = admin.CreateContainer(t.Context(), flociBucket, nil) + require.NoError(t, err) + + testProvider(t, endpoint, func(endpoint string) storage.Config { + c := *conf + c.Endpoint = endpoint + "/" + account + "/" + return &c + }, objectsSuite{bulk: true, multipartIgnoresIfAbsent: true, commit: func(r *http.Request) (bool, bool) { + return r.Method == http.MethodPut && r.URL.Query().Get("comp") == "blocklist", r.Header.Get("If-None-Match") == "*" + }}) + }) + } +} + +func TestFlociGCP(t *testing.T) { + endpoint := floci(t, "floci/floci-gcp:0.11.0@sha256:eae9683d806ceb643bcb36aee97cdb8cad13c3e2959cd053bf40e481994fde12", "4588") + conf := &storage.GCPConfig{ + // A service account of its own, to sign URLs. + CredentialsJSON: gcpServiceAccount(t), + Bucket: flociBucket, + Endpoint: endpoint, + } + admin, err := gcs.NewClient(t.Context(), option.WithEndpoint(endpoint+"/storage/v1/"), option.WithoutAuthentication()) + require.NoError(t, err) + t.Cleanup(func() { _ = admin.Close() }) + require.NoError(t, admin.Bucket(flociBucket).Create(t.Context(), "floci-local", nil)) + + testProvider(t, endpoint, func(endpoint string) storage.Config { + c := *conf + c.Endpoint = endpoint + return &c + }, objectsSuite{bulk: true, commit: func(r *http.Request) (bool, bool) { + // A resumable upload takes its condition at the request that starts it, + // the one without an upload_id. + q := r.URL.Query() + return r.Method == http.MethodPost && q.Get("uploadType") == "resumable" && !q.Has("upload_id"), q.Get("ifGenerationMatch") == "0" + }}) +} + +// flociTenancy is floci-oci's tenancy, which is also the root compartment. +const flociTenancy = "ocid1.tenancy.oc1..flocilocaltenancy0000000000000000000000000000000000000000" + +func TestFlociOCI(t *testing.T) { + endpoint := floci(t, "floci/floci-oci:0.5.0@sha256:3cb5af867fb2a4e8b60f8cf6fd0acf4873108f01d481c0aac23c01ca00f935ae", "4599") + // floci-oci reads the tenancy and user from the request signature without + // verifying it, so any key signs. + conf := &storage.OCIConfig{ + TenancyOCID: flociTenancy, + UserOCID: "ocid1.user.oc1..livekit", + Fingerprint: "00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00", + PrivateKey: rsaKeyPEM(t), + Region: "us-ashburn-1", + Endpoint: endpoint, + Bucket: flociBucket, + } + provider := common.NewRawConfigurationProvider(conf.TenancyOCID, conf.UserOCID, conf.Region, conf.Fingerprint, conf.PrivateKey, nil) + admin, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(provider) + require.NoError(t, err) + admin.Host = endpoint + ns, err := admin.GetNamespace(t.Context(), objectstorage.GetNamespaceRequest{}) + require.NoError(t, err) + _, err = admin.CreateBucket(t.Context(), objectstorage.CreateBucketRequest{ + NamespaceName: ns.Value, + CreateBucketDetails: objectstorage.CreateBucketDetails{ + Name: common.String(flociBucket), + CompartmentId: common.String(flociTenancy), + }, + }) + require.NoError(t, err) + + testProvider(t, endpoint, func(endpoint string) storage.Config { + c := *conf + c.Endpoint = endpoint + // The namespace lookup is a read, so a write-only deployment names it. + c.Namespace = *ns.Value + return &c + }, objectsSuite{bulk: true, singlePartIfAbsentRaces: true, multipartIgnoresIfAbsent: true, commit: func(r *http.Request) (bool, bool) { + return r.Method == http.MethodPost && r.URL.Query().Has("uploadId"), r.Header.Get("If-None-Match") == "*" + }}) +} + +// testProvider runs every suite against the store at endpoint. at gives the +// provider's config for an endpoint, so the suites can reach the store through +// proxies that refuse some requests. +func testProvider(t *testing.T, endpoint string, at func(endpoint string) storage.Config, suite objectsSuite) { + s, err := storage.New(at(endpoint)) + require.NoError(t, err) + writeOnly, err := storage.New(at(refuse(t, endpoint, isRead))) + require.NoError(t, err) + noDeletes, err := storage.New(at(refuse(t, endpoint, isDelete))) + require.NoError(t, err) + t.Run("Storage", func(t *testing.T) { testStorage(t, s) }) + t.Run("Bucket", func(t *testing.T) { testObjectsWith(t, s, suite) }) + t.Run("write only", func(t *testing.T) { testWriteOnly(t, s, writeOnly, suite) }) + t.Run("deletes refused", func(t *testing.T) { testDeletesRefused(t, s, noDeletes) }) + if suite.commit != nil { + t.Run("multipart IfAbsent on the wire", func(t *testing.T) { + testConditionalCommit(t, s, endpoint, at, suite.commit) + }) + } +} + +func s3Commit(r *http.Request) (bool, bool) { + return r.Method == http.MethodPost && r.URL.Query().Has("uploadId"), r.Header.Get("If-None-Match") == "*" +} + +// testConditionalCommit uploads a multipart object IfAbsent through a proxy +// that sees every request: the one that makes the object must carry the +// condition, whether or not the store enforces it. +func testConditionalCommit(t *testing.T, s storage.Storage, endpoint string, at func(string) storage.Config, commit func(*http.Request) (bool, bool)) { + ctx := t.Context() + var commits, conditional atomic.Int32 + watched, err := storage.New(at(refuse(t, endpoint, func(r *http.Request) bool { + if isCommit, isConditional := commit(r); isCommit { + commits.Add(1) + if isConditional { + conditional.Add(1) + } + } + return false + }))) + require.NoError(t, err) + + path := fmt.Sprintf("conditional-commit-%d", time.Now().UnixNano()) + t.Cleanup(func() { _ = s.DeleteObjects(context.Background(), []string{path}) }) + body := bytes.Repeat([]byte("0123456789abcdef"), (17<<20)/16) + require.NoError(t, watched.UploadObject(ctx, path, bytes.NewReader(body), storage.PutOptions{IfAbsent: true})) + require.Positive(t, commits.Load(), "no commit request seen") + require.Equal(t, commits.Load(), conditional.Load(), "a commit went out without the condition") +} + +// testDeletesRefused deletes where the store refuses it: the objects are still +// there, so the call must fail, naming each of them, past the first refusal. +func testDeletesRefused(t *testing.T, s, noDeletes storage.Storage) { + ctx := t.Context() + prefix := fmt.Sprintf("deletes-refused-%d/", time.Now().UnixNano()) + paths := []string{prefix + "a", prefix + "b"} + for _, p := range paths { + require.NoError(t, s.PutObject(ctx, p, []byte("kept"), storage.PutOptions{})) + } + t.Cleanup(func() { _ = s.DeleteObjects(context.Background(), paths) }) + + var failed *storage.DeleteError + require.ErrorAs(t, noDeletes.DeleteObjects(ctx, paths), &failed) + for _, p := range paths { + require.Contains(t, failed.Failed, p) + exists, err := s.ObjectExists(ctx, p) + require.NoError(t, err) + require.True(t, exists) + } +} + +// testWriteOnly writes IfAbsent with credentials that may write but not read: +// any check before uploading fails, and the store's condition alone refuses +// the second upload. +func testWriteOnly(t *testing.T, s, writeOnly storage.Storage, suite objectsSuite) { + ctx := t.Context() + prefix := fmt.Sprintf("write-only-%d/", time.Now().UnixNano()) + bodies := map[string][]byte{"single part": []byte("written without reads")} + if !suite.multipartIgnoresIfAbsent { + bodies["multipart"] = bytes.Repeat([]byte("0123456789abcdef"), (17<<20)/16) + } + for name, body := range bodies { + t.Run(name, func(t *testing.T) { + path := prefix + name + t.Cleanup(func() { _ = s.DeleteObjects(context.Background(), []string{path}) }) + require.NoError(t, writeOnly.UploadObject(ctx, path, bytes.NewReader(body), storage.PutOptions{IfAbsent: true})) + err := writeOnly.UploadObject(ctx, path, bytes.NewReader(body), storage.PutOptions{IfAbsent: true}) + require.ErrorIs(t, err, storage.ErrExists) + _, err = writeOnly.GetObject(ctx, path) + require.Error(t, err, "reads must be denied") + got, err := s.GetObject(ctx, path) + require.NoError(t, err) + require.Equal(t, body, got) + }) + } +} + +func isRead(r *http.Request) bool { + return r.Method == http.MethodGet || r.Method == http.MethodHead +} + +// isDelete is a single delete, or an S3 or AliOSS bulk one (POST ?delete). +func isDelete(r *http.Request) bool { + return r.Method == http.MethodDelete || (r.Method == http.MethodPost && r.URL.Query().Has("delete")) +} + +// refuse is a proxy to endpoint that answers the requests matched with 403, +// like a bucket policy that withholds that permission. It sees every request. +func refuse(t *testing.T, endpoint string, matched func(*http.Request) bool) string { + target, err := url.Parse(endpoint) + require.NoError(t, err) + proxy := httputil.NewSingleHostReverseProxy(target) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if matched(r) { + http.Error(w, "refused", http.StatusForbidden) + return + } + proxy.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + return srv.URL +} + +// floci starts image, repo:tag@digest, and returns its http:// endpoint once +// it answers. A tag pushed again fails the test rather than change what it +// runs. It skips the test where Docker is unavailable, except in CI. +func floci(t *testing.T, image, port string) string { + ref, digest, _ := strings.Cut(image, "@") + repo, tag, _ := strings.Cut(ref, ":") + pool, err := dockertest.NewPool("") + if err == nil { + err = pool.Client.Ping() + } + if err != nil { + if os.Getenv("CI") != "" { + t.Fatalf("docker: %v", err) + } + t.Skipf("docker unavailable: %v", err) + } + + res, err := pool.RunWithOptions(&dockertest.RunOptions{Repository: repo, Tag: tag}, func(hc *docker.HostConfig) { + hc.AutoRemove = true + }) + require.NoError(t, err) + t.Cleanup(func() { _ = pool.Purge(res) }) + img, err := pool.Client.InspectImage(ref) + require.NoError(t, err) + require.Contains(t, img.RepoDigests, repo+"@"+digest, "%s is not the image pinned", ref) + + endpoint := "http://" + res.GetHostPort(port+"/tcp") + require.NoError(t, pool.Retry(func() error { + resp, err := pool.Client.HTTPClient.Get(endpoint) + if err == nil { + resp.Body.Close() + } + return err + })) + return endpoint +} + +func rsaKeyPEM(t *testing.T) string { + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + der, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(t, err) + return string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})) +} + +func gcpServiceAccount(t *testing.T) string { + b, err := json.Marshal(map[string]string{ + "type": "service_account", + "project_id": "floci-local", + "client_email": "livekit@floci-local.iam.gserviceaccount.com", + "private_key": rsaKeyPEM(t), + "token_uri": "https://oauth2.googleapis.com/token", + }) + require.NoError(t, err) + return string(b) +} diff --git a/e2e/go.mod b/e2e/go.mod new file mode 100644 index 0000000..8e25098 --- /dev/null +++ b/e2e/go.mod @@ -0,0 +1,133 @@ +module github.com/livekit/storage/e2e + +go 1.26.0 + +require ( + cloud.google.com/go/storage v1.69.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 + github.com/aws/aws-sdk-go-v2 v1.47.1 + github.com/aws/aws-sdk-go-v2/credentials v1.20.6 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 + github.com/joho/godotenv v1.5.1 + github.com/livekit/storage v0.0.0 + github.com/oracle/oci-go-sdk/v65 v65.126.1 + github.com/ory/dockertest/v3 v3.12.0 + github.com/stretchr/testify v1.12.1 + google.golang.org/api v0.300.0 +) + +require ( + cel.dev/expr v0.25.2 // indirect + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.24.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.3.0 // indirect + cloud.google.com/go/compute/metadata v0.10.0 // indirect + cloud.google.com/go/iam v1.12.0 // indirect + cloud.google.com/go/monitoring v1.30.0 // indirect + dario.cat/mergo v1.0.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect + github.com/apache/arrow-go/v18 v18.7.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect + github.com/aws/aws-sdk-go-v2/config v1.33.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 // indirect + github.com/aws/smithy-go v1.28.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/containerd/continuity v0.4.5 // indirect + github.com/docker/cli v27.4.1+incompatible // indirect + github.com/docker/docker v27.1.1+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/goccy/go-json v0.10.6 // indirect + github.com/gofrs/flock v0.10.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect + github.com/google/s2a-go v0.1.10 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.22 // indirect + github.com/googleapis/gax-go/v2 v2.26.2 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/livekit/protocol v1.52.1 // indirect + github.com/livekit/storage/objects v0.1.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/sys/user v0.3.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/opencontainers/runc v1.2.3 // indirect + github.com/pierrec/lz4/v4 v4.1.28 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect + github.com/puzpuzpuz/xsync/v4 v4.5.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/sony/gobreaker/v2 v2.4.0 // indirect + github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.28.0 // indirect + go.uber.org/zap/exp v0.3.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/oauth2 v0.37.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + golang.org/x/time v0.16.0 // indirect + google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 // indirect + google.golang.org/grpc v1.84.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) + +replace github.com/livekit/storage => ../ + +replace github.com/livekit/storage/objects => ../objects diff --git a/e2e/go.sum b/e2e/go.sum new file mode 100644 index 0000000..7338446 --- /dev/null +++ b/e2e/go.sum @@ -0,0 +1,361 @@ +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 h1:s6hzCXtND/ICdGPTMGk7C+/BFlr2Jg5GyH0NKf4XGXg= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM= +buf.build/go/protovalidate v1.2.0 h1:DQVrUWkmGTBij+kOYv/x2LLxwcLaGKMdzShj1/6/3H0= +buf.build/go/protovalidate v1.2.0/go.mod h1:7rYiQEhqvAipoazpVNBBH2S2f8bjG4huMVy1V2Yofn4= +buf.build/go/protoyaml v0.7.0 h1:z4oVoFicbpPefhT7WAykxUdfp0yEQlhMQ2mCZOY5V38= +buf.build/go/protoyaml v0.7.0/go.mod h1:+a0cavd0uMvirb87xdu2ZMMmjlIQoiH/N2Ich5MGSQ0= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.24.0 h1:UYMbF8otPZnLAkNJ5/LYQYOq0ARcJS1P4JqTeMKbCYU= +cloud.google.com/go/auth v0.24.0/go.mod h1:IFG/AMA1VWfuTrdbieEsB2GcpJyJV/phGAvogkOoPR4= +cloud.google.com/go/auth/oauth2adapt v0.3.0 h1:FY8oSZpCYoUNv6QxVODuMjQz4IlSOVeiQtZ08vLPz88= +cloud.google.com/go/auth/oauth2adapt v0.3.0/go.mod h1:7+2uCm7++XFO+/lN06c2HXpDXb/NMNn2/UwyBPbTnkk= +cloud.google.com/go/compute/metadata v0.10.0 h1:pyKMUQSwchgkIBBJGdILqQbs/BNJXqwSA7Ej6LAvvtY= +cloud.google.com/go/compute/metadata v0.10.0/go.mod h1:rGFHRrIif570kSibjFTMbt6/4/tzgJWFGI/HVol4GIk= +cloud.google.com/go/iam v1.12.0 h1:Aki3bX9aHUDKPHfnRJfDcTdVedvy6quGBQcTqx3DRXk= +cloud.google.com/go/iam v1.12.0/go.mod h1:FEZ4lXpADAC2AIpQY7LANNjjwyQ2jK439CI2VaD+sLY= +cloud.google.com/go/logging v1.19.0 h1:NCqhdVUg3wQ8Cobdf16FDSuTGi3+6+hdSBHrY5TsR6Q= +cloud.google.com/go/logging v1.19.0/go.mod h1:i40NZCHC9Gqvod4yE+yQfDWwlgwW/SrshkkGibCHxcA= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= +cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY= +cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM= +cloud.google.com/go/storage v1.69.0 h1:jAAMC1411HEh78nKsU0Zns+eFj3TnhjAWIhg5Ud/XBM= +cloud.google.com/go/storage v1.69.0/go.mod h1:PELYsxTYm2peE4mwLEC1+mS1dA/kUSRUxNv56rOy44g= +cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2 h1:utpeoEeZjd+A8J41zvoLsOOrqXHhX1Kx/X/tCW9dEYQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 h1:tQJdgQjbKKffEiMOKJmcQPkY01eFHRqWIZVefqP44nI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2/go.mod h1:kTkpJchXAdO7LGkKZP97a+Ou8TgaF4NT6MHwOIy5uSQ= +github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 h1:X/bTwe0qpvmvPPutc+LWo7T4cKMqDP9JHP4k+Co9ZB4= +github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0/go.mod h1:+kX7ahyFelQ42F3RScM2CFgHJljp0phc/FieKLYVpFI= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0 h1:bN1gA3of5bXtbnLsRPrwfmbbe7A5UWFlcTHseujLnpc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.35.0/go.mod h1:Yj5vHEz/aAepZGliRJsA6uvHAVAQyEwajq9ORCHPxzM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 h1:jLdiS1vO+XJFyDSWRHBx56r4s/NNtcl5J6KyCcWUX/w= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0/go.mod h1:8lmpHY+1VRoteiOwyrQMDt1YGXOrFKCz+1wJW7n3ODY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0 h1:cSjUzZ7KU8hicTgzaSv9NmSyM9fTVK3y5lsBUl3wOis= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 h1:RoO5+d7uCmDqovLrHCr2/BuViUXvdcrNxyNM1pN9dDQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= +github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= +github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= +github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= +github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= +github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= +github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug= +github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE= +github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw= +github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.11 h1:wgxEej5cFj+EfutuAPZPIFcMvQ3Doamt01lMtPoMpls= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.11/go.mod h1:dMcCQXtMtzVmEUO7YO+1xtYAvo8BcKgnN3Wppo8hbmA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4/go.mod h1:wm04I5DMuNVvZHFe/dHnUxincvNbbK7AiNBbYsQivek= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZjfhz0zXtjs6gdE8s= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.1/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM= +github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= +github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= +github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dennwc/iters v1.2.2 h1:XH2/Etihiy9ZvPOVCR+icQXeYlhbvS7k0qro4x/2qQo= +github.com/dennwc/iters v1.2.2/go.mod h1:M9KuuMBeyEXYTmB7EnI9SCyALFCmPWOIxn5W1L0CjGg= +github.com/docker/cli v27.4.1+incompatible h1:VzPiUlRJ/xh+otB75gva3r05isHMo5wXDfPRi5/b4hI= +github.com/docker/cli v27.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/gofrs/flock v0.10.0 h1:SHMXenfaB03KbroETaCMtbBg3Yn29v4w1r+tgy4ff4k= +github.com/gofrs/flock v0.10.0/go.mod h1:FirDy1Ing0mI2+kB6wk+vyyAH+e6xiE+EYA0jnzV9jc= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.29.0 h1:fEG+Ja3YRwNOqnQxTyJwoByAUAvTuxUGiro/jhrm4F4= +github.com/google/cel-go v0.29.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/s2a-go v0.1.10 h1:EMp+aOuXN6l8cE/gjF5Bt+vyZxsUuyCWe9chDWR/+uU= +github.com/google/s2a-go v0.1.10/go.mod h1:pz4tyvwXvJLLbyrkh6FW1eS2zPUXMaTmyNhYtyP2tNw= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.22 h1:NU4XpII6jD+Dxcot94fqjE+AfJoE/lQP9q3faYGzC/c= +github.com/googleapis/enterprise-certificate-proxy v0.3.22/go.mod h1:L3D/IQExI6LqEjBdXcZQ1WluSgigQmSwBboFstVPM4w= +github.com/googleapis/gax-go/v2 v2.26.2 h1:ydkmNXxj7bEmmeK5AihkKnWxyOyBR9TDebvp5L5izk8= +github.com/googleapis/gax-go/v2 v2.26.2/go.mod h1:sMKqnMesnKH+3wiRJROcttA+cJoZoGbZl1vDQ8XYtGk= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5ATTo469PQPkqzdoU7be46ryiCDO3boc= +github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= +github.com/livekit/protocol v1.52.1 h1:w8wRjRlj/57AJBxwomEzpEMzf+lT0qOoLQLO7awdTMU= +github.com/livekit/protocol v1.52.1/go.mod h1:t5CV/LCsWbI9zsSTHTJKDyLtByqvvVkioSP7jgqpIGA= +github.com/livekit/psrpc v0.8.0 h1:uo3x+DL8pK2OIaCxg/s9sLUO2cIocS5HwdPhhI+m+fQ= +github.com/livekit/psrpc v0.8.0/go.mod h1:6KnU8vULoizX0Er9NNfeKJDV8ug87j3fuo19M0bhj6Q= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= +github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/runc v1.2.3 h1:fxE7amCzfZflJO2lHXf4y/y8M1BoAqp+FVmG19oYB80= +github.com/opencontainers/runc v1.2.3/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM= +github.com/oracle/oci-go-sdk/v65 v65.126.1 h1:WmQ2Igq7/L/cHlR2jZnJkBC2UI51Ams4DJZW3CZGVBo= +github.com/oracle/oci-go-sdk/v65 v65.126.1/go.mod h1:YmvgbsnUfrsiJ410XAGQ1BRsrUYJJ7W3O5wUikmeI8w= +github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCyRCw= +github.com/ory/dockertest/v3 v3.12.0/go.mod h1:aKNDTva3cp8dwOWwb9cWuX84aH5akkxXRvO7KCwWVjE= +github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio= +github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 h1:S1hI5JiKP7883xBzZAr1ydcxrKNSVNm7+3+JwjxZEsg= +github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25/go.mod h1:ZQntvDG8TkPgljxtA0R9frDoND4QORU1VXz015N5Ks4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= +github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg= +github.com/sony/gobreaker/v2 v2.4.0/go.mod h1:pTyFJgcZ3h2tdQVLZZruK2C0eoFL1fb/G83wK1ZQl+s= +github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= +github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJXP61mNV3/7iuU= +github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0 h1:9jR0ZPRok9ryaOQ2Wx8rg5F7Aon59mxrqbVI60/vlBk= +go.opentelemetry.io/contrib/detectors/gcp v1.45.0/go.mod h1:VSme3o2fvSg5bVg0dRzyHaj4Z5EVhG+g2Fde6LKzmQA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 h1:2yEATaop1/a1I4psnSLgWVPLWwCzkqWakgJy7xTDVy0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0/go.mod h1:D7J12YRapIekYyPWgGPlA/23pRmpSEZC5xJC/TTLI9U= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0 h1:dm9iyzn6tioYZtwqaiBSU0TSI8Yu/8dTIbfG0+B49DY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.45.0/go.mod h1:xAvxYjYK28qvt+yu4BYZ/zMmAjwMXINXD6JiMyeB8iI= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= +go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= +go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY= +golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= +golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.300.0 h1:2rvPV2bqnPuHOaF4gGOBiT1IIc6JVXYyHCkZeqdzjNk= +google.golang.org/api v0.300.0/go.mod h1:tKfTSDfK+0FlOVl8N30VL5fU5TuaEkJjvdyTIKNwzPg= +google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d h1:C9v1o0/4quuhOAfmRXA2j+we0PqZIp8traLdeogF3Ms= +google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d/go.mod h1:Wz2wFJntZFmLGo7pLDXZ3wYk5hyc0Mb+SkHhDDXT+lU= +google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d h1:QwnJwPte4XXAkhPu26LTDIahnsMSUV0kK8HkxbC+Pc4= +google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d/go.mod h1:WRrQ7/7N19PypuT0fxLOL5Lq0waoiRri4FbtHDEKrGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 h1:b0xCahf3FK2m2Cv0p4vTozGPWncCvLfwV86UNg8xWU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/e2e/minio_test.go b/e2e/minio_test.go new file mode 100644 index 0000000..2bd94e7 --- /dev/null +++ b/e2e/minio_test.go @@ -0,0 +1,100 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "net" + "net/http" + "os" + "os/exec" + "strconv" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/require" + + "github.com/livekit/storage" +) + +// TestMinIO runs the suites on a real S3 implementation, whose conditional +// writes are atomic where floci's are not, so the IfAbsent races run in full. +// It takes the minio binary from MINIO_BIN or PATH; CI installs it. +func TestMinIO(t *testing.T) { + endpoint := minio(t) + conf := &storage.S3Config{ + AccessKey: "minioadmin", + Secret: "minioadmin", + Region: "us-east-1", + Endpoint: endpoint, + ForcePathStyle: true, + Bucket: flociBucket, + } + admin := s3.New(s3.Options{ + Region: conf.Region, + BaseEndpoint: &conf.Endpoint, + UsePathStyle: true, + Credentials: credentials.NewStaticCredentialsProvider(conf.AccessKey, conf.Secret, ""), + }) + _, err := admin.CreateBucket(t.Context(), &s3.CreateBucketInput{Bucket: aws.String(flociBucket)}) + require.NoError(t, err) + + testProvider(t, endpoint, func(endpoint string) storage.Config { + c := *conf + c.Endpoint = endpoint + return &c + }, objectsSuite{bulk: true, commit: s3Commit}) +} + +// minio starts a MinIO server on a free port and returns its endpoint once it +// is live. It skips the test without a binary, except in CI. +func minio(t *testing.T) string { + bin := os.Getenv("MINIO_BIN") + if bin == "" { + bin, _ = exec.LookPath("minio") + } + if bin == "" { + if os.Getenv("CI") != "" { + t.Fatal("minio binary not found: set MINIO_BIN") + } + t.Skip("minio binary not found: set MINIO_BIN") + } + + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + addr := "127.0.0.1:" + strconv.Itoa(l.Addr().(*net.TCPAddr).Port) + require.NoError(t, l.Close()) + + cmd := exec.Command(bin, "server", t.TempDir(), "--address", addr, "--console-address", "127.0.0.1:0", "--quiet") + cmd.Env = append(os.Environ(), "MINIO_ROOT_USER=minioadmin", "MINIO_ROOT_PASSWORD=minioadmin") + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + + endpoint := "http://" + addr + require.Eventually(t, func() bool { + resp, err := http.Get(endpoint + "/minio/health/live") + if err != nil { + return false + } + resp.Body.Close() + return resp.StatusCode == http.StatusOK + }, 30*time.Second, 100*time.Millisecond) + return endpoint +} diff --git a/e2e/object_test.go b/e2e/object_test.go new file mode 100644 index 0000000..cb0e658 --- /dev/null +++ b/e2e/object_test.go @@ -0,0 +1,322 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/iotest" + "time" + + "github.com/stretchr/testify/require" + + "github.com/livekit/storage" +) + +// objectsSuite is what testObjectsWith runs beyond the core cases. Its gaps +// name what an emulator gets wrong, and skip the cases that would show it; a +// real store has none. +type objectsSuite struct { + // bulk deletes more keys than one batch takes, too slow for a real account. + bulk bool + // singlePartIfAbsentRaces: two racing IfAbsent puts can both win. + singlePartIfAbsentRaces bool + // multipartIfAbsentRaces: two racing IfAbsent multipart uploads can both win. + multipartIfAbsentRaces bool + // multipartIgnoresIfAbsent: a multipart upload's commit ignores IfAbsent. + multipartIgnoresIfAbsent bool + // overwritesInPlace: a write replaces an object as it goes, so a failed + // one leaves none (local, as a filesystem). + overwritesInPlace bool + // commit tells the request that makes a multipart upload's object, and + // whether it carries IfAbsent's condition, which a store that ignores the + // condition cannot show. + commit func(r *http.Request) (commit, conditional bool) +} + +func testObjects(t *testing.T, s storage.Storage) { + testObjectsWith(t, s, objectsSuite{}) +} + +func testObjectsWith(t *testing.T, s storage.Storage, suite objectsSuite) { + ctx := context.Background() + prefix := fmt.Sprintf("objects-test-%d/", time.Now().UnixNano()) + small := prefix + "small" + large := prefix + "large" + missing := prefix + "missing" + payload := []byte("hello from PutObject") + // Larger than one upload part, so UploadObject takes the multipart path. + big := bytes.Repeat([]byte("0123456789abcdef"), (17<<20)/16) + + t.Cleanup(func() { + _ = s.DeleteObjects(ctx, []string{small, large}) + }) + + t.Run("PutObject IfAbsent", func(t *testing.T) { + require.NoError(t, s.PutObject(ctx, small, payload, storage.PutOptions{IfAbsent: true})) + err := s.PutObject(ctx, small, []byte("other"), storage.PutOptions{IfAbsent: true}) + require.ErrorIs(t, err, storage.ErrExists) + got, err := s.GetObject(ctx, small) + require.NoError(t, err) + require.Equal(t, payload, got) + }) + + t.Run("UploadObject", func(t *testing.T) { + require.NoError(t, s.UploadObject(ctx, large, bytes.NewReader(big), storage.PutOptions{IfAbsent: true})) + err := s.UploadObject(ctx, large, bytes.NewReader(big), storage.PutOptions{IfAbsent: true}) + require.ErrorIs(t, err, storage.ErrExists) + var buf bytes.Buffer + n, err := s.DownloadObject(ctx, large, &buf) + require.NoError(t, err) + require.Equal(t, int64(len(big)), n) + require.Equal(t, big, buf.Bytes()) + }) + + t.Run("GetObjectRange", func(t *testing.T) { + got, err := s.GetObjectRange(ctx, small, 6, 4) + require.NoError(t, err) + require.Equal(t, payload[6:10], got) + got, err = s.GetObjectRange(ctx, small, 6, 1000) + require.NoError(t, err) + require.Equal(t, payload[6:], got) + got, err = s.GetObjectRange(ctx, small, int64(len(payload)), 4) + require.NoError(t, err) + require.Empty(t, got) + }) + + t.Run("not found", func(t *testing.T) { + _, err := s.GetObject(ctx, missing) + require.ErrorIs(t, err, storage.ErrNotFound) + _, err = s.GetObjectRange(ctx, missing, 0, 1) + require.ErrorIs(t, err, storage.ErrNotFound) + _, err = s.DownloadObject(ctx, missing, io.Discard) + require.ErrorIs(t, err, storage.ErrNotFound) + // A failed download leaves the file it would have written as it was. + kept := filepath.Join(t.TempDir(), "kept") + require.NoError(t, os.WriteFile(kept, []byte("kept"), 0o600)) + _, err = s.DownloadFile(ctx, missing, kept) + require.ErrorIs(t, err, storage.ErrNotFound) + got, err := os.ReadFile(kept) + require.NoError(t, err) + require.Equal(t, []byte("kept"), got) + exists, err := s.ObjectExists(ctx, missing) + require.NoError(t, err) + require.False(t, exists) + exists, err = s.ObjectExists(ctx, small) + require.NoError(t, err) + require.True(t, exists) + }) + + t.Run("ListObjects and DeleteObjects", func(t *testing.T) { + keys, err := s.ListObjects(ctx, prefix) + require.NoError(t, err) + require.Equal(t, []string{large, small}, keys) + require.NoError(t, s.DeleteObjects(ctx, []string{small, large, missing})) + keys, err = s.ListObjects(ctx, prefix) + require.NoError(t, err) + require.Empty(t, keys) + }) + + t.Run("PresignedURL", func(t *testing.T) { + url, err := s.PresignedURL(ctx, small, time.Minute) + require.NoError(t, err) + require.NotEmpty(t, url) + }) + + t.Run("IfAbsent race", func(t *testing.T) { + sizes := map[string][]byte{} + if !suite.singlePartIfAbsentRaces { + sizes["single part"] = payload + } + if !suite.multipartIfAbsentRaces && !suite.multipartIgnoresIfAbsent { + sizes["multipart"] = big + } + for name, body := range sizes { + t.Run(name, func(t *testing.T) { + testIfAbsentRace(t, s, prefix+"race-"+name, body) + }) + } + }) + + // An upload whose body fails partway stores nothing: a key it would create + // stays empty, and an object it would replace keeps its bytes. + t.Run("failed body", func(t *testing.T) { + for name, size := range map[string]int{"single part": 1 << 10, "multipart": 17 << 20} { + t.Run(name, func(t *testing.T) { + path := prefix + "failed-body/" + strings.ReplaceAll(name, " ", "-") + t.Cleanup(func() { _ = s.DeleteObjects(context.Background(), []string{path}) }) + failing := func() io.Reader { + return io.MultiReader(bytes.NewReader(make([]byte, size)), iotest.ErrReader(errors.New("body failed"))) + } + + require.Error(t, s.UploadObject(ctx, path, failing(), storage.PutOptions{})) + exists, err := s.ObjectExists(ctx, path) + require.NoError(t, err) + require.False(t, exists, "a failed upload created the object") + + require.NoError(t, s.PutObject(ctx, path, []byte("old"), storage.PutOptions{})) + require.Error(t, s.UploadObject(ctx, path, failing(), storage.PutOptions{})) + got, err := s.GetObject(ctx, path) + if suite.overwritesInPlace { + require.ErrorIs(t, err, storage.ErrNotFound) + return + } + require.NoError(t, err) + require.Equal(t, []byte("old"), got, "a failed upload replaced the object") + }) + } + }) + + t.Run("edge objects", func(t *testing.T) { + empty := prefix + "edge/empty" + odd := prefix + "edge/a dir/caf\u00e9 & r\u00e9sum\u00e9+1.txt" + nested := prefix + "edge/a dir/deeper/leaf" + t.Cleanup(func() { _ = s.DeleteObjects(ctx, []string{empty, odd, nested}) }) + + require.NoError(t, s.PutObject(ctx, empty, nil, storage.PutOptions{})) + got, err := s.GetObject(ctx, empty) + require.NoError(t, err) + require.Empty(t, got) + got, err = s.GetObjectRange(ctx, empty, 0, 4) + require.NoError(t, err) + require.Empty(t, got) + + require.NoError(t, s.PutObject(ctx, odd, []byte("first"), storage.PutOptions{})) + require.NoError(t, s.PutObject(ctx, odd, []byte("second"), storage.PutOptions{})) + got, err = s.GetObject(ctx, odd) + require.NoError(t, err) + require.Equal(t, []byte("second"), got) + require.NoError(t, s.PutObject(ctx, nested, []byte("leaf"), storage.PutOptions{})) + + keys, err := s.ListObjects(ctx, prefix+"edge/") + require.NoError(t, err) + require.Equal(t, []string{odd, nested, empty}, keys) + exists, err := s.ObjectExists(ctx, prefix+"edge/a dir") + require.NoError(t, err) + require.False(t, exists, "a prefix is not an object") + + // A prefix matches keys byte for byte, as on S3. + sibling := prefix + "edge/a dirt" + t.Cleanup(func() { _ = s.DeleteObjects(ctx, []string{sibling}) }) + require.NoError(t, s.PutObject(ctx, sibling, []byte("sibling"), storage.PutOptions{})) + keys, err = s.ListObjects(ctx, prefix+"edge/a dir/") + require.NoError(t, err) + require.Equal(t, []string{odd, nested}, keys) + keys, err = s.ListObjects(ctx, prefix+"edge/a dir") + require.NoError(t, err) + require.Equal(t, []string{odd, nested, sibling}, keys) + + _, err = s.GetObject(ctx, odd+"/child") + require.ErrorIs(t, err, storage.ErrNotFound, "a path through an object") + // A prefix holds no object, so deleting it succeeds and removes nothing. + require.NoError(t, s.DeleteObjects(ctx, []string{prefix + "edge/a dir"})) + exists, err = s.ObjectExists(ctx, nested) + require.NoError(t, err) + require.True(t, exists) + require.NoError(t, s.DeleteObjects(ctx, []string{sibling})) + + require.NoError(t, s.DeleteObjects(ctx, []string{empty, odd, nested})) + keys, err = s.ListObjects(ctx, prefix+"edge/") + require.NoError(t, err) + require.Empty(t, keys) + }) + + if suite.bulk { + t.Run("bulk delete", func(t *testing.T) { + // One past S3's and OSS's 1000 keys per request. + paths := make([]string, 1001) + for i := range paths { + paths[i] = fmt.Sprintf("%sbulk/%04d", prefix, i) + require.NoError(t, s.PutObject(ctx, paths[i], []byte("x"), storage.PutOptions{})) + } + keys, err := s.ListObjects(ctx, prefix+"bulk/") + require.NoError(t, err) + require.True(t, sort.StringsAreSorted(keys)) + require.Equal(t, paths, keys) + require.NoError(t, s.DeleteObjects(ctx, paths)) + keys, err = s.ListObjects(ctx, prefix+"bulk/") + require.NoError(t, err) + require.Empty(t, keys) + }) + } +} + +// testIfAbsentRace sends two IfAbsent uploads of body to path that both pass +// any check before uploading: each body holds its first read until the other's +// starts. Only the store's condition can then refuse one, and exactly one must +// win. +func testIfAbsentRace(t *testing.T, s storage.Storage, path string, body []byte) { + ctx := context.Background() + t.Cleanup(func() { _ = s.DeleteObjects(ctx, []string{path}) }) + + var arrived sync.WaitGroup + var late atomic.Bool + arrived.Add(2) + errs := make(chan error, 2) + for i := range 2 { + r := &heldReader{r: bytes.NewReader(append([]byte{byte(i)}, body...)), arrived: &arrived, late: &late} + go func() { errs <- s.UploadObject(ctx, path, r, storage.PutOptions{IfAbsent: true}) }() + } + var won []byte + for range 2 { + err := <-errs + if err == nil { + require.Nil(t, won, "both uploads won") + won = []byte{} + continue + } + require.ErrorIs(t, err, storage.ErrExists) + } + require.NotNil(t, won, "no upload won") + require.False(t, late.Load(), "the uploads never overlapped, so nothing raced") + // The object is one upload's whole body: the loser wrote nothing. + got, err := s.GetObject(ctx, path) + require.NoError(t, err) + require.Contains(t, [][]byte{append([]byte{0}, body...), append([]byte{1}, body...)}, got) +} + +// heldReader holds its first read until every reader sharing arrived has +// started one, or a while has passed, when it sets late. +type heldReader struct { + r io.Reader + arrived *sync.WaitGroup + late *atomic.Bool + once sync.Once +} + +func (h *heldReader) Read(p []byte) (int, error) { + h.once.Do(func() { + h.arrived.Done() + done := make(chan struct{}) + go func() { h.arrived.Wait(); close(done) }() + select { + case <-done: + case <-time.After(30 * time.Second): + h.late.Store(true) + } + }) + return h.r.Read(p) +} diff --git a/storage_test.go b/e2e/storage_test.go similarity index 75% rename from storage_test.go rename to e2e/storage_test.go index 0632202..dffcea1 100644 --- a/storage_test.go +++ b/e2e/storage_test.go @@ -12,15 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -package storage_test +package e2e import ( "bytes" + "context" "fmt" "io" "net/http" "net/url" "os" + "path/filepath" "strings" "testing" "time" @@ -50,6 +52,7 @@ func TestAliOSS(t *testing.T) { require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func TestAzure(t *testing.T) { @@ -69,6 +72,7 @@ func TestAzure(t *testing.T) { require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func TestGCP(t *testing.T) { @@ -86,13 +90,17 @@ func TestGCP(t *testing.T) { require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func TestLocal(t *testing.T) { + // An empty StorageDir is the working directory. + t.Chdir(t.TempDir()) s, err := storage.New(&storage.LocalConfig{}) require.NoError(t, err) testStorage(t, s) + testObjectsWith(t, s, objectsSuite{overwritesInPlace: true}) } func TestOCI(t *testing.T) { @@ -117,6 +125,7 @@ func TestOCI(t *testing.T) { require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func TestOCINative(t *testing.T) { @@ -145,6 +154,7 @@ func TestOCINative(t *testing.T) { require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func TestSupabase(t *testing.T) { @@ -169,6 +179,7 @@ func TestSupabase(t *testing.T) { require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func TestS3(t *testing.T) { @@ -180,16 +191,21 @@ func TestS3(t *testing.T) { t.Skip("Missing env vars") } + // S3_ENDPOINT reaches an S3-compatible store instead of AWS. + endpoint := os.Getenv("S3_ENDPOINT") s, err := storage.New(&storage.S3Config{ - AccessKey: key, - Secret: secret, - SessionToken: os.Getenv("AWS_SESSION_TOKEN"), - Region: os.Getenv("AWS_REGION"), - Bucket: bucket, + AccessKey: key, + Secret: secret, + SessionToken: os.Getenv("AWS_SESSION_TOKEN"), + Region: os.Getenv("AWS_REGION"), + Endpoint: endpoint, + ForcePathStyle: endpoint != "", + Bucket: bucket, }) require.NoError(t, err) testStorage(t, s) + testObjects(t, s) } func testStorage(t *testing.T, s storage.Storage) { @@ -199,21 +215,18 @@ func testStorage(t *testing.T, s storage.Storage) { pathBulk1 := prefix + "-bulk-1.txt" pathBulk2 := prefix + "-bulk-2.txt" - dataPayload := []byte("hello from UploadData") + dataPayload := []byte("hello from PutObject") filePayload := []byte("hello from UploadFile") bulk1Payload := []byte("bulk delete 1") bulk2Payload := []byte("bulk delete 2") // best-effort cleanup so a mid-test failure doesn't leave orphans in the bucket t.Cleanup(func() { - _ = s.DeleteObjects([]string{pathData, pathFile, pathBulk1, pathBulk2}) + _ = s.DeleteObjects(context.Background(), []string{pathData, pathFile, pathBulk1, pathBulk2}) }) - t.Run("UploadData", func(t *testing.T) { - loc, size, err := s.UploadData(dataPayload, pathData, "text/plain") - require.NoError(t, err) - require.Equal(t, int64(len(dataPayload)), size) - assertLocation(t, loc, pathData) + t.Run("PutObject", func(t *testing.T) { + require.NoError(t, s.PutObject(context.Background(), pathData, dataPayload, storage.PutOptions{ContentType: "text/plain"})) }) t.Run("UploadFile", func(t *testing.T) { @@ -225,14 +238,14 @@ func testStorage(t *testing.T, s storage.Storage) { require.NoError(t, err) require.NoError(t, tmp.Close()) - loc, size, err := s.UploadFile(tmp.Name(), pathFile, "text/plain") + loc, size, err := s.UploadFile(context.Background(), pathFile, tmp.Name(), storage.PutOptions{ContentType: "text/plain"}) require.NoError(t, err) require.Equal(t, int64(len(filePayload)), size) assertLocation(t, loc, pathFile) }) t.Run("ListObjects", func(t *testing.T) { - items, err := s.ListObjects(prefix) + items, err := s.ListObjects(context.Background(), prefix) require.NoError(t, err) require.Len(t, items, 2) @@ -241,7 +254,7 @@ func testStorage(t *testing.T, s storage.Storage) { }) t.Run("DownloadData", func(t *testing.T) { - downloaded, err := s.DownloadData(pathData) + downloaded, err := s.GetObject(context.Background(), pathData) require.NoError(t, err) require.Equal(t, dataPayload, downloaded) }) @@ -252,7 +265,7 @@ func testStorage(t *testing.T, s storage.Storage) { require.NoError(t, tmp.Close()) defer os.Remove(tmp.Name()) - size, err := s.DownloadFile(tmp.Name(), pathFile) + size, err := s.DownloadFile(context.Background(), pathFile, tmp.Name()) require.NoError(t, err) require.Equal(t, int64(len(filePayload)), size) @@ -261,14 +274,9 @@ func testStorage(t *testing.T, s storage.Storage) { require.Equal(t, filePayload, got) }) - t.Run("GeneratePresignedUrl", func(t *testing.T) { - rawURL, err := s.GeneratePresignedUrl(pathData, 5*time.Minute) - if err != nil { - // Some configurations (e.g. Azure without OAuth) don't support presigning; - // the method exists and reported a clear error, which is the contract. - t.Logf("presigned URL not available in this config: %v", err) - return - } + t.Run("PresignedURL", func(t *testing.T) { + rawURL, err := s.PresignedURL(context.Background(), pathData, 5*time.Minute) + require.NoError(t, err) require.NotEmpty(t, rawURL) // The URL should at least be parseable and reference the key we asked for. @@ -285,15 +293,16 @@ func testStorage(t *testing.T, s storage.Storage) { require.NoError(t, err) defer resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) + require.Equal(t, "text/plain", resp.Header.Get("Content-Type"), "the content type written") body, err := io.ReadAll(resp.Body) require.NoError(t, err) require.Equal(t, dataPayload, body) }) - t.Run("DeleteObject", func(t *testing.T) { - require.NoError(t, s.DeleteObject(pathData)) + t.Run("DeleteObjects one path", func(t *testing.T) { + require.NoError(t, s.DeleteObjects(context.Background(), []string{pathData})) - items, err := s.ListObjects(prefix) + items, err := s.ListObjects(context.Background(), prefix) require.NoError(t, err) require.Len(t, items, 1) require.True(t, strings.HasSuffix(items[0], pathFile)) @@ -301,14 +310,12 @@ func testStorage(t *testing.T, s storage.Storage) { t.Run("DeleteObjects", func(t *testing.T) { // Upload two more objects, then bulk-delete them along with the leftover pathFile. - _, _, err := s.UploadData(bulk1Payload, pathBulk1, "text/plain") - require.NoError(t, err) - _, _, err = s.UploadData(bulk2Payload, pathBulk2, "text/plain") - require.NoError(t, err) + require.NoError(t, s.PutObject(context.Background(), pathBulk1, bulk1Payload, storage.PutOptions{})) + require.NoError(t, s.PutObject(context.Background(), pathBulk2, bulk2Payload, storage.PutOptions{})) - require.NoError(t, s.DeleteObjects([]string{pathFile, pathBulk1, pathBulk2})) + require.NoError(t, s.DeleteObjects(context.Background(), []string{pathFile, pathBulk1, pathBulk2})) - items, err := s.ListObjects(prefix) + items, err := s.ListObjects(context.Background(), prefix) require.NoError(t, err) require.Empty(t, items) }) @@ -316,22 +323,24 @@ func testStorage(t *testing.T, s storage.Storage) { t.Run("Multipart", func(t *testing.T) { pathMultipart := prefix + "-multipart.bin" t.Cleanup(func() { - _ = s.DeleteObject(pathMultipart) + _ = s.DeleteObjects(context.Background(), []string{pathMultipart}) }) - // 6 MiB exceeds the S3 manager's default 5 MiB part size, forcing the - // multipart upload path (CreateMultipartUpload / UploadPart / Complete). - payload := make([]byte, 6<<20) + // 17 MiB is more than one 16 MiB part on S3, and more than one ranged + // GET reads it back. + payload := make([]byte, 17<<20) for i := range payload { payload[i] = byte(i) } + file := filepath.Join(t.TempDir(), "multipart") + require.NoError(t, os.WriteFile(file, payload, 0o600)) - loc, size, err := s.UploadData(payload, pathMultipart, "application/octet-stream") + loc, size, err := s.UploadFile(context.Background(), pathMultipart, file, storage.PutOptions{ContentType: "application/octet-stream"}) require.NoError(t, err) require.Equal(t, int64(len(payload)), size) assertLocation(t, loc, pathMultipart) - downloaded, err := s.DownloadData(pathMultipart) + downloaded, err := s.GetObject(context.Background(), pathMultipart) require.NoError(t, err) require.Equal(t, len(payload), len(downloaded)) require.True(t, bytes.Equal(payload, downloaded), "downloaded multipart content does not match uploaded") @@ -347,7 +356,7 @@ func hasSuffixIn(items []string, suffix string) bool { return false } -// assertLocation verifies the URL returned by UploadData/UploadFile is non-empty, +// assertLocation verifies the URL returned by UploadFile is non-empty, // parseable, and references the key we just uploaded. func assertLocation(t *testing.T, loc, key string) { t.Helper() diff --git a/error.go b/error.go deleted file mode 100644 index 99deaed..0000000 --- a/error.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import "fmt" - -var _ error = (*ErrorWithStatusCode)(nil) - -type ErrorWithStatusCode struct { - Err error - StatusCode int -} - -func (e *ErrorWithStatusCode) Error() string { - return fmt.Sprintf("err: %s, status code: %d", e.Err, e.StatusCode) -} - -func (e *ErrorWithStatusCode) Unwrap() error { - return e.Err -} diff --git a/error_test.go b/error_test.go index 3f0966b..a235772 100644 --- a/error_test.go +++ b/error_test.go @@ -22,9 +22,9 @@ import ( "cloud.google.com/go/storage" "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/aliyun/aliyun-oss-go-sdk/oss" awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" smithyhttp "github.com/aws/smithy-go/transport/http" - "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/stretchr/testify/require" "google.golang.org/api/googleapi" ) diff --git a/gcp.go b/gcp.go index c68b32a..ddf1301 100644 --- a/gcp.go +++ b/gcp.go @@ -17,21 +17,25 @@ package storage import ( "bytes" "context" - "encoding/base64" "errors" "fmt" "io" "net/http" "net/url" "os" + "slices" + "strings" "time" "cloud.google.com/go/storage" "github.com/googleapis/gax-go/v2" + "golang.org/x/oauth2" "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" "google.golang.org/api/googleapi" "google.golang.org/api/iterator" "google.golang.org/api/option" + htransport "google.golang.org/api/transport/http" ) const storageScope = "https://www.googleapis.com/auth/devstorage.read_write" @@ -59,6 +63,10 @@ func wrapGCPError(err error) error { type gcpStorage struct { conf *GCPConfig client *storage.Client + // signer is CredentialsJSON's service account, which signs URLs: with the + // credentials passed as a token source, the SDK finds no signer of its own + // off GCE. + signer *jwt.Config } func NewGCP(conf *GCPConfig) (Storage, error) { @@ -66,80 +74,76 @@ func NewGCP(conf *GCPConfig) (Storage, error) { conf: conf, } + // An http:// endpoint is an emulator, which takes no credentials; any other + // endpoint, such as Private Service Connect, keeps them. + emulator := strings.HasPrefix(conf.Endpoint, "http://") + setProxy, err := proxySetter(conf.ProxyConfig) + if err != nil { + return nil, err + } + t := transportWithTimeouts(http.DefaultTransport) + setProxy(t) + var opts []option.ClientOption if conf.CredentialsJSON != "" { jwtConfig, err := google.JWTConfigFromJSON([]byte(conf.CredentialsJSON), storageScope) if err != nil { return nil, err } - opts = append(opts, option.WithTokenSource(jwtConfig.TokenSource(context.Background()))) + u.signer = jwtConfig + if !emulator { + // Tokens are fetched through the same transport, so with the proxy + // and timeouts, not through http.DefaultClient. + tokens := &http.Client{Transport: t, Timeout: time.Minute} + opts = append(opts, option.WithTokenSource(jwtConfig.TokenSource(context.WithValue(context.Background(), oauth2.HTTPClient, tokens)))) + } + } + if conf.Endpoint != "" { + opts = append(opts, option.WithEndpoint(strings.TrimSuffix(conf.Endpoint, "/")+"/storage/v1/")) + } + if emulator { + opts = append(opts, option.WithoutAuthentication()) } - defaultTransport := http.DefaultTransport.(*http.Transport) - transportClone := defaultTransport.Clone() - - if conf.ProxyConfig != nil { - proxyUrl, err := url.Parse(conf.ProxyConfig.Url) + // The client sends through t, with the SDK's auth added on top. The SDK + // chooses a client certificate and the mTLS endpoint only on a transport of + // its own, so where GOOGLE_API_USE_CLIENT_CERTIFICATE asks for them, the + // client keeps the SDK's, without this package's timeouts and proxy. + ctx := context.Background() + if os.Getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE") != "true" { + authed, err := htransport.NewTransport(ctx, t, append(opts, option.WithScopes(storage.ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform"))...) if err != nil { return nil, err } - defaultTransport.Proxy = http.ProxyURL(proxyUrl) - if conf.ProxyConfig.Username != "" && conf.ProxyConfig.Password != "" { - auth := fmt.Sprintf("%s:%s", conf.ProxyConfig.Username, conf.ProxyConfig.Password) - basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth)) - defaultTransport.ProxyConnectHeader = http.Header{} - defaultTransport.ProxyConnectHeader.Add("Proxy-Authorization", basicAuth) - } + opts = append(opts, option.WithHTTPClient(&http.Client{Transport: authed})) } - client, err := storage.NewClient(context.Background(), opts...) - - // restore default transport - http.DefaultTransport = transportClone - if err != nil { + if u.client, err = storage.NewClient(ctx, opts...); err != nil { return nil, err } - - u.client = client return u, nil } -func (s *gcpStorage) UploadData(data []byte, storagePath, contentType string) (string, int64, error) { - return s.upload(bytes.NewReader(data), storagePath, contentType) -} - -func (s *gcpStorage) UploadFile(filepath, storagePath, contentType string) (string, int64, error) { +func (s *gcpStorage) UploadFile(ctx context.Context, storagePath, filepath string, opts PutOptions) (string, int64, error) { file, err := os.Open(filepath) if err != nil { return "", 0, err } defer file.Close() - return s.upload(file, storagePath, contentType) -} - -func (s *gcpStorage) upload(reader io.Reader, storagePath, contentType string) (string, int64, error) { - wc := s.client.Bucket(s.conf.Bucket).Object(storagePath).Retryer( - storage.WithBackoff(gax.Backoff{ - Initial: time.Millisecond * 100, - Max: time.Second * 5, - Multiplier: 2, - }), - storage.WithMaxAttempts(5), - storage.WithPolicy(storage.RetryAlways), - ).NewWriter(context.Background()) - wc.ContentType = contentType - wc.ChunkRetryDeadline = 0 - - n, err := io.Copy(wc, reader) + stat, err := file.Stat() if err != nil { - return "", 0, wrapGCPError(err) + return "", 0, err } - - if err = wc.Close(); err != nil { - return "", 0, wrapGCPError(err) + if err = s.UploadObject(ctx, storagePath, file, opts); err != nil { + return "", 0, err } + return s.location(storagePath), stat.Size(), nil +} - return s.location(storagePath), n, nil +func (s *gcpStorage) DownloadFile(ctx context.Context, storagePath, filepath string) (int64, error) { + return downloadFile(filepath, func(f *os.File) (int64, error) { + return s.DownloadObject(ctx, storagePath, f) + }) } func (s *gcpStorage) location(storagePath string) string { @@ -150,96 +154,168 @@ func (s *gcpStorage) location(storagePath string) string { }).String() } -func (s *gcpStorage) ListObjects(prefix string) ([]string, error) { - it := s.client.Bucket(s.conf.Bucket).Objects(context.Background(), &storage.Query{ - Prefix: prefix, - }) - - var objects []string - for { - attr, err := it.Next() - if err != nil { - if errors.Is(err, iterator.Done) { - return objects, nil - } - return nil, wrapGCPError(err) - } - objects = append(objects, attr.Name) +func (s *gcpStorage) PresignedURL(_ context.Context, storagePath string, expiration time.Duration) (string, error) { + opts := &storage.SignedURLOptions{ + Method: "GET", + Expires: time.Now().Add(expiration), } -} - -func (s *gcpStorage) DownloadData(storagePath string) ([]byte, error) { - rc, err := s.download(storagePath) - if err != nil { - return nil, err + // A V2 URL, which has no limit on its expiry, is always https; an http:// + // emulator takes V4, which signs for at most 7 days. A private endpoint, + // such as Private Service Connect, is not where the URL's reader is, so it + // names the public host, as location does. + if strings.HasPrefix(s.conf.Endpoint, "http://") { + opts.Scheme = storage.SigningSchemeV4 + opts.Insecure = true + } else if s.conf.Endpoint != "" { + opts.Hostname = "storage.googleapis.com" } - defer rc.Close() - - data, err := io.ReadAll(rc) + if s.signer != nil { + opts.GoogleAccessID = s.signer.Email + opts.PrivateKey = s.signer.PrivateKey + } + // Without a key of its own, signing calls IAM, which the SDK otherwise + // retries until the call's context ends, and SignedURL takes none. + u, err := s.client.Bucket(s.conf.Bucket).Retryer(storage.WithMaxAttempts(5)).SignedURL(storagePath, opts) if err != nil { - return nil, wrapGCPError(err) + return "", wrapGCPError(err) } - return data, nil + return u, nil } -func (s *gcpStorage) DownloadFile(filepath, storagePath string) (int64, error) { - file, err := os.Create(filepath) - if err != nil { - return 0, err +func (s *gcpStorage) object(storagePath string) *storage.ObjectHandle { + return s.client.Bucket(s.conf.Bucket).Object(storagePath) +} + +// PutObject sizes the upload buffer to data, so a small put sends one request +// without a full chunk's allocation. +func (s *gcpStorage) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + return s.write(ctx, storagePath, bytes.NewReader(data), opts, min(len(data)+1, googleapi.DefaultUploadChunkSize)) +} + +// UploadObject sends body in resumable chunks; IfAbsent is a precondition on +// the whole upload, so it is atomic here. +func (s *gcpStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + return s.write(ctx, storagePath, body, opts, googleapi.DefaultUploadChunkSize) +} + +// gcsRetryAlways retries a call the SDK would not, as it is not idempotent +// on its own: an overwrite, or a delete, where a missing object counts as +// deleted. +var gcsRetryAlways = []storage.RetryOption{ + storage.WithBackoff(gax.Backoff{Initial: 100 * time.Millisecond, Max: 5 * time.Second, Multiplier: 2}), + storage.WithMaxAttempts(5), + storage.WithPolicy(storage.RetryAlways), +} + +func (s *gcpStorage) write(ctx context.Context, storagePath string, body io.Reader, opts PutOptions, chunkSize int) error { + obj := s.object(storagePath) + if opts.IfAbsent { + // Conditional, so the SDK retries it by default. + obj = obj.If(storage.Conditions{DoesNotExist: true}) + } else { + obj = obj.Retryer(gcsRetryAlways...) } - defer file.Close() - rc, err := s.download(storagePath) - if err != nil { - return 0, err + ctx, cancel := context.WithCancel(ctx) + defer cancel() + w := obj.NewWriter(ctx) + w.ContentType = opts.ContentType + w.ChunkSize = chunkSize + // The SDK gives up retrying a chunk after 32s, which a 16 MiB chunk on a + // slow link takes to send; past the idle timeout, a stalled one is retried. + w.ChunkRetryDeadline = 2 * idleTimeout + if _, err := io.Copy(w, body); err != nil { + // Cancelling aborts the upload; Close would store what was sent. + cancel() + return s.writeError(err, opts.IfAbsent) } + return s.writeError(w.Close(), opts.IfAbsent) +} - _, err = io.Copy(file, rc) - _ = rc.Close() - if err != nil { - return 0, wrapGCPError(err) +// writeError maps a failed precondition to ErrExists only where IfAbsent +// set one: GCS answers other refusals with 412 too. +func (s *gcpStorage) writeError(err error, ifAbsent bool) error { + var apiErr *googleapi.Error + if ifAbsent && errors.As(err, &apiErr) && apiErr.Code == http.StatusPreconditionFailed { + return fmt.Errorf("%w: %w", ErrExists, wrapGCPError(err)) } + return wrapGCPObjectError(err) +} - return rc.Attrs.Size, nil +func (s *gcpStorage) GetObject(ctx context.Context, storagePath string) ([]byte, error) { + return getObject(ctx, s, storagePath) } -func (s *gcpStorage) download(storagePath string) (*storage.Reader, error) { - r, err := s.client.Bucket(s.conf.Bucket).Object(storagePath).Retryer( - storage.WithBackoff( - gax.Backoff{ - Initial: time.Millisecond * 100, - Max: time.Second * 5, - Multiplier: 2, - }), - storage.WithPolicy(storage.RetryAlways), - ).NewReader(context.Background()) +func (s *gcpStorage) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + r, err := s.object(storagePath).NewRangeReader(ctx, offset, length) + var apiErr *googleapi.Error + if errors.As(err, &apiErr) && apiErr.Code == http.StatusRequestedRangeNotSatisfiable { + return []byte{}, nil + } if err != nil { - return nil, wrapGCPError(err) + return nil, wrapGCPObjectError(err) } - return r, nil + defer r.Close() + data, err := io.ReadAll(r) + return data, wrapGCPObjectError(err) } -func (s *gcpStorage) GeneratePresignedUrl(storagePath string, expiration time.Duration) (string, error) { - u, err := s.client.Bucket(s.conf.Bucket).SignedURL(storagePath, &storage.SignedURLOptions{ - Method: "GET", - Expires: time.Now().Add(expiration), - }) +func (s *gcpStorage) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { + r, err := s.object(storagePath).NewReader(ctx) if err != nil { - return "", wrapGCPError(err) + return 0, wrapGCPObjectError(err) } - return u, nil + defer r.Close() + n, err := io.Copy(w, r) + return n, wrapGCPObjectError(err) } -func (s *gcpStorage) DeleteObject(storagePath string) error { - return wrapGCPError(s.client.Bucket(s.conf.Bucket).Object(storagePath).Delete(context.Background())) +func (s *gcpStorage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + _, err := s.object(storagePath).Attrs(ctx) + if errors.Is(err, storage.ErrObjectNotExist) { + return false, nil + } + return err == nil, wrapGCPObjectError(err) } -func (s *gcpStorage) DeleteObjects(storagePaths []string) error { - bucket := s.client.Bucket(s.conf.Bucket) - for _, path := range storagePaths { - if err := bucket.Object(path).Delete(context.Background()); err != nil { - return wrapGCPError(err) +func (s *gcpStorage) ListObjects(ctx context.Context, prefix string) ([]string, error) { + q := &storage.Query{Prefix: prefix} + if err := q.SetAttrSelection([]string{"Name"}); err != nil { + return nil, err + } + it := s.client.Bucket(s.conf.Bucket).Objects(ctx, q) + var keys []string + for { + attrs, err := it.Next() + if errors.Is(err, iterator.Done) { + slices.Sort(keys) + return keys, nil + } + if err != nil { + return nil, wrapGCPObjectError(err) } + keys = append(keys, attrs.Name) + } +} + +func (s *gcpStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { + return deleteEach(ctx, storagePaths, func(ctx context.Context, p string) error { + if err := s.object(p).Retryer(gcsRetryAlways...).Delete(ctx); err != nil && !errors.Is(err, storage.ErrObjectNotExist) { + return wrapGCPObjectError(err) + } + return nil + }) +} + +func wrapGCPObjectError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, storage.ErrObjectNotExist) { + return fmt.Errorf("%w: %w", ErrNotFound, wrapGCPError(err)) } - return nil + return wrapGCPError(err) } diff --git a/go.mod b/go.mod index ea3dbcd..dba78e6 100644 --- a/go.mod +++ b/go.mod @@ -15,11 +15,13 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 github.com/aws/smithy-go v1.28.1 github.com/googleapis/gax-go/v2 v2.26.2 - github.com/joho/godotenv v1.5.1 github.com/livekit/protocol v1.52.1 + github.com/livekit/storage/objects v0.1.0 github.com/oracle/oci-go-sdk/v65 v65.126.1 github.com/stretchr/testify v1.12.1 + golang.org/x/net v0.59.0 golang.org/x/oauth2 v0.37.0 + golang.org/x/sync v0.23.0 google.golang.org/api v0.300.0 ) @@ -88,8 +90,6 @@ require ( go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirect - golang.org/x/net v0.59.0 // indirect - golang.org/x/sync v0.23.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.16.0 // indirect diff --git a/go.sum b/go.sum index cd5cfc6..43f3c04 100644 --- a/go.sum +++ b/go.sum @@ -143,8 +143,6 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.22 h1:NU4XpII6jD+Dxcot94 github.com/googleapis/enterprise-certificate-proxy v0.3.22/go.mod h1:L3D/IQExI6LqEjBdXcZQ1WluSgigQmSwBboFstVPM4w= github.com/googleapis/gax-go/v2 v2.26.2 h1:ydkmNXxj7bEmmeK5AihkKnWxyOyBR9TDebvp5L5izk8= github.com/googleapis/gax-go/v2 v2.26.2/go.mod h1:sMKqnMesnKH+3wiRJROcttA+cJoZoGbZl1vDQ8XYtGk= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= diff --git a/go.work b/go.work new file mode 100644 index 0000000..0b274c8 --- /dev/null +++ b/go.work @@ -0,0 +1,7 @@ +go 1.26.0 + +use ( + . + ./e2e + ./objects +) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 0000000..410d71e --- /dev/null +++ b/go.work.sum @@ -0,0 +1 @@ +cloud.google.com/go/compute v1.62.0 h1:tJ7lKJ8YEVa6vZX03Jc8o1YePbjKDOQhDw1BscMZ1bs= diff --git a/local.go b/local.go index 03677fa..65c2fb4 100644 --- a/local.go +++ b/local.go @@ -15,12 +15,18 @@ package storage import ( + "bytes" + "context" + "errors" + "fmt" "io" + "io/fs" "net/url" "os" - "path" "path/filepath" + "slices" "strings" + "syscall" "time" ) @@ -39,173 +45,288 @@ func NewLocal(conf *LocalConfig) (Storage, error) { }, nil } -func (u *localUploader) UploadFile(localPath, storagePath string, _ string) (string, int64, error) { - storagePath = u.location(storagePath) +// file is the path that holds storagePath's object. A key that is empty or +// leads out of StorageDir names no object and is refused. +func (u *localUploader) file(storagePath string) (string, error) { + p := filepath.Join(u.StorageDir, filepath.FromSlash(storagePath)) + rel, err := filepath.Rel(u.StorageDir, p) + if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("storage: key %q is not under %s", storagePath, u.StorageDir) + } + return p, nil +} - local, err := os.Open(localPath) +func (u *localUploader) UploadFile(ctx context.Context, storagePath, localPath string, opts PutOptions) (string, int64, error) { + file, err := os.Open(localPath) if err != nil { return "", 0, err } - defer local.Close() + defer file.Close() + return u.upload(ctx, storagePath, file, opts) +} - if dir, _ := path.Split(storagePath); dir != "" { - if err = os.MkdirAll(dir, 0755); err != nil { - return "", 0, err - } +func (u *localUploader) DownloadFile(_ context.Context, storagePath, localPath string) (int64, error) { + f, err := u.open(storagePath) + if err != nil { + return 0, err } + defer f.Close() + return downloadFile(localPath, func(local *os.File) (int64, error) { + return io.Copy(local, f) + }) +} + +const localUploadPrefix = ".upload-" + +func (u *localUploader) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + _, _, err := u.upload(ctx, storagePath, bytes.NewReader(data), opts) + return err +} + +func (u *localUploader) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + _, _, err := u.upload(ctx, storagePath, body, opts) + return err +} - storage, err := os.Create(storagePath) +// upload writes the object in place, as on main: a mounted volume (gcsfuse, +// s3fs) sends a rename to its store as a copy, and a consumer watching for the +// file to be closed sees it. A reader can see the object while it is written, +// and a failed write removes it. IfAbsent writes a temporary file and links it +// into place, which is atomic. It returns the object's path and the bytes +// written, synced. +func (u *localUploader) upload(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) (string, int64, error) { + dst, err := u.file(storagePath) if err != nil { return "", 0, err } - defer storage.Close() + if err = ctx.Err(); err != nil { + return "", 0, err + } + if opts.IfAbsent { + return uploadIfAbsent(dst, body) + } - size, err := io.Copy(storage, local) + f, err := createFile(dst, os.O_TRUNC) if err != nil { return "", 0, err } - - return storagePath, size, nil + n, err := writeFile(f, body) + if err != nil { + _ = os.Remove(dst) + return "", 0, err + } + return dst, n, nil } -func (u *localUploader) UploadData(data []byte, storagePath, _ string) (string, int64, error) { - storagePath = u.location(storagePath) - - if dir, _ := path.Split(storagePath); dir != "" { - if err := os.MkdirAll(dir, 0755); err != nil { - return "", 0, err - } +func uploadIfAbsent(dst string, body io.Reader) (string, int64, error) { + tmp, err := createTemp(filepath.Dir(dst), localUploadPrefix) + if err != nil { + return "", 0, err } + defer os.Remove(tmp.Name()) - storage, err := os.Create(storagePath) + n, err := writeFile(tmp, body) if err != nil { return "", 0, err } - defer storage.Close() - - size, err := storage.Write(data) + // A hard link fails where it already exists, and on filesystems without + // hard links, such as SMB and most FUSE mounts. + if err = os.Link(tmp.Name(), dst); errors.Is(err, fs.ErrExist) { + return "", 0, ErrExists + } if err != nil { return "", 0, err } - - return storagePath, int64(size), nil + // The directory holds the new name: syncing it keeps the object across a + // crash. Best effort, as some filesystems, FUSE mounts among them, refuse + // to sync a directory. + if d, err := os.Open(filepath.Dir(dst)); err == nil { + _ = d.Sync() + _ = d.Close() + } + return dst, n, nil } -func (u *localUploader) location(storagePath string) string { - return path.Join(u.StorageDir, storagePath) +func (u *localUploader) GetObject(_ context.Context, storagePath string) ([]byte, error) { + f, err := u.open(storagePath) + if err != nil { + return nil, err + } + defer f.Close() + return io.ReadAll(f) } -func (u *localUploader) ListObjects(prefix string) ([]string, error) { - absPrefix := path.Join(u.StorageDir, prefix) - dir, filenamePrefix := path.Split(absPrefix) - - var files []string - entries, err := os.ReadDir(dir) +func (u *localUploader) GetObjectRange(_ context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + f, err := u.open(storagePath) if err != nil { - if os.IsNotExist(err) { - return nil, nil - } return nil, err } - - for _, entry := range entries { - if !strings.HasPrefix(entry.Name(), filenamePrefix) { - continue - } - - entryPath := path.Join(dir, entry.Name()) - if entry.IsDir() { - if err = filepath.Walk(entryPath, func(p string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - rel, err := filepath.Rel(u.StorageDir, p) - if err != nil { - return err - } - files = append(files, rel) - } - return nil - }); err != nil { - return nil, err - } - } else { - rel, err := filepath.Rel(u.StorageDir, entryPath) - if err != nil { - return nil, err - } - files = append(files, rel) - } + defer f.Close() + info, err := f.Stat() + if err != nil { + return nil, err } - return files, nil -} - -func (u *localUploader) DownloadData(storagePath string) ([]byte, error) { - return os.ReadFile(path.Join(u.StorageDir, storagePath)) + buf := make([]byte, max(0, min(length, info.Size()-offset))) + n, err := f.ReadAt(buf, offset) + if errors.Is(err, io.EOF) { + err = nil + } + return buf[:n], err } -func (u *localUploader) DownloadFile(localPath, storagePath string) (int64, error) { - storagePath = path.Join(u.StorageDir, storagePath) - - storage, err := os.Open(storagePath) +func (u *localUploader) DownloadObject(_ context.Context, storagePath string, w io.Writer) (int64, error) { + f, err := u.open(storagePath) if err != nil { return 0, err } - defer storage.Close() - - if dir, _ := path.Split(localPath); dir != "" { - if err = os.MkdirAll(dir, 0755); err != nil { - return 0, err - } - } + defer f.Close() + return io.Copy(w, f) +} - local, err := os.Create(localPath) +// open is the object's file. A directory is no object, as a prefix is not on +// a store. +func (u *localUploader) open(storagePath string) (*os.File, error) { + src, err := u.file(storagePath) if err != nil { - return 0, err + return nil, err } - defer local.Close() - - size, err := io.Copy(local, storage) + f, err := os.Open(src) if err != nil { - return 0, err + return nil, localObjectError(err) } - - return size, nil + if info, err := f.Stat(); err != nil || info.IsDir() { + f.Close() + if err == nil { + err = syscall.EISDIR + } + return nil, localObjectError(err) + } + return f, nil } -func (u *localUploader) GeneratePresignedUrl(storagePath string, _ time.Duration) (string, error) { - abs := filepath.Join(u.StorageDir, storagePath) - return (&url.URL{Scheme: "file", Path: filepath.ToSlash(abs)}).String(), nil +// ObjectExists reports a directory as no object, as a store reports a prefix. +func (u *localUploader) ObjectExists(_ context.Context, storagePath string) (bool, error) { + src, err := u.file(storagePath) + if err != nil { + return false, err + } + info, err := os.Stat(src) + if errors.Is(localObjectError(err), ErrNotFound) { + return false, nil + } + return err == nil && !info.IsDir(), err } -func (u *localUploader) DeleteObject(storagePath string) error { - storagePath = path.Join(u.StorageDir, storagePath) +// ListObjects walks the directory the prefix names and keeps the keys that +// start with it, as on S3: "a/" lists a's files, "a" those of a, ab and a.txt. +// A leading "/" is dropped, as from keys. UploadObject's temporary files, in +// progress or left by a crash, are left out. +func (u *localUploader) ListObjects(_ context.Context, prefix string) ([]string, error) { + prefix = strings.TrimLeft(prefix, "/") + root := u.StorageDir + if i := strings.LastIndex(prefix, "/"); i >= 0 { + dir, err := u.file(prefix[:i]) + if err != nil { + return nil, err + } + root = dir + } - for { - if err := os.Remove(storagePath); err != nil { + // The trailing separator has the walk follow a root that is a symlink. + walkRoot := root + string(filepath.Separator) + var keys []string + err := filepath.WalkDir(walkRoot, func(p string, d fs.DirEntry, err error) error { + if err != nil { + // A directory a delete removes mid-walk held no more keys; only a + // missing root means the prefix holds none. + if p != walkRoot && errors.Is(localObjectError(err), ErrNotFound) { + return nil + } return err } - - storagePath, _ = path.Split(storagePath) - storagePath = storagePath[:len(storagePath)-1] // remove trailing slash - entries, err := os.ReadDir(storagePath) + rel, err := filepath.Rel(u.StorageDir, p) if err != nil { return err } - - if storagePath == u.StorageDir || len(entries) > 0 { + key := filepath.ToSlash(rel) + if d.IsDir() { + // A directory none of whose keys can start with the prefix is not + // walked, so a listing reads only what it may return. + if p != walkRoot && !strings.HasPrefix(key+"/", prefix) && !strings.HasPrefix(prefix, key+"/") { + return fs.SkipDir + } return nil } + if !strings.HasPrefix(d.Name(), localUploadPrefix) && strings.HasPrefix(key, prefix) { + keys = append(keys, key) + } + return nil + }) + if errors.Is(localObjectError(err), ErrNotFound) { + return nil, nil } + slices.Sort(keys) + return keys, err } -func (u *localUploader) DeleteObjects(storagePaths []string) error { +func (u *localUploader) DeleteObjects(ctx context.Context, storagePaths []string) error { + failed := map[string]error{} for _, p := range storagePaths { - if err := u.DeleteObject(p); err != nil { - return err + err := ctx.Err() + if err == nil { + err = u.deleteObject(p) + } + if err != nil { + failed[p] = err + } + } + return deleteError(failed) +} + +// deleteObject removes the file, then each parent directory it leaves empty. A +// key that holds no file, a directory among them, is already deleted. +func (u *localUploader) deleteObject(storagePath string) error { + p, err := u.file(storagePath) + if err != nil { + return err + } + info, err := os.Lstat(p) + if errors.Is(localObjectError(err), ErrNotFound) || (err == nil && info.IsDir()) { + return nil + } + if err != nil { + return err + } + if err = os.Remove(p); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + + // A directory another upload just wrote into is not empty, and stays. + for dir := filepath.Dir(p); dir != u.StorageDir; dir = filepath.Dir(dir) { + if os.Remove(dir) != nil { + return nil } } return nil } + +func (u *localUploader) PresignedURL(_ context.Context, storagePath string, _ time.Duration) (string, error) { + p, err := u.file(storagePath) + if err != nil { + return "", err + } + return (&url.URL{Scheme: "file", Path: filepath.ToSlash(p)}).String(), nil +} + +// localObjectError reports a missing file as ErrNotFound, and so a path +// through a file (ENOTDIR) or onto a directory (EISDIR), where no object is. +func localObjectError(err error) error { + if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) || errors.Is(err, syscall.EISDIR) { + return fmt.Errorf("%w: %w", ErrNotFound, err) + } + return err +} diff --git a/location_test.go b/location_test.go index 6ea3646..acefa70 100644 --- a/location_test.go +++ b/location_test.go @@ -137,7 +137,7 @@ func TestAzureLocation(t *testing.T) { } for _, tc := range cases { s := &azureBLOBStorage{conf: &tc.conf} - got := s.location(tc.storagePath) + got := s.blobURL(tc.storagePath).String() require.Equal(t, tc.want, got, tc.name) requireWellFormedHTTPLocation(t, got, tc.name) } @@ -377,7 +377,16 @@ func TestLocalLocation(t *testing.T) { } for _, tc := range cases { u := &localUploader{StorageDir: tc.storageDir} - require.Equal(t, tc.want, u.location(tc.storagePath)) + got, err := u.file(tc.storagePath) + require.NoError(t, err) + require.Equal(t, tc.want, got) + } + + // No key reaches outside the directory or names the directory itself. + u := &localUploader{StorageDir: "/var/storage"} + for _, key := range []string{"", ".", "/", "..", "../x", "a/../../x", "a/.."} { + _, err := u.file(key) + require.Error(t, err, key) } } diff --git a/logging.go b/logging.go index 19eee3e..e39fe64 100644 --- a/logging.go +++ b/logging.go @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package storage import ( @@ -23,20 +24,20 @@ import ( "github.com/livekit/protocol/logger" ) -// S3Logger only logs aws messages on upload failure -type S3Logger struct { +// s3Logger only logs aws messages on upload failure +type s3Logger struct { mu sync.Mutex msgs []string idx int } -func NewS3Logger() *S3Logger { - return &S3Logger{ +func newS3Logger() *s3Logger { + return &s3Logger{ msgs: make([]string, 10), } } -func (l *S3Logger) Logf(classification logging.Classification, format string, v ...interface{}) { +func (l *s3Logger) Logf(classification logging.Classification, format string, v ...interface{}) { format = "aws %s: " + format v = append([]interface{}{strings.ToLower(string(classification))}, v...) @@ -46,7 +47,7 @@ func (l *S3Logger) Logf(classification logging.Classification, format string, v l.mu.Unlock() } -func (l *S3Logger) WriteLogs() { +func (l *s3Logger) WriteLogs() { l.mu.Lock() size := len(l.msgs) for range size { diff --git a/objects/go.mod b/objects/go.mod new file mode 100644 index 0000000..d3b4ffa --- /dev/null +++ b/objects/go.mod @@ -0,0 +1,3 @@ +module github.com/livekit/storage/objects + +go 1.26.0 diff --git a/objects/objects.go b/objects/objects.go new file mode 100644 index 0000000..c20e8b5 --- /dev/null +++ b/objects/objects.go @@ -0,0 +1,137 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package objects is the storage interfaces and errors without any provider +// SDK, for code that implements or accepts them without creating one, such as +// a fake: the root package imports every provider's SDK. +package objects + +import ( + "context" + "errors" + "fmt" + "io" + "maps" + "slices" + "time" +) + +// Bucket is object storage for a service that keeps and manages its own +// objects. A read of a missing object reports ErrNotFound and a write IfAbsent +// refuses ErrExists; test them with errors.Is. Where a store differs, every +// backend follows S3. +// +// A key is a store's key as given. The local directory is the exception, as a +// filesystem: it drops a leading "/", cleans "//" and "." segments, refuses a +// key that leads out of it, cannot hold both "a" and "a/b", and keeps its +// temporary files under names starting with ".upload-", which it never lists. +type Bucket interface { + // PutObject sends data, which is retried as a whole where a call fails. + PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error + // UploadObject streams body in parts and does not rewind it, so a retry + // needs a new reader. Memory is about the part size times the parts in + // flight: 16 MiB x 6 on S3, 4 MiB x 16 on Azure, one 16 MiB part on GCS and + // OCI, none on AliOSS (one chunked request, up to 5 GB, sent once) and local. + UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error + + GetObject(ctx context.Context, storagePath string) ([]byte, error) + // GetObjectRange reads length bytes from offset, with offset >= 0 and + // length > 0. A range running past the end returns what the object holds, + // one starting at or past it no bytes. + GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) + // DownloadObject streams the object to w, which holds part of it when an + // error comes back. An object replaced during a read fails it, with no + // sentinel. + DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) + // ObjectExists reports a missing object as false, not ErrNotFound. + ObjectExists(ctx context.Context, storagePath string) (bool, error) + + // ListObjects returns every key under prefix, recursively, as full paths, + // in byte order. + ListObjects(ctx context.Context, prefix string) ([]string, error) + // DeleteObjects succeeds for paths that hold nothing, as S3's delete does. + // It tries every path, and when some are not deleted fails with a + // *DeleteError naming them; every other path is deleted, and a retry of the + // same paths is safe. GCS, Azure and OCI send one request per path, 16 at + // once. + DeleteObjects(ctx context.Context, storagePaths []string) error + + // PresignedURL is a GET URL; local returns a file:// URL. + PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) +} + +// Storage is Bucket and the calls that publish and fetch whole files. +type Storage interface { + Bucket + + // UploadFile writes the file at storagePath and returns its location: a + // URL, or the file's path on local. + UploadFile(ctx context.Context, storagePath, filepath string, opts PutOptions) (location string, size int64, err error) + // DownloadFile writes the object to filepath, making its directory, only + // once the whole object is read: a failed download leaves filepath as it + // was. + DownloadFile(ctx context.Context, storagePath, filepath string) (size int64, err error) +} + +type PutOptions struct { + ContentType string + // IfAbsent fails with ErrExists when the path holds an object, atomically + // where the store honors the condition: an S3-compatible store that ignores + // If-None-Match overwrites, and AliOSS ignores it on a versioned bucket. A + // retry after a lost response can see the caller's own write as existing. + IfAbsent bool +} + +var ( + ErrNotFound = errors.New("storage: object not found") + ErrExists = errors.New("storage: object exists") +) + +// DeleteError is DeleteObjects' failure: Failed holds each path it did not +// delete, with why. Every other path it was given is deleted. +type DeleteError struct { + Failed map[string]error +} + +func (e *DeleteError) Error() string { + paths := slices.Sorted(maps.Keys(e.Failed)) + msg := fmt.Sprintf("storage: %d paths not deleted", len(paths)) + for i, p := range paths { + if i == 3 { + return msg + fmt.Sprintf(", and %d more", len(paths)-i) + } + msg += fmt.Sprintf(", %s: %v", p, e.Failed[p]) + } + return msg +} + +// Unwrap is the paths' errors, so errors.Is and errors.As see them. +func (e *DeleteError) Unwrap() []error { + return slices.Collect(maps.Values(e.Failed)) +} + +// ErrorWithStatusCode carries the HTTP status a store answered with, under +// ErrNotFound or ErrExists where those apply; find it with errors.As. +type ErrorWithStatusCode struct { + Err error + StatusCode int +} + +func (e *ErrorWithStatusCode) Error() string { + return fmt.Sprintf("err: %s, status code: %d", e.Err, e.StatusCode) +} + +func (e *ErrorWithStatusCode) Unwrap() error { + return e.Err +} diff --git a/oci.go b/oci.go index 1b7f5b5..4ee95a4 100644 --- a/oci.go +++ b/oci.go @@ -17,12 +17,15 @@ package storage import ( "bytes" "context" + "crypto/tls" "errors" "fmt" "io" + "net/http" "net/url" "os" "path" + "slices" "strings" "time" @@ -36,6 +39,8 @@ type ociStorage struct { client objectstorage.ObjectStorageClient namespace string host string + // insecure is an http:// Endpoint, such as an emulator's. + insecure bool } func NewOCI(conf *OCIConfig) (Storage, error) { @@ -55,10 +60,31 @@ func NewOCI(conf *OCIConfig) (Storage, error) { if conf.Endpoint != "" { client.Host = conf.Endpoint } + // The SDK's 60s http.Client.Timeout covers reading the body and would cut a + // long transfer; the idle timeout catches a dead connection instead. + // Its transport wrapper stays, as it loads OCI_DEFAULT_CERTS_PATH and + // refreshes the TLS config. + if hc, ok := client.HTTPClient.(*http.Client); ok { + if w, ok := hc.Transport.(*common.OciHTTPTransportWrapper); ok { + w.TransportTemplate = func(tlsConfig *tls.Config) (http.RoundTripper, error) { + t, err := common.DefaultTransport(tlsConfig) + if err != nil { + return nil, err + } + return transportWithTimeouts(t), nil + } + } + c := &http.Client{Transport: hc.Transport, CheckRedirect: hc.CheckRedirect, Jar: hc.Jar} + if os.Getenv(common.CustomClientTimeoutEnvVar) != "" { + c.Timeout = hc.Timeout + } + client.HTTPClient = c + } // The SDK only prepends a scheme to client.Host on the first request, so // read it before GetNamespace below. host := strings.TrimPrefix(client.Host, "https://") + insecure := strings.HasPrefix(host, "http://") host = strings.TrimPrefix(host, "http://") s := &ociStorage{ @@ -66,6 +92,7 @@ func NewOCI(conf *OCIConfig) (Storage, error) { client: client, namespace: conf.Namespace, host: host, + insecure: insecure, } if s.namespace == "" { @@ -73,16 +100,39 @@ func NewOCI(conf *OCIConfig) (Storage, error) { if conf.CompartmentID != "" { req.CompartmentId = &conf.CompartmentID } - resp, err := client.GetNamespace(context.Background(), req) - if err != nil { - return nil, wrapOCIError(err) + if s.namespace, err = lookupNamespace(client, req); err != nil { + return nil, err } - s.namespace = *resp.Value } return s, nil } +// lookupNamespace is the tenancy's namespace, or an error after a minute, as +// with the SDK's own client timeout. A canceled OCI call races in the SDK, so +// the lookup is not canceled but left to finish. +func lookupNamespace(client objectstorage.ObjectStorageClient, req objectstorage.GetNamespaceRequest) (string, error) { + type result struct { + namespace string + err error + } + done := make(chan result, 1) + go func() { + resp, err := client.GetNamespace(context.Background(), req) + if err != nil { + done <- result{err: wrapOCIError(err)} + return + } + done <- result{namespace: *resp.Value} + }() + select { + case r := <-done: + return r.namespace, r.err + case <-time.After(time.Minute): + return "", errors.New("storage: OCI namespace lookup took over a minute") + } +} + func newConfigProvider(conf *OCIConfig) (common.ConfigurationProvider, error) { if conf.UseWorkloadIdentity { return auth.OkeWorkloadIdentityConfigurationProvider() @@ -106,197 +156,330 @@ func newConfigProvider(conf *OCIConfig) (common.ConfigurationProvider, error) { return common.DefaultConfigProvider(), nil } -func (s *ociStorage) UploadData(data []byte, storagePath, contentType string) (string, int64, error) { - size := int64(len(data)) - body := common.NewOCIReadSeekCloser(io.NopCloser(bytes.NewReader(data))) - - location, err := s.upload(body, size, storagePath, contentType) - if err != nil { - return "", 0, err - } - return location, size, nil -} - -func (s *ociStorage) UploadFile(filepath, storagePath, contentType string) (string, int64, error) { +// UploadFile sends the file in one PutObject, up to OCI's 50 GiB, which the SDK +// can retry by seeking it, as on main: a multipart upload, which OCI advises +// past 100 MiB, needs more permissions and can leave uncommitted parts behind. +func (s *ociStorage) UploadFile(ctx context.Context, storagePath, filepath string, opts PutOptions) (string, int64, error) { file, err := os.Open(filepath) if err != nil { return "", 0, err } - defer func() { - _ = file.Close() - }() + defer file.Close() stat, err := file.Stat() if err != nil { return "", 0, err } - - location, err := s.upload(file, stat.Size(), storagePath, contentType) - if err != nil { + if err = s.put(ctx, storagePath, file, stat.Size(), opts); err != nil { return "", 0, err } - return location, stat.Size(), nil + return s.location(storagePath), stat.Size(), nil } -// upload takes a seekable body: the SDK re-reads it when it retries. -func (s *ociStorage) upload(body io.ReadCloser, size int64, storagePath, contentType string) (string, error) { - contentDisposition := "inline" - _, err := s.client.PutObject(context.Background(), objectstorage.PutObjectRequest{ - NamespaceName: &s.namespace, - BucketName: &s.conf.Bucket, - ObjectName: &storagePath, - PutObjectBody: body, - ContentLength: &size, - ContentType: &contentType, - ContentDisposition: &contentDisposition, +func (s *ociStorage) location(storagePath string) string { + return s.url(path.Join("/n", s.namespace, "b", s.conf.Bucket, "o", storagePath)) +} + +func (s *ociStorage) url(p string) string { + scheme := "https" + if s.insecure { + scheme = "http" + } + return (&url.URL{Scheme: scheme, Host: s.host, Path: p}).String() +} + +func (s *ociStorage) DownloadFile(ctx context.Context, storagePath, filepath string) (int64, error) { + return downloadFile(filepath, func(f *os.File) (int64, error) { + return s.DownloadObject(ctx, storagePath, f) + }) +} + +// PresignedURL creates a pre-authenticated request. Unlike an S3 +// presigned URL, a PAR is a durable object: it is listable and revocable, it +// costs an API call to mint, and it needs PAR_MANAGE on the bucket. Its access +// URI is returned only here and cannot be read back afterwards. +func (s *ociStorage) PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { + name := fmt.Sprintf("livekit-storage-%s-%d", storagePath, time.Now().UnixNano()) + expires := common.SDKTime{Time: time.Now().Add(expiration)} + + resp, err := s.client.CreatePreauthenticatedRequest(ctx, objectstorage.CreatePreauthenticatedRequestRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + CreatePreauthenticatedRequestDetails: objectstorage.CreatePreauthenticatedRequestDetails{ + Name: &name, + AccessType: objectstorage.CreatePreauthenticatedRequestDetailsAccessTypeObjectread, + ObjectName: &storagePath, + TimeExpires: &expires, + }, }) if err != nil { return "", wrapOCIError(err) } - return s.location(storagePath), nil + return s.url(*resp.AccessUri), nil } -func (s *ociStorage) location(storagePath string) string { - return (&url.URL{ - Scheme: "https", - Host: s.host, - Path: path.Join("/n", s.namespace, "b", s.conf.Bucket, "o", storagePath), - }).String() +func wrapOCIError(err error) error { + if err == nil { + return nil + } + var se common.ServiceError + if errors.As(err, &se) { + return &ErrorWithStatusCode{ + Err: err, + StatusCode: se.GetHTTPStatusCode(), + } + } + return err } -func (s *ociStorage) ListObjects(prefix string) ([]string, error) { - var objects []string - var start *string +const ociUploadPartSize = 16 << 20 - for { - resp, err := s.client.ListObjects(context.Background(), objectstorage.ListObjectsRequest{ - NamespaceName: &s.namespace, - BucketName: &s.conf.Bucket, - Prefix: &prefix, - Start: start, - }) - if err != nil { - return nil, wrapOCIError(err) - } +func (s *ociStorage) PutObject(ctx context.Context, storagePath string, data []byte, opts PutOptions) error { + body := common.NewOCIReadSeekCloser(io.NopCloser(bytes.NewReader(data))) + return s.put(ctx, storagePath, body, int64(len(data)), opts) +} - for _, obj := range resp.Objects { - objects = append(objects, *obj.Name) - } +// ociContentDisposition has browsers show an object rather than download it, +// as on S3. +var ociContentDisposition = common.String("inline") - if resp.NextStartWith == nil { - return objects, nil - } - start = resp.NextStartWith +// put sends body in one request. It must be seekable, so the SDK can re-read +// it when it retries. +func (s *ociStorage) put(ctx context.Context, storagePath string, body io.ReadCloser, size int64, opts PutOptions) error { + req := objectstorage.PutObjectRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &storagePath, + PutObjectBody: body, + ContentLength: &size, + ContentDisposition: ociContentDisposition, + } + if opts.ContentType != "" { + req.ContentType = &opts.ContentType + } + if opts.IfAbsent { + req.IfNoneMatch = common.String("*") } + _, err := s.client.PutObject(ctx, req) + return wrapOCIObjectError(err) } -func (s *ociStorage) DownloadData(storagePath string) ([]byte, error) { - resp, err := s.download(storagePath) - if err != nil { - return nil, err +// UploadObject sends a body smaller than a part with PutObject, otherwise a +// multipart upload of sequential parts through one buffer. IfAbsent is checked +// at create and again at commit, so it is atomic here. +func (s *ociStorage) UploadObject(ctx context.Context, storagePath string, body io.Reader, opts PutOptions) error { + // The condition is checked again at create and commit; this check spares a + // refused upload its traffic, and refuses it where a store ignores + // if-none-match on multipart uploads, as floci-oci does. + if opts.IfAbsent && presentBeforeUpload(ctx, s, storagePath) { + return ErrExists } - defer func() { - _ = resp.Content.Close() - }() - - data, err := io.ReadAll(resp.Content) - if err != nil { - return nil, wrapOCIError(err) + // The buffer grows only as far as the body goes, so a small one costs no + // whole part. + var first bytes.Buffer + if _, err := first.ReadFrom(io.LimitReader(body, ociUploadPartSize)); err != nil { + return err } - return data, nil -} + if first.Len() < ociUploadPartSize { + return s.PutObject(ctx, storagePath, first.Bytes(), opts) + } + part := first.Bytes() -func (s *ociStorage) DownloadFile(filepath, storagePath string) (int64, error) { - file, err := os.Create(filepath) + var ifNoneMatch, contentType *string + if opts.IfAbsent { + ifNoneMatch = common.String("*") + } + if opts.ContentType != "" { + contentType = &opts.ContentType + } + created, err := s.client.CreateMultipartUpload(ctx, objectstorage.CreateMultipartUploadRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + IfNoneMatch: ifNoneMatch, + CreateMultipartUploadDetails: objectstorage.CreateMultipartUploadDetails{ + Object: &storagePath, + ContentType: contentType, + ContentDisposition: ociContentDisposition, + }, + }) if err != nil { - return 0, err + return wrapOCIObjectError(err) } - defer file.Close() - resp, err := s.download(storagePath) - if err != nil { - return 0, err + // The abort runs on: it cannot be bounded by canceling it, which races in + // the SDK, and the caller's deadline may be what failed the upload. + if err = s.uploadParts(ctx, storagePath, created.UploadId, body, part, len(part), ifNoneMatch); err != nil { + go func() { + _, _ = s.client.AbortMultipartUpload(context.WithoutCancel(ctx), objectstorage.AbortMultipartUploadRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &storagePath, + UploadId: created.UploadId, + }) + }() } - defer func() { - _ = resp.Content.Close() - }() + return err +} + +// uploadParts sends part[:n], then the rest of body a part at a time, and +// commits. The condition goes on the commit only: on a part, a retry after a +// part that landed would read as the object existing. +func (s *ociStorage) uploadParts(ctx context.Context, storagePath string, uploadID *string, body io.Reader, part []byte, n int, ifNoneMatch *string) error { + var parts []objectstorage.CommitMultipartUploadPartDetails + for num := 1; n > 0; num++ { + size := int64(n) + resp, err := s.client.UploadPart(ctx, objectstorage.UploadPartRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &storagePath, + UploadId: uploadID, + UploadPartNum: common.Int(num), + ContentLength: &size, + UploadPartBody: io.NopCloser(bytes.NewReader(part[:n])), + }) + if err != nil { + return wrapOCIObjectError(err) + } + parts = append(parts, objectstorage.CommitMultipartUploadPartDetails{PartNum: common.Int(num), Etag: resp.ETag}) - if _, err = io.Copy(file, resp.Content); err != nil { - return 0, wrapOCIError(err) + n, err = io.ReadFull(body, part) + if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { + return err + } } - return *resp.ContentLength, nil + _, err := s.client.CommitMultipartUpload(ctx, objectstorage.CommitMultipartUploadRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &storagePath, + UploadId: uploadID, + IfNoneMatch: ifNoneMatch, + CommitMultipartUploadDetails: objectstorage.CommitMultipartUploadDetails{PartsToCommit: parts}, + }) + return wrapOCIObjectError(err) } -func (s *ociStorage) download(storagePath string) (objectstorage.GetObjectResponse, error) { - resp, err := s.client.GetObject(context.Background(), objectstorage.GetObjectRequest{ - NamespaceName: &s.namespace, - BucketName: &s.conf.Bucket, - ObjectName: &storagePath, - }) - return resp, wrapOCIError(err) +func (s *ociStorage) GetObject(ctx context.Context, storagePath string) ([]byte, error) { + return getObject(ctx, s, storagePath) } -// GeneratePresignedUrl creates a pre-authenticated request. Unlike an S3 -// presigned URL, a PAR is a durable object: it is listable and revocable, it -// costs an API call to mint, and it needs PAR_MANAGE on the bucket. Its access -// URI is returned only here and cannot be read back afterwards. -func (s *ociStorage) GeneratePresignedUrl(storagePath string, expiration time.Duration) (string, error) { - name := fmt.Sprintf("livekit-storage-%s-%d", storagePath, time.Now().UnixNano()) - expires := common.SDKTime{Time: time.Now().Add(expiration)} +func (s *ociStorage) GetObjectRange(ctx context.Context, storagePath string, offset, length int64) ([]byte, error) { + if offset < 0 || length <= 0 { + return nil, fmt.Errorf("storage: invalid range offset %d length %d", offset, length) + } + var buf bytes.Buffer + rng := fmt.Sprintf("bytes=%d-%d", offset, offset+length-1) + _, err := s.download(ctx, storagePath, &buf, &rng) + var se common.ServiceError + if errors.As(err, &se) && se.GetHTTPStatusCode() == http.StatusRequestedRangeNotSatisfiable { + return []byte{}, nil + } + // A range starting at the end can also come back 400 InvalidParameter, as on + // floci-oci; the object's size tells it from a bad request. + if errors.As(err, &se) && se.GetHTTPStatusCode() == http.StatusBadRequest { + head, headErr := s.client.HeadObject(ctx, objectstorage.HeadObjectRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &storagePath, + }) + if headErr == nil && head.ContentLength != nil && offset >= *head.ContentLength { + return []byte{}, nil + } + } + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (s *ociStorage) DownloadObject(ctx context.Context, storagePath string, w io.Writer) (int64, error) { + return s.download(ctx, storagePath, w, nil) +} - resp, err := s.client.CreatePreauthenticatedRequest(context.Background(), objectstorage.CreatePreauthenticatedRequestRequest{ +func (s *ociStorage) download(ctx context.Context, storagePath string, w io.Writer, rng *string) (int64, error) { + resp, err := s.client.GetObject(ctx, objectstorage.GetObjectRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, - CreatePreauthenticatedRequestDetails: objectstorage.CreatePreauthenticatedRequestDetails{ - Name: &name, - AccessType: objectstorage.CreatePreauthenticatedRequestDetailsAccessTypeObjectread, - ObjectName: &storagePath, - TimeExpires: &expires, - }, + ObjectName: &storagePath, + Range: rng, }) if err != nil { - return "", wrapOCIError(err) + return 0, wrapOCIObjectError(err) } - - return (&url.URL{ - Scheme: "https", - Host: s.host, - Path: *resp.AccessUri, - }).String(), nil + defer resp.Content.Close() + n, err := io.Copy(w, resp.Content) + return n, wrapOCIObjectError(err) } -func (s *ociStorage) DeleteObject(storagePath string) error { - _, err := s.client.DeleteObject(context.Background(), objectstorage.DeleteObjectRequest{ +func (s *ociStorage) ObjectExists(ctx context.Context, storagePath string) (bool, error) { + _, err := s.client.HeadObject(ctx, objectstorage.HeadObjectRequest{ NamespaceName: &s.namespace, BucketName: &s.conf.Bucket, ObjectName: &storagePath, }) - return wrapOCIError(err) + err = wrapOCIObjectError(err) + if errors.Is(err, ErrNotFound) { + return false, nil + } + return err == nil, err } -// DeleteObjects deletes one at a time — OCI has no bulk equivalent — and stops -// at the first failure, leaving earlier deletions in place, as gcp.go does. -func (s *ociStorage) DeleteObjects(storagePaths []string) error { - for _, storagePath := range storagePaths { - if err := s.DeleteObject(storagePath); err != nil { - return err +func (s *ociStorage) ListObjects(ctx context.Context, prefix string) ([]string, error) { + var keys []string + var start *string + for { + resp, err := s.client.ListObjects(ctx, objectstorage.ListObjectsRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + Prefix: &prefix, + Start: start, + }) + if err != nil { + return nil, wrapOCIObjectError(err) + } + for _, obj := range resp.Objects { + keys = append(keys, *obj.Name) + } + if resp.NextStartWith == nil { + slices.Sort(keys) + return keys, nil } + start = resp.NextStartWith } - return nil } -func wrapOCIError(err error) error { +func (s *ociStorage) DeleteObjects(ctx context.Context, storagePaths []string) error { + return deleteEach(ctx, storagePaths, func(ctx context.Context, p string) error { + _, err := s.client.DeleteObject(ctx, objectstorage.DeleteObjectRequest{ + NamespaceName: &s.namespace, + BucketName: &s.conf.Bucket, + ObjectName: &p, + }) + if err = wrapOCIObjectError(err); err != nil && !errors.Is(err, ErrNotFound) { + return err + } + return nil + }) +} + +func wrapOCIObjectError(err error) error { if err == nil { return nil } var se common.ServiceError if errors.As(err, &se) { - return &ErrorWithStatusCode{ - Err: err, - StatusCode: se.GetHTTPStatusCode(), + switch se.GetHTTPStatusCode() { + case http.StatusNotFound: + // A HEAD has no body, so its code is the SDK's BadErrorResponse. + // NotAuthorizedOrNotFound stays an error. + switch se.GetCode() { + case "ObjectNotFound", "BadErrorResponse": + return fmt.Errorf("%w: %w", ErrNotFound, wrapOCIError(err)) + } + case http.StatusPreconditionFailed: + return fmt.Errorf("%w: %w", ErrExists, wrapOCIError(err)) } } - return err + return wrapOCIError(err) } diff --git a/s3.go b/s3.go index ecba17d..6c723e4 100644 --- a/s3.go +++ b/s3.go @@ -16,20 +16,24 @@ package storage import ( "bytes" + "cmp" "context" - "encoding/base64" "errors" "fmt" "io" + "math/rand/v2" "net/http" "net/url" "os" "path" + "slices" "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/ratelimit" "github.com/aws/aws-sdk-go-v2/aws/retry" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/credentials/stscreds" @@ -37,7 +41,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/aws-sdk-go-v2/service/sts" - "github.com/aws/smithy-go/logging" + "github.com/aws/smithy-go" "github.com/aws/smithy-go/middleware" ) @@ -58,8 +62,8 @@ func wrapS3Error(err error) error { const defaultBucketLocation = "us-east-1" type s3Storage struct { - conf *S3Config - awsConf *aws.Config + conf *S3Config + client *s3.Client } func NewS3(conf *S3Config) (Storage, error) { @@ -80,11 +84,11 @@ func NewS3(conf *S3Config) (Storage, error) { return nil, err } - if conf.AssumeRoleArn != "" { + if conf.AssumeRoleARN != "" { stsSvc := sts.NewFromConfig(*awsConf) - cp = stscreds.NewAssumeRoleProvider(stsSvc, conf.AssumeRoleArn, func(o *stscreds.AssumeRoleOptions) { - if conf.AssumeRoleExternalId != "" { - o.ExternalID = aws.String(conf.AssumeRoleExternalId) + cp = stscreds.NewAssumeRoleProvider(stsSvc, conf.AssumeRoleARN, func(o *stscreds.AssumeRoleOptions) { + if conf.AssumeRoleExternalID != "" { + o.ExternalID = aws.String(conf.AssumeRoleExternalID) } }) awsConf, err = getConf(conf, cp) @@ -100,8 +104,8 @@ func NewS3(conf *S3Config) (Storage, error) { } return &s3Storage{ - conf: conf, - awsConf: awsConf, + conf: conf, + client: newS3Client(awsConf, conf), }, nil } @@ -116,8 +120,14 @@ func getConf(conf *S3Config, cp aws.CredentialsProvider) (*aws.Config, error) { o.Credentials = cp o.Retryer = func() aws.Retryer { return retry.NewStandard(func(o *retry.StandardOptions) { - o.MaxAttempts = conf.MaxRetries - o.MaxBackoff = conf.MaxRetryDelay + if conf.MaxRetries > 0 { + o.MaxAttempts = conf.MaxRetries + 1 + } + o.Backoff = retryBackoff{min: conf.MinRetryDelay, max: cmp.Or(conf.MaxRetryDelay, retry.DefaultMaxBackoff)} + // A Storage sends every call with one client, and the SDK's quota + // on retries is per client: one outage would use it up and fail + // every call at once. + o.RateLimiter = ratelimit.None o.Retryables = append(o.Retryables, retry.RetryableHTTPStatusCode{ Codes: map[int]struct{}{ http.StatusConflict: {}, @@ -127,21 +137,19 @@ func getConf(conf *S3Config, cp aws.CredentialsProvider) (*aws.Config, error) { }) } - if conf.ProxyConfig != nil { - proxyUrl, err := url.Parse(conf.ProxyConfig.Url) - if err != nil { - return err - } - s3Transport := http.DefaultTransport.(*http.Transport).Clone() - s3Transport.Proxy = http.ProxyURL(proxyUrl) - if conf.ProxyConfig.Username != "" && conf.ProxyConfig.Password != "" { - auth := fmt.Sprintf("%s:%s", conf.ProxyConfig.Username, conf.ProxyConfig.Password) - basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth)) - s3Transport.ProxyConnectHeader = http.Header{} - s3Transport.ProxyConnectHeader.Add("Proxy-Authorization", basicAuth) - } - o.HTTPClient = &http.Client{Transport: s3Transport} + // The SDK's own client, which keeps its pool defaults and is the only + // kind a custom CA bundle (AWS_CA_BUNDLE) can be added to. + setProxy, err := proxySetter(conf.ProxyConfig) + if err != nil { + return err } + // The idle timeout goes on in newS3Client: a defaults mode + // (AWS_DEFAULTS_MODE) replaces the dialer when a client is made. The + // SDK's read timeout is off, as the idle timeout bounds reads and writes. + o.HTTPClient = awshttp.NewBuildableClient(). + WithDialerOptions(setDialTimeouts). + WithTransportOptions(func(t *http.Transport) { t.TLSHandshakeTimeout = tlsHandshakeTimeout }, setProxy). + WithReadTimeout(0) return nil } @@ -163,7 +171,9 @@ func updateRegion(awsConf *aws.Config, bucket string) error { Bucket: &bucket, } - resp, err := s3.NewFromConfig(*awsConf).GetBucketLocation(context.Background(), req) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + resp, err := s3.NewFromConfig(*awsConf).GetBucketLocation(ctx, req) if err != nil { return wrapS3Error(err) } @@ -175,107 +185,146 @@ func updateRegion(awsConf *aws.Config, bucket string) error { return nil } -func (s *s3Storage) getClient(l logging.Logger) *s3.Client { - return s3.NewFromConfig(*s.awsConf, func(o *s3.Options) { - if l != nil { - o.Logger = l - o.ClientLogMode = aws.LogRequest | aws.LogResponse | aws.LogRetries - } +// retryBackoff waits a random time between min and a ceiling that doubles +// from a second each attempt, up to max: the SDK's full jitter, with a floor. +type retryBackoff struct{ min, max time.Duration } - o.UsePathStyle = s.conf.ForcePathStyle +func (b retryBackoff) BackoffDelay(attempt int, _ error) (time.Duration, error) { + ceiling := b.max + if attempt < 30 { + ceiling = min(b.max, time.Second< len(storagePaths) { - end = len(storagePaths) +func (s *s3Storage) ListObjects(ctx context.Context, prefix string) ([]string, error) { + var keys []string + paginator := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{ + Bucket: aws.String(s.conf.Bucket), + Prefix: aws.String(prefix), + }) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return nil, wrapS3Error(err) } - - objects := make([]types.ObjectIdentifier, 0, end-i) - for _, path := range storagePaths[i:end] { - objects = append(objects, types.ObjectIdentifier{Key: aws.String(path)}) + for _, obj := range page.Contents { + keys = append(keys, aws.ToString(obj.Key)) } + } + slices.Sort(keys) + return keys, nil +} - deleteInput := &s3.DeleteObjectsInput{ +// DeleteObjects fails when any key fails, even within a batch S3 accepted. +func (s *s3Storage) DeleteObjects(ctx context.Context, storagePaths []string) error { + failed := map[string]error{} + for i := 0; i < len(storagePaths); i += 1000 { + batch := storagePaths[i:min(i+1000, len(storagePaths))] + objects := make([]types.ObjectIdentifier, 0, len(batch)) + for _, p := range batch { + objects = append(objects, types.ObjectIdentifier{Key: aws.String(p)}) + } + input := &s3.DeleteObjectsInput{ Bucket: aws.String(s.conf.Bucket), - Delete: &types.Delete{ - Objects: objects, - Quiet: aws.Bool(true), - }, + Delete: &types.Delete{Objects: objects, Quiet: aws.Bool(true)}, } - if s.conf.Endpoint != "" { - // SHA256 is supported by most S3-compatible backends - deleteInput.ChecksumAlgorithm = types.ChecksumAlgorithmSha256 + // The request needs an integrity header: stores other than AWS take + // SHA-256 (OCI takes it or CRC32C), not the SDK's default CRC32. + if !awsEndpoint(s.conf.Endpoint) { + input.ChecksumAlgorithm = types.ChecksumAlgorithmSha256 } - - _, err := client.DeleteObjects(context.Background(), deleteInput) + out, err := s.client.DeleteObjects(ctx, input) if err != nil { - return wrapS3Error(err) + for _, p := range batch { + failed[p] = wrapS3Error(err) + } + continue + } + // S3 answers 200 and lists the keys it could not delete, each with a + // code but no status, kept as the API error its other calls return. + // NoSuchKey is a key already gone; any other is still there. + for _, e := range out.Errors { + if aws.ToString(e.Code) != "NoSuchKey" { + failed[aws.ToString(e.Key)] = &smithy.GenericAPIError{Code: aws.ToString(e.Code), Message: aws.ToString(e.Message)} + } } } + return deleteError(failed) +} - return nil +func (s *s3Storage) PresignedURL(ctx context.Context, storagePath string, expiration time.Duration) (string, error) { + res, err := s3.NewPresignClient(s.client).PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.conf.Bucket), + Key: aws.String(storagePath), + }, s3.WithPresignExpires(expiration)) + if err != nil { + return "", wrapS3Error(err) + } + return res.URL, nil +} + +// wrapS3ObjectError is wrapS3Error with ErrNotFound and ErrExists +// for the codes S3-compatible stores answer them with. +func wrapS3ObjectError(err error) error { + if err == nil { + return nil + } + var noSuchKey *types.NoSuchKey + var notFound *types.NotFound + if errors.As(err, &noSuchKey) || errors.As(err, ¬Found) { + return fmt.Errorf("%w: %w", ErrNotFound, wrapS3Error(err)) + } + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch apiErr.ErrorCode() { + case "NoSuchKey", "NotFound", "404": + return fmt.Errorf("%w: %w", ErrNotFound, wrapS3Error(err)) + } + } + return wrapS3Error(err) +} + +// wrapS3WriteError is wrapS3ObjectError with a failed precondition as +// ErrExists where IfAbsent set one. A read can fail one too: the downloader +// pins each range to the first's ETag, and an object replaced mid-read is no +// object that exists. +func wrapS3WriteError(err error, ifAbsent bool) error { + var apiErr smithy.APIError + if ifAbsent && errors.As(err, &apiErr) && apiErr.ErrorCode() == "PreconditionFailed" { + return fmt.Errorf("%w: %w", ErrExists, wrapS3Error(err)) + } + return wrapS3ObjectError(err) } diff --git a/storage.go b/storage.go index 0317964..be0b0be 100644 --- a/storage.go +++ b/storage.go @@ -12,25 +12,318 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package storage is object storage on S3 and S3-compatible stores, GCS, +// Azure Blob, OCI Object Storage, AliOSS and a local directory, behind the +// interfaces in storage/objects. New builds a Storage from a config. package storage -import "time" +import ( + "bytes" + "context" + "errors" + "io" + "io/fs" + "math/rand/v2" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" -type Storage interface { - UploadData(data []byte, storagePath, contentType string) (location string, size int64, err error) - UploadFile(filepath, storagePath, contentType string) (location string, size int64, err error) + "golang.org/x/net/http/httpproxy" + "golang.org/x/sync/errgroup" - ListObjects(prefix string) ([]string, error) + "github.com/livekit/storage/objects" +) - DownloadData(storagePath string) (data []byte, err error) - DownloadFile(filepath, storagePath string) (size int64, err error) +// The interfaces live in storage/objects so that code implementing or +// accepting them without creating one doesn't pull in every provider SDK. +type ( + Storage = objects.Storage + Bucket = objects.Bucket + PutOptions = objects.PutOptions + ErrorWithStatusCode = objects.ErrorWithStatusCode + DeleteError = objects.DeleteError +) - GeneratePresignedUrl(storagePath string, expiration time.Duration) (url string, err error) +var ( + ErrNotFound = objects.ErrNotFound + ErrExists = objects.ErrExists +) - DeleteObject(storagePath string) error - DeleteObjects(storagePaths []string) error +// presentBeforeUpload reports whether IfAbsent can refuse an upload before it is +// sent, because the object is there. A check that fails, such as a 403 for +// credentials that may write but not read, leaves the refusal to the condition +// the store checks when the upload completes. +func presentBeforeUpload(ctx context.Context, b Bucket, storagePath string) bool { + exists, err := b.ObjectExists(ctx, storagePath) + return err == nil && exists } -func New(conf Config) (Storage, error) { - return newStorage(conf) +// getObject is GetObject for a backend whose reads are streams. +func getObject(ctx context.Context, s Bucket, storagePath string) ([]byte, error) { + var buf bytes.Buffer + if _, err := s.DownloadObject(ctx, storagePath, &buf); err != nil { + return nil, err + } + return buf.Bytes(), nil } + +// createFile opens name for writing, with its parents, in the mode os.Create +// gives, 0666 less the umask, so other users, such as a host reading a +// container's volume, can read it. A delete can remove the parent once empty +// before the file is in it, so it is made again. +func createFile(name string, flag int) (*os.File, error) { + for { + if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil { + return nil, err + } + f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|flag, 0o666) + if !errors.Is(err, fs.ErrNotExist) { + return f, err + } + } +} + +// createTemp is a new file in dir named prefix and a random suffix. +func createTemp(dir, prefix string) (*os.File, error) { + for { + f, err := createFile(filepath.Join(dir, prefix+strconv.FormatUint(rand.Uint64(), 36)), os.O_EXCL) + if !errors.Is(err, fs.ErrExist) { + return f, err + } + } +} + +// writeFile copies body into f, syncs and closes it. +func writeFile(f *os.File, body io.Reader) (int64, error) { + n, err := io.Copy(f, body) + if err == nil { + err = f.Sync() + } + if closeErr := f.Close(); err == nil { + err = closeErr + } + return n, err +} + +// downloadFile fetches an object into a file beside name, renamed to name once +// complete, so a missing object or a failed download leaves name as it was. +func downloadFile(name string, fetch func(f *os.File) (int64, error)) (int64, error) { + tmp, err := createTemp(filepath.Dir(name), "."+filepath.Base(name)+".download-") + if err != nil { + return 0, err + } + defer os.Remove(tmp.Name()) + + n, err := fetch(tmp) + if closeErr := tmp.Close(); err == nil { + err = closeErr + } + if err != nil { + return 0, err + } + return n, os.Rename(tmp.Name(), name) +} + +// The clients this package builds cut a connection that stalls, never a slow +// but live transfer: a call's context bounds the call. +const ( + dialTimeout = 30 * time.Second + tlsHandshakeTimeout = 10 * time.Second +) + +// deleteConcurrency is how many deletes a DeleteObjects sends at once where a +// store takes one object per request (GCS, Azure, OCI): sequential, a large +// delete is one round trip per object. +const deleteConcurrency = 16 + +// deleteEach deletes every path with del, deleteConcurrency at a time, where a +// store takes one path per request. Once the context ends, the paths left are +// failed without a request: an OCI call canceled mid-flight races in its SDK. +func deleteEach(ctx context.Context, paths []string, del func(ctx context.Context, path string) error) error { + var mu sync.Mutex + failed := map[string]error{} + var g errgroup.Group + g.SetLimit(deleteConcurrency) + for _, p := range paths { + g.Go(func() error { + err := ctx.Err() + if err == nil { + err = del(ctx, p) + } + if err != nil { + mu.Lock() + failed[p] = err + mu.Unlock() + } + return nil + }) + } + _ = g.Wait() + return deleteError(failed) +} + +// deleteError is a *DeleteError for the paths in failed, or nil for none. +func deleteError(failed map[string]error) error { + if len(failed) == 0 { + return nil + } + return &DeleteError{Failed: failed} +} + +// idleTimeout is how long a connection may move no bytes either way. A store +// can take minutes to answer a large commit. +var idleTimeout = 5 * time.Minute + +func setDialTimeouts(d *net.Dialer) { + d.Timeout = dialTimeout + d.KeepAlive = 30 * time.Second +} + +// setIdleTimeout has t's connections fail once idle for idleTimeout. A dialer t +// already has, such as one through a VPN, is kept. +func setIdleTimeout(t *http.Transport) { + dial := t.DialContext + if dial == nil { + d := &net.Dialer{} + setDialTimeouts(d) + dial = d.DialContext + } + timeout := idleTimeout + t.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + conn, err := dial(ctx, network, addr) + if err != nil { + return nil, err + } + return &idleConn{Conn: conn, timeout: timeout}, nil + } +} + +// idleConn fails once no bytes move either way for timeout. Each read or write +// moves both deadlines: the transport's read for the response is already +// waiting while a long body is still being written. A deadline set from above, +// such as TLS's on close or net/http's to interrupt a read, is never extended. +type idleConn struct { + net.Conn + timeout time.Duration + // readBy and writeBy are the deadlines set from above, in Unix + // nanoseconds, 0 for none. + readBy, writeBy atomic.Int64 +} + +func (c *idleConn) Read(p []byte) (int, error) { + if err := c.extend(); err != nil { + return 0, err + } + return c.Conn.Read(p) +} + +func (c *idleConn) Write(p []byte) (int, error) { + if err := c.extend(); err != nil { + return 0, err + } + return c.Conn.Write(p) +} + +func (c *idleConn) extend() error { + idle := time.Now().Add(c.timeout) + if err := c.Conn.SetReadDeadline(earliest(idle, c.readBy.Load())); err != nil { + return err + } + return c.Conn.SetWriteDeadline(earliest(idle, c.writeBy.Load())) +} + +func (c *idleConn) SetDeadline(t time.Time) error { + c.readBy.Store(unixNano(t)) + c.writeBy.Store(unixNano(t)) + return c.Conn.SetDeadline(t) +} + +func (c *idleConn) SetReadDeadline(t time.Time) error { + c.readBy.Store(unixNano(t)) + return c.Conn.SetReadDeadline(t) +} + +func (c *idleConn) SetWriteDeadline(t time.Time) error { + c.writeBy.Store(unixNano(t)) + return c.Conn.SetWriteDeadline(t) +} + +// earliest is idle, or the deadline set from above when that comes first. +func earliest(idle time.Time, by int64) time.Time { + if by != 0 && by < idle.UnixNano() { + return time.Unix(0, by) + } + return idle +} + +func unixNano(t time.Time) int64 { + if t.IsZero() { + return 0 + } + return t.UnixNano() +} + +// maxIdleConnsPerHost keeps a connection for each request a call can have in +// flight, deletes and upload blocks: Go's default keeps two and redials the rest. +const maxIdleConnsPerHost = 32 + +// transportWithTimeouts is a clone of t, or of Go's default transport, with +// every timeout set. +func transportWithTimeouts(rt http.RoundTripper) *http.Transport { + // A wrapped transport, such as a tracing one, cannot be cloned, so the + // client gets a plain one with Go's defaults instead. + t, ok := rt.(*http.Transport) + if ok { + t = t.Clone() + } else { + t = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + ExpectContinueTimeout: time.Second, + } + } + t.MaxIdleConnsPerHost = max(t.MaxIdleConnsPerHost, maxIdleConnsPerHost) + t.TLSHandshakeTimeout = tlsHandshakeTimeout + setIdleTimeout(t) + return t +} + +// proxySetter routes a transport through p, or leaves it as it is when p is +// nil. A bad URL fails here, so the setter itself can be a transport option. +func proxySetter(p *ProxyConfig) (func(*http.Transport), error) { + if p == nil { + return func(*http.Transport) {}, nil + } + u, err := url.Parse(p.URL) + if err != nil { + return nil, err + } + // Credentials in the URL go to the proxy for http targets as well as on + // CONNECT for https ones. + if p.Username != "" && p.Password != "" { + u.User = url.UserPassword(p.Username, p.Password) + } + // NO_PROXY is honored, and the cloud metadata services, which hand out + // instance credentials, are never proxied: a proxy cannot reach them. + proxy := (&httpproxy.Config{ + HTTPProxy: u.String(), + HTTPSProxy: u.String(), + NoProxy: strings.Join([]string{os.Getenv("NO_PROXY"), os.Getenv("no_proxy"), metadataHosts}, ","), + }).ProxyFunc() + return func(t *http.Transport) { + t.Proxy = func(r *http.Request) (*url.URL, error) { return proxy(r.URL) } + }, nil +} + +// metadataHosts are the instance metadata and container credential endpoints +// of EC2, ECS, EKS Pod Identity and GCE. +const metadataHosts = "169.254.169.254,169.254.170.2,169.254.170.23,fd00:ec2::254,fd00:ec2::23,metadata.google.internal" diff --git a/transport_test.go b/transport_test.go new file mode 100644 index 0000000..a6135bd --- /dev/null +++ b/transport_test.go @@ -0,0 +1,127 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// A connection is cut when no bytes move for idleTimeout, and never while a +// slow transfer is still moving them. +func TestIdleTimeout(t *testing.T) { + saved := idleTimeout + idleTimeout = time.Second + t.Cleanup(func() { idleTimeout = saved }) + client := &http.Client{Transport: transportWithTimeouts(nil)} + + t.Run("slow upload", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + })) + t.Cleanup(srv.Close) + // The second request reuses the first's connection. + for range 2 { + resp, err := client.Post(srv.URL, "text/plain", &trickle{n: 10}) + require.NoError(t, err) + resp.Body.Close() + } + }) + + t.Run("slow download", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(w, &trickle{n: 10, flush: w.(http.Flusher)}) + })) + t.Cleanup(srv.Close) + resp, err := client.Get(srv.URL) + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Len(t, body, 10) + }) + + t.Run("stalled", func(t *testing.T) { + start := time.Now() + _, err := client.Get(stalled(t)) + require.Error(t, err) + require.Less(t, time.Since(start), 5*time.Second) + }) + + // Other defaults modes have the SDK replace the dialer the timeout is on. + t.Run("S3 in standard defaults mode", func(t *testing.T) { + t.Setenv("AWS_DEFAULTS_MODE", "standard") + s, err := NewS3(&S3Config{AccessKey: "key", Secret: "secret", Region: "us-east-1", Endpoint: stalled(t), ForcePathStyle: true, Bucket: "bucket"}) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + _, err = s.ObjectExists(ctx, "path") + require.Error(t, err) + require.NoError(t, ctx.Err(), "the context, not the idle timeout, ended the call") + }) +} + +// A deadline set from above, as TLS sets one to send its close alert, holds: +// reads and writes do not push it out to the idle timeout. +func TestIdleConnKeepsDeadlines(t *testing.T) { + client, server := net.Pipe() + t.Cleanup(func() { client.Close(); server.Close() }) + c := &idleConn{Conn: client, timeout: time.Hour} + + require.NoError(t, c.SetDeadline(time.Now().Add(100*time.Millisecond))) + start := time.Now() + _, err := c.Write([]byte("x")) // nobody reads the other end + require.ErrorIs(t, err, os.ErrDeadlineExceeded) + _, err = c.Read(make([]byte, 1)) + require.ErrorIs(t, err, os.ErrDeadlineExceeded) + require.Less(t, time.Since(start), 5*time.Second) +} + +// stalled is a server that accepts requests and never answers them. +func stalled(t *testing.T) string { + done := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-done + })) + t.Cleanup(func() { close(done); srv.Close() }) + return srv.URL +} + +// trickle yields n bytes, one every 100ms, a tenth of the idle timeout. +type trickle struct { + n int + flush http.Flusher +} + +func (r *trickle) Read(p []byte) (int, error) { + if r.n == 0 { + return 0, io.EOF + } + if r.flush != nil { + r.flush.Flush() + } + time.Sleep(100 * time.Millisecond) + r.n-- + p[0] = 'x' + return 1, nil +}