feat(server/s3): support multipart upload

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/KarpelesLab/reflink v1.0.2
 	github.com/KirCute/zip v1.0.1
 	github.com/OpenListTeam/go-cache v0.1.0
+	github.com/OpenListTeam/gofakes3 v0.8.0
 	github.com/OpenListTeam/sftpd-openlist v1.0.1
 	github.com/OpenListTeam/tache v0.2.2
 	github.com/OpenListTeam/times v0.1.0
@@ -51,7 +52,6 @@ require (
 	github.com/hekmon/transmissionrpc/v3 v3.0.0
 	github.com/henrybear327/go-proton-api v1.0.0
 	github.com/ipfs/go-ipfs-api v0.7.0
-	github.com/itsHenry35/gofakes3 v0.0.8
 	github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb
 	github.com/json-iterator/go v1.1.12
 	github.com/kdomanski/iso9660 v0.4.0

go.sum

@@ -39,6 +39,8 @@ github.com/OpenListTeam/115-sdk-go v0.2.5 h1:E4O7GZmEXlGnouZ/e9//xU7NepT9JJkWtnF
 github.com/OpenListTeam/115-sdk-go v0.2.5/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU=
 github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI=
 github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4=
+github.com/OpenListTeam/gofakes3 v0.8.0 h1:Y+Stbb29qfDUrGYmK0Tqzk8c0QXwK4Pkmk6lG5zpWwk=
+github.com/OpenListTeam/gofakes3 v0.8.0/go.mod h1:DsJxAVrGCaleujFKrAYyGAjB5FtbPVF8IwAFYZ6pBKM=
 github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs=
 github.com/OpenListTeam/gsync v0.1.0/go.mod h1:h/Rvv9aX/6CdW/7B8di3xK3xNV8dUg45Fehrd/ksZ9s=
 github.com/OpenListTeam/reflink v0.0.0-20260701021214-78760eaeafef h1:67uGHancMF/abMrnkc8abVUWQiG73Wk5d8CKt3RzkFo=
@@ -427,8 +429,6 @@ github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg=
 github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk=
 github.com/ipfs/go-ipfs-api v0.7.0 h1:CMBNCUl0b45coC+lQCXEVpMhwoqjiaCwUIrM+coYW2Q=
 github.com/ipfs/go-ipfs-api v0.7.0/go.mod h1:AIxsTNB0+ZhkqIfTZpdZ0VR/cpX5zrXjATa3prSay3g=
-github.com/itsHenry35/gofakes3 v0.0.8 h1:1AgOl04IgoUV5r/WSK7ycnvwfpgharYLfVTmnzk5miw=
-github.com/itsHenry35/gofakes3 v0.0.8/go.mod h1:gQwOJ7LoH5QSpCVmjzC6oKp+MS71utLS7GHtonsvD0c=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=

internal/conf/config.go

@@ -83,9 +83,10 @@ type Cors struct {
 }
 
 type S3 struct {
-	Enable bool `json:"enable" env:"ENABLE"`
-	Port   int  `json:"port" env:"PORT"`
-	SSL    bool `json:"ssl" env:"SSL"`
+	Enable       bool   `json:"enable" env:"ENABLE"`
+	Port         int    `json:"port" env:"PORT"`
+	SSL          bool   `json:"ssl" env:"SSL"`
+	MultipartTTL string `json:"multipart_ttl" env:"MULTIPART_TTL"`
 }
 
 type FTP struct {

server/s3/backend.go

@@ -23,7 +23,7 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/internal/stream"
 	"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
-	"github.com/itsHenry35/gofakes3"
+	"github.com/OpenListTeam/gofakes3"
 	"github.com/ncw/swift/v2"
 	log "github.com/sirupsen/logrus"
 )
@@ -33,17 +33,23 @@ var (
 	timeFormat  = "Mon, 2 Jan 2006 15:04:05 GMT"
 )
 
-// s3Backend implements the gofacess3.Backend interface to make an S3
-// backend for gofakes3
+// s3Backend implements the gofakes3.Backend interface to make an S3
+// backend for gofakes3. It also implements gofakes3.MultipartBackend so that
+// multipart uploads are streamed to local temp files part-by-part and
+// assembled into storage on completion, instead of being buffered in memory.
 type s3Backend struct {
-	meta *sync.Map
+	meta    *sync.Map
+	uploads *sync.Map // map[gofakes3.UploadID]*multipartState
 }
 
 // newBackend creates a new SimpleBucketBackend.
 func newBackend() gofakes3.Backend {
-	return &s3Backend{
-		meta: new(sync.Map),
+	b := &s3Backend{
+		meta:    new(sync.Map),
+		uploads: new(sync.Map),
 	}
+	b.startReaper()
+	return b
 }
 
 // ListBuckets always returns the default bucket.
@@ -232,9 +238,20 @@ func (b *s3Backend) PutObject(
 	meta map[string]string,
 	input io.Reader, size int64,
 ) (result gofakes3.PutObjectResult, err error) {
+	return result, b.putStream(ctx, bucketName, objectName, meta, input, size)
+}
+
+// putStream stores the given object into the underlying storage. It is shared
+// by PutObject and the multipart-upload Complete step so both paths apply the
+// same directory creation, metadata and ignore rules.
+func (b *s3Backend) putStream(
+	ctx context.Context, bucketName, objectName string,
+	meta map[string]string,
+	input io.Reader, size int64,
+) error {
 	bucket, err := getBucketByName(bucketName)
 	if err != nil {
-		return result, err
+		return err
 	}
 	bucketPath := bucket.Path
 
@@ -260,15 +277,15 @@ func (b *s3Backend) PutObject(
 			log.Debugf("reqPath: %s not found and objectName contains /, need to makeDir", reqPath)
 			err = fs.MakeDir(ctx, reqPath)
 			if err != nil {
-				return result, errors.WithMessagef(err, "failed to makeDir, reqPath: %s", reqPath)
+				return errors.WithMessagef(err, "failed to makeDir, reqPath: %s", reqPath)
 			}
 		} else {
-			return result, gofakes3.KeyNotFound(objectName)
+			return gofakes3.KeyNotFound(objectName)
 		}
 	}
 
 	if isDir {
-		return result, nil
+		return nil
 	}
 
 	var ti time.Time
@@ -294,7 +311,7 @@ func (b *s3Backend) PutObject(
 	}
 	// Check if system file should be ignored
 	if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) {
-		return result, errs.IgnoredSystemFile
+		return errs.IgnoredSystemFile
 	}
 	stream := &stream.FileStream{
 		Obj:      &obj,
@@ -304,18 +321,12 @@ func (b *s3Backend) PutObject(
 
 	err = fs.PutDirectly(ctx, reqPath, stream)
 	if err != nil {
-		return result, err
+		return err
 	}
 
-	// if err := stream.Close(); err != nil {
-	// 	// remove file when close error occurred (FsPutErr)
-	// 	_ = fs.Remove(ctx, fp)
-	// 	return result, err
-	// }
-
 	b.meta.Store(fp, meta)
 
-	return result, nil
+	return nil
 }
 
 // DeleteMulti deletes multiple objects in a single request.

server/s3/list.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/itsHenry35/gofakes3"
+	"github.com/OpenListTeam/gofakes3"
 	log "github.com/sirupsen/logrus"
 )
 

server/s3/logger.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
-	"github.com/itsHenry35/gofakes3"
+	"github.com/OpenListTeam/gofakes3"
 )
 
 // logger output formatted message