From 75003f1d5f0c188efb37d8a90f72b05576c3594d Mon Sep 17 00:00:00 2001 From: Julien Salleyron Date: Fri, 6 Nov 2026 13:00:23 +0100 Subject: [PATCH 1/3] feat: add plugin states endpoint and partial update for the scraper Piceus can load the state of all plugins in one call and update only stars, hidden or tagsETag without sending the whole plugin. --- cmd/serve/serve.go | 8 ++ pkg/db/db.go | 26 ++++++ pkg/db/mongodb/bootstrap.go | 2 +- pkg/db/mongodb/plugindb.go | 84 ++++++++++++++++- pkg/db/mongodb/plugindb_test.go | 132 ++++++++++++++++++++++++++- pkg/handlers/dbmock_test.go | 10 +++ pkg/handlers/handlers.go | 1 + pkg/handlers/state.go | 116 ++++++++++++++++++++++++ pkg/handlers/state_test.go | 155 ++++++++++++++++++++++++++++++++ 9 files changed, 528 insertions(+), 6 deletions(-) create mode 100644 pkg/handlers/state.go create mode 100644 pkg/handlers/state_test.go diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index cd29bf9..50758b3 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -57,6 +57,7 @@ func run(ctx context.Context, cfg Config) error { r.Handle("/external/", buildExternalRouter(handler)) registerInternalBlacklistRoutes(r, handler) + registerInternalStatesRoute(r, handler) r.HandleFunc("/live", healthChecker.Live) r.HandleFunc("/ready", healthChecker.Ready) @@ -82,6 +83,7 @@ func buildInternalRouter(handler handlers.Handlers) http.Handler { r.Handler(http.MethodGet, "/", otelhttp.NewHandler(http.HandlerFunc(handler.List), "internal_list")) r.Handler(http.MethodPost, "/", otelhttp.NewHandler(http.HandlerFunc(handler.Create), "internal_create")) r.Handler(http.MethodPut, "/:uuid", otelhttp.NewHandler(http.HandlerFunc(handler.Update), "internal_update")) + r.Handler(http.MethodPatch, "/:uuid", otelhttp.NewHandler(http.HandlerFunc(handler.Patch), "internal_patch")) r.Handler(http.MethodDelete, "/:uuid", otelhttp.NewHandler(http.HandlerFunc(handler.Delete), "internal_delete")) r.NotFound = http.HandlerFunc(handlers.NotFound) @@ -99,6 +101,12 @@ func registerInternalBlacklistRoutes(r *http.ServeMux, handler handlers.Handlers r.Handle("DELETE /internal/blacklist/{owner}/{repo}", otelhttp.NewHandler(http.HandlerFunc(handler.DeleteFromBlacklist), "internal_delete_from_blacklist")) } +// registerInternalStatesRoute registers the plugin states endpoint directly on +// the ServeMux, like the blacklist endpoints. +func registerInternalStatesRoute(r *http.ServeMux, handler handlers.Handlers) { + r.Handle("GET /internal/states", otelhttp.NewHandler(http.HandlerFunc(handler.ListStates), "internal_list_states")) +} + func buildExternalRouter(handler handlers.Handlers) http.Handler { r := httprouter.New() diff --git a/pkg/db/db.go b/pkg/db/db.go index 54d3bf8..8db1e21 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -25,6 +25,32 @@ type Plugin struct { Disabled bool `json:"disabled,omitempty" bson:"disabled"` Hidden bool `json:"hidden,omitempty" bson:"hidden"` UseUnsafe bool `json:"useUnsafe,omitempty" bson:"useUnsafe"` + // TagsETag is the ETag of the last GitHub tags response seen by the scraper. + TagsETag string `json:"tagsETag,omitempty" bson:"tagsETag"` +} + +// PluginState is the part of a plugin the scraper needs to know if a plugin changed. +type PluginState struct { + ID string `json:"id" bson:"id"` + Name string `json:"name" bson:"name"` + LatestVersion string `json:"latestVersion" bson:"latestVersion"` + Stars int `json:"stars" bson:"stars"` + Hidden bool `json:"hidden" bson:"hidden"` + Disabled bool `json:"disabled" bson:"disabled"` + TagsETag string `json:"tagsETag" bson:"tagsETag"` +} + +// PluginPatch holds the plugin fields that can be updated alone. +// A nil field is not updated. +type PluginPatch struct { + Stars *int `json:"stars,omitempty"` + Hidden *bool `json:"hidden,omitempty"` + TagsETag *string `json:"tagsETag,omitempty"` +} + +// IsEmpty returns true when the patch updates nothing. +func (p PluginPatch) IsEmpty() bool { + return p.Stars == nil && p.Hidden == nil && p.TagsETag == nil } // BlacklistEntry represents a repository excluded from the plugin scraping. diff --git a/pkg/db/mongodb/bootstrap.go b/pkg/db/mongodb/bootstrap.go index 860abb8..4e5d8d6 100644 --- a/pkg/db/mongodb/bootstrap.go +++ b/pkg/db/mongodb/bootstrap.go @@ -30,7 +30,7 @@ func (m *MongoDB) Bootstrap() error { Options: &options.IndexOptions{ Name: new("_by_stars"), }, - Keys: bson.D{{Key: "stars", Value: 1}}, + Keys: bson.D{{Key: fieldStars, Value: 1}}, }, { Options: &options.IndexOptions{ diff --git a/pkg/db/mongodb/plugindb.go b/pkg/db/mongodb/plugindb.go index 5441002..1c35452 100644 --- a/pkg/db/mongodb/plugindb.go +++ b/pkg/db/mongodb/plugindb.go @@ -24,6 +24,8 @@ const ( fieldHashes = "hashes" fieldDisabled = "disabled" fieldHidden = "hidden" + fieldTagsETag = "tagsETag" + fieldStars = "stars" ) // MongoDB is a mongoDB client. @@ -160,7 +162,7 @@ func (m *MongoDB) List(ctx context.Context, page db.Pagination) ([]db.Plugin, st opts := &options.FindOptions{} opts.SetLimit(int64(page.Size + 1)) opts.SetProjection(bson.D{{Key: fieldHashes, Value: 0}}) - opts.SetSort(bson.D{{Key: "stars", Value: -1}}) + opts.SetSort(bson.D{{Key: fieldStars, Value: -1}}) cursor, err := m.client.Collection(m.collName).Find(ctx, criteria, opts) if err != nil { @@ -326,6 +328,86 @@ func (m *MongoDB) Update(ctx context.Context, id string, plugin db.Plugin) (db.P return updated, nil } +// ListStates returns the state of all plugins, hidden and disabled plugins included. +func (m *MongoDB) ListStates(ctx context.Context) ([]db.PluginState, error) { + ctx, span := m.tracer.Start(ctx, "db_list_states") + defer span.End() + + opts := &options.FindOptions{} + opts.SetProjection(bson.D{ + {Key: "_id", Value: 0}, + {Key: "id", Value: 1}, + {Key: "name", Value: 1}, + {Key: "latestVersion", Value: 1}, + {Key: fieldStars, Value: 1}, + {Key: fieldHidden, Value: 1}, + {Key: fieldDisabled, Value: 1}, + {Key: fieldTagsETag, Value: 1}, + }) + + cursor, err := m.client.Collection(m.collName).Find(ctx, bson.D{}, opts) + if err != nil { + span.RecordError(err) + + return nil, fmt.Errorf("unable to find plugin states: %w", err) + } + + states := make([]db.PluginState, 0) + + if err = cursor.All(ctx, &states); err != nil { + span.RecordError(err) + + return nil, fmt.Errorf("unable to unmarshal plugin states: %w", err) + } + + return states, nil +} + +// Patch updates only the non-nil fields of the patch. +func (m *MongoDB) Patch(ctx context.Context, id string, patch db.PluginPatch) (db.Plugin, error) { + ctx, span := m.tracer.Start(ctx, "db_patch") + defer span.End() + + set := bson.D{} + + if patch.Stars != nil { + set = append(set, bson.E{Key: fieldStars, Value: *patch.Stars}) + } + + if patch.Hidden != nil { + set = append(set, bson.E{Key: fieldHidden, Value: *patch.Hidden}) + } + + if patch.TagsETag != nil { + set = append(set, bson.E{Key: fieldTagsETag, Value: *patch.TagsETag}) + } + + if len(set) == 0 { + return db.Plugin{}, errors.New("nothing to update") + } + + filter := bson.D{{Key: "id", Value: id}} + update := bson.D{{Key: "$set", Value: set}} + + opts := &options.FindOneAndUpdateOptions{} + opts.SetReturnDocument(options.After) + opts.SetProjection(bson.D{{Key: fieldHashes, Value: 0}}) + + var updated db.Plugin + + if err := m.client.Collection(m.collName).FindOneAndUpdate(ctx, filter, update, opts).Decode(&updated); err != nil { + span.RecordError(err) + + if errors.Is(err, mongo.ErrNoDocuments) { + return db.Plugin{}, db.NotFoundError{Err: err} + } + + return db.Plugin{}, fmt.Errorf("unable to patch plugin: %w", err) + } + + return updated, nil +} + // CreateHash creates a new plugin hash. func (m *MongoDB) CreateHash(ctx context.Context, module, version, hash string) (db.PluginHash, error) { ctx, span := m.tracer.Start(ctx, "db_create_hash") diff --git a/pkg/db/mongodb/plugindb_test.go b/pkg/db/mongodb/plugindb_test.go index 03077fb..429495d 100644 --- a/pkg/db/mongodb/plugindb_test.go +++ b/pkg/db/mongodb/plugindb_test.go @@ -43,6 +43,7 @@ func TestMongoDB_Create(t *testing.T) { "something": "there", }, CreatedAt: time.Now().Add(-2 * time.Hour), + TagsETag: `W/"abc"`, } got, err := store.Create(ctx, plugin) @@ -650,8 +651,9 @@ func TestMongoDB_Update(t *testing.T) { Versions: []string{ "v1.1.1", }, - Stars: 10, - Snippet: nil, + Stars: 10, + Snippet: nil, + TagsETag: `W/"old"`, }, Hashes: []db.PluginHash{ {Name: "plugin@v1.1.1", Hash: "123"}, @@ -676,14 +678,16 @@ func TestMongoDB_Update(t *testing.T) { Versions: []string{ "v1.1.1", }, - Stars: 10, - Snippet: nil, + Stars: 10, + Snippet: nil, + TagsETag: `W/"new"`, }) require.NoError(t, err) want := fixtures["plugin"].Plugin want.Name = "New Name" want.Author = "New Author" + want.TagsETag = `W/"new"` assert.Equal(t, want, got) @@ -987,3 +991,123 @@ func buildNextID(t *testing.T, next db.Plugin) string { return base64.RawStdEncoding.EncodeToString(b) } + +func TestMongoDB_ListStates(t *testing.T) { + ctx := context.Background() + + store, _ := createDatabase(t, []fixture{ + { + key: "active", + plugin: pluginDocument{ + Plugin: db.Plugin{ + ID: "123", + Name: "active", + Readme: "readme", + LatestVersion: "v1.0.0", + Stars: 10, + Snippet: map[string]any{"something": "there"}, + TagsETag: `W/"abc"`, + }, + Hashes: []db.PluginHash{{Name: "active@v1.0.0", Hash: "123"}}, + }, + }, + { + key: "hidden", + plugin: pluginDocument{ + Plugin: db.Plugin{ID: "234", Name: "hidden", LatestVersion: "v2.0.0", Stars: 3, Hidden: true}, + }, + }, + { + key: "disabled", + plugin: pluginDocument{ + Plugin: db.Plugin{ID: "345", Name: "disabled", LatestVersion: "v3.0.0", Disabled: true}, + }, + }, + }) + + states, err := store.ListStates(ctx) + require.NoError(t, err) + + assert.ElementsMatch(t, []db.PluginState{ + {ID: "123", Name: "active", LatestVersion: "v1.0.0", Stars: 10, TagsETag: `W/"abc"`}, + {ID: "234", Name: "hidden", LatestVersion: "v2.0.0", Stars: 3, Hidden: true}, + {ID: "345", Name: "disabled", LatestVersion: "v3.0.0", Disabled: true}, + }, states) +} + +func TestMongoDB_ListStates_empty(t *testing.T) { + store, _ := createDatabase(t, nil) + + states, err := store.ListStates(context.Background()) + require.NoError(t, err) + + assert.NotNil(t, states) + assert.Empty(t, states) +} + +func TestMongoDB_Patch(t *testing.T) { + ctx := context.Background() + + store, fixtures := createDatabase(t, []fixture{ + { + key: "plugin", + plugin: pluginDocument{ + Plugin: db.Plugin{ + ID: "123", + Name: "plugin", + DisplayName: "plugin", + Author: "author", + Readme: "readme", + LatestVersion: "v1.1.1", + Versions: []string{"v1.1.1"}, + Stars: 10, + Snippet: map[string]any{"something": "there"}, + Disabled: true, + TagsETag: `W/"old"`, + }, + Hashes: []db.PluginHash{ + {Name: "plugin@v1.1.1", Hash: "123"}, + }, + }, + }, + }) + + want := fixtures["plugin"].Plugin + + // Only stars. + got, err := store.Patch(ctx, "123", db.PluginPatch{Stars: new(42)}) + require.NoError(t, err) + + want.Stars = 42 + assert.Equal(t, want, toUTCPlugin(got)) + + // Only hidden. + got, err = store.Patch(ctx, "123", db.PluginPatch{Hidden: new(true)}) + require.NoError(t, err) + + want.Hidden = true + assert.Equal(t, want, toUTCPlugin(got)) + + // Only ETag, zero values are written. + got, err = store.Patch(ctx, "123", db.PluginPatch{Stars: new(0), TagsETag: new(`W/"new"`)}) + require.NoError(t, err) + + want.Stars = 0 + want.TagsETag = `W/"new"` + assert.Equal(t, want, toUTCPlugin(got)) + + // Check the stored document, hashes included, is not changed in other fields. + stored, ok := getPlugin(t, store, "123") + require.True(t, ok) + + assert.Equal(t, want, toUTCPlugin(stored.Plugin)) + assert.Equal(t, fixtures["plugin"].Hashes, stored.Hashes) + + // Empty patch. + _, err = store.Patch(ctx, "123", db.PluginPatch{}) + require.Error(t, err) + + // Unknown plugin. + _, err = store.Patch(ctx, "456", db.PluginPatch{Stars: new(1)}) + require.ErrorAs(t, err, &db.NotFoundError{}) +} diff --git a/pkg/handlers/dbmock_test.go b/pkg/handlers/dbmock_test.go index 0bc162b..ff0afe8 100644 --- a/pkg/handlers/dbmock_test.go +++ b/pkg/handlers/dbmock_test.go @@ -14,6 +14,8 @@ type mockDB struct { getByNameFn func(context.Context, string, bool) (db.Plugin, error) searchByNameFn func(context.Context, string, db.Pagination) ([]db.Plugin, string, error) updateFn func(context.Context, string, db.Plugin) (db.Plugin, error) + listStatesFn func(ctx context.Context) ([]db.PluginState, error) + patchFn func(ctx context.Context, id string, patch db.PluginPatch) (db.Plugin, error) deleteHashFn func(ctx context.Context, id string) error createHashFn func(ctx context.Context, module, version, hash string) (db.PluginHash, error) @@ -53,6 +55,14 @@ func (m mockDB) Update(ctx context.Context, id string, plugin db.Plugin) (db.Plu return m.updateFn(ctx, id, plugin) } +func (m mockDB) ListStates(ctx context.Context) ([]db.PluginState, error) { + return m.listStatesFn(ctx) +} + +func (m mockDB) Patch(ctx context.Context, id string, patch db.PluginPatch) (db.Plugin, error) { + return m.patchFn(ctx, id, patch) +} + func (m mockDB) DeleteHash(ctx context.Context, id string) error { return m.deleteHashFn(ctx, id) } diff --git a/pkg/handlers/handlers.go b/pkg/handlers/handlers.go index c63a147..7867973 100644 --- a/pkg/handlers/handlers.go +++ b/pkg/handlers/handlers.go @@ -43,6 +43,7 @@ type PluginStorer interface { // Storer is capable of storing plugins and the plugin blacklist. type Storer interface { PluginStorer + PluginStateStorer BlacklistStorer } diff --git a/pkg/handlers/state.go b/pkg/handlers/state.go new file mode 100644 index 0000000..05d2acb --- /dev/null +++ b/pkg/handlers/state.go @@ -0,0 +1,116 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + + "github.com/rs/zerolog/log" + "github.com/traefik/plugin-service/pkg/db" +) + +// PluginStateStorer is capable of reading and updating the plugin state used by the scraper. +type PluginStateStorer interface { + ListStates(ctx context.Context) ([]db.PluginState, error) + Patch(ctx context.Context, id string, patch db.PluginPatch) (db.Plugin, error) +} + +// ListStates returns the state of all plugins, hidden and disabled plugins included, without pagination. +func (h Handlers) ListStates(rw http.ResponseWriter, req *http.Request) { + ctx, span := h.tracer.Start(req.Context(), "handler_list_states") + defer span.End() + + rw.Header().Set("Content-Type", "application/json") + + states, err := h.store.ListStates(ctx) + if err != nil { + span.RecordError(err) + log.Error().Err(err).Msg("Error fetching plugin states") + JSONInternalServerError(rw) + + return + } + + if err := json.NewEncoder(rw).Encode(states); err != nil { + span.RecordError(err) + log.Error().Err(err).Msg("Failed to encode plugin states") + JSONInternalServerError(rw) + + return + } +} + +// Patch updates only the given fields of a plugin (stars, hidden, tagsETag). +func (h Handlers) Patch(rw http.ResponseWriter, req *http.Request) { + ctx, span := h.tracer.Start(req.Context(), "handler_patch") + defer span.End() + + rw.Header().Set("Content-Type", "application/json") + + id, err := getPathParam(req.URL) + if err != nil { + span.RecordError(err) + JSONError(rw, http.StatusBadRequest, "Missing plugin id") + + return + } + + logger := log.With().Str("plugin_id", id).Logger() + + body, err := io.ReadAll(req.Body) + if err != nil { + span.RecordError(err) + logger.Error().Err(err).Msg("Unable to read body for patch") + JSONError(rw, http.StatusBadRequest, err.Error()) + + return + } + + var patch db.PluginPatch + + if len(body) > 0 { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.DisallowUnknownFields() + + if err = dec.Decode(&patch); err != nil { + span.RecordError(err) + logger.Error().Err(err).Msg("Unable to decode plugin patch") + JSONError(rw, http.StatusBadRequest, err.Error()) + + return + } + } + + if patch.IsEmpty() { + JSONError(rw, http.StatusBadRequest, "nothing to update") + + return + } + + updated, err := h.store.Patch(ctx, id, patch) + if err != nil { + span.RecordError(err) + + if errors.As(err, &db.NotFoundError{}) { + NotFound(rw, req) + + return + } + + logger.Error().Err(err).Msg("Error patching plugin") + JSONInternalServerError(rw) + + return + } + + if err := json.NewEncoder(rw).Encode(updated); err != nil { + span.RecordError(err) + logger.Error().Err(err).Msg("Failed to encode patched plugin") + JSONInternalServerError(rw) + + return + } +} diff --git a/pkg/handlers/state_test.go b/pkg/handlers/state_test.go new file mode 100644 index 0000000..9b10193 --- /dev/null +++ b/pkg/handlers/state_test.go @@ -0,0 +1,155 @@ +package handlers + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/traefik/plugin-service/pkg/db" +) + +func TestHandlers_ListStates(t *testing.T) { + testDB := mockDB{ + listStatesFn: func(_ context.Context) ([]db.PluginState, error) { + return []db.PluginState{ + {ID: "123", Name: "github.com/traefik/plugindemo", LatestVersion: "v0.2.1", Stars: 22, TagsETag: `W/"abc"`}, + {ID: "456", Name: "github.com/traefik/hidden", LatestVersion: "v1.0.0", Hidden: true, Disabled: true}, + }, nil + }, + } + + rw := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/internal/states", http.NoBody) + + New(testDB, nil, nil, "").ListStates(rw, req) + + require.Equal(t, http.StatusOK, rw.Code) + assert.JSONEq(t, `[ + {"id":"123","name":"github.com/traefik/plugindemo","latestVersion":"v0.2.1","stars":22,"hidden":false,"disabled":false,"tagsETag":"W/\"abc\""}, + {"id":"456","name":"github.com/traefik/hidden","latestVersion":"v1.0.0","stars":0,"hidden":true,"disabled":true,"tagsETag":""} + ]`, rw.Body.String()) +} + +func TestHandlers_ListStates_empty(t *testing.T) { + testDB := mockDB{ + listStatesFn: func(_ context.Context) ([]db.PluginState, error) { + return []db.PluginState{}, nil + }, + } + + rw := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/internal/states", http.NoBody) + + New(testDB, nil, nil, "").ListStates(rw, req) + + require.Equal(t, http.StatusOK, rw.Code) + assert.JSONEq(t, `[]`, rw.Body.String()) +} + +func TestHandlers_Patch(t *testing.T) { + testCases := []struct { + desc string + body string + wantPatch db.PluginPatch + }{ + { + desc: "stars only", + body: `{"stars":12}`, + wantPatch: db.PluginPatch{Stars: new(12)}, + }, + { + desc: "hidden only", + body: `{"hidden":false}`, + wantPatch: db.PluginPatch{Hidden: new(false)}, + }, + { + desc: "empty ETag", + body: `{"tagsETag":""}`, + wantPatch: db.PluginPatch{TagsETag: new("")}, + }, + { + desc: "all fields", + body: `{"stars":0,"hidden":true,"tagsETag":"W/\"abc\""}`, + wantPatch: db.PluginPatch{Stars: new(0), Hidden: new(true), TagsETag: new(`W/"abc"`)}, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + var ( + gotID string + gotPatch db.PluginPatch + ) + + testDB := mockDB{ + patchFn: func(_ context.Context, id string, patch db.PluginPatch) (db.Plugin, error) { + gotID = id + gotPatch = patch + + return db.Plugin{ID: id, Name: "github.com/traefik/plugindemo"}, nil + }, + } + + rw := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPatch, "/123", strings.NewReader(test.body)) + + New(testDB, nil, nil, "").Patch(rw, req) + + require.Equal(t, http.StatusOK, rw.Code) + assert.Equal(t, "123", gotID) + assert.Equal(t, test.wantPatch, gotPatch) + }) + } +} + +func TestHandlers_Patch_badRequest(t *testing.T) { + testDB := mockDB{ + patchFn: func(_ context.Context, _ string, _ db.PluginPatch) (db.Plugin, error) { + t.Fatal("store should not be called on invalid input") + return db.Plugin{}, nil + }, + } + + testCases := []struct { + desc string + path string + body string + }{ + {desc: "empty body", path: "/123", body: ""}, + {desc: "empty object", path: "/123", body: `{}`}, + {desc: "null values", path: "/123", body: `{"stars":null}`}, + {desc: "unknown field", path: "/123", body: `{"name":"other"}`}, + {desc: "invalid JSON", path: "/123", body: `{`}, + {desc: "missing id", path: "/", body: `{"stars":1}`}, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + rw := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPatch, test.path, strings.NewReader(test.body)) + + New(testDB, nil, nil, "").Patch(rw, req) + + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) + } +} + +func TestHandlers_Patch_notFound(t *testing.T) { + testDB := mockDB{ + patchFn: func(_ context.Context, _ string, _ db.PluginPatch) (db.Plugin, error) { + return db.Plugin{}, db.NotFoundError{} + }, + } + + rw := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPatch, "/123", strings.NewReader(`{"stars":1}`)) + + New(testDB, nil, nil, "").Patch(rw, req) + + assert.Equal(t, http.StatusNotFound, rw.Code) +} From c0499f2864bc9cec5621249d15e26fcbd990e08b Mon Sep 17 00:00:00 2001 From: mmatur Date: Fri, 20 Nov 2026 15:54:15 +0100 Subject: [PATCH 2/3] fix: keep the stored tags ETag on an update without it --- pkg/db/db.go | 3 ++- pkg/db/mongodb/plugindb_test.go | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/db/db.go b/pkg/db/db.go index 8db1e21..b0bd864 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -26,7 +26,8 @@ type Plugin struct { Hidden bool `json:"hidden,omitempty" bson:"hidden"` UseUnsafe bool `json:"useUnsafe,omitempty" bson:"useUnsafe"` // TagsETag is the ETag of the last GitHub tags response seen by the scraper. - TagsETag string `json:"tagsETag,omitempty" bson:"tagsETag"` + // omitempty: an update without the ETag (an older scraper) does not erase it. + TagsETag string `json:"tagsETag,omitempty" bson:"tagsETag,omitempty"` } // PluginState is the part of a plugin the scraper needs to know if a plugin changed. diff --git a/pkg/db/mongodb/plugindb_test.go b/pkg/db/mongodb/plugindb_test.go index 429495d..62498ca 100644 --- a/pkg/db/mongodb/plugindb_test.go +++ b/pkg/db/mongodb/plugindb_test.go @@ -707,6 +707,15 @@ func TestMongoDB_Update(t *testing.T) { assert.Equal(t, want, got) + // An update without the ETag (an older scraper) keeps the stored one. + withoutETag := got + withoutETag.TagsETag = "" + + got, err = store.Update(ctx, "123", withoutETag) + require.NoError(t, err) + + assert.Equal(t, want, got) + // Check that we get a db.NotFound when no plugin have the given id. _, err = store.Update(ctx, "456", got) require.ErrorAs(t, err, &db.NotFoundError{}) From 479bb7a15cd491d6d17b075bb3011462b16c6275 Mon Sep 17 00:00:00 2001 From: mmatur Date: Fri, 20 Nov 2026 16:00:44 +0100 Subject: [PATCH 3/3] fix: route the states endpoint in the internal router and harden the patch decoding --- cmd/serve/serve.go | 9 ++------- pkg/handlers/state.go | 36 ++++++++++++++++++++---------------- pkg/handlers/state_test.go | 4 ++++ 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index 50758b3..8020bbb 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -57,7 +57,6 @@ func run(ctx context.Context, cfg Config) error { r.Handle("/external/", buildExternalRouter(handler)) registerInternalBlacklistRoutes(r, handler) - registerInternalStatesRoute(r, handler) r.HandleFunc("/live", healthChecker.Live) r.HandleFunc("/ready", healthChecker.Ready) @@ -84,6 +83,8 @@ func buildInternalRouter(handler handlers.Handlers) http.Handler { r.Handler(http.MethodPost, "/", otelhttp.NewHandler(http.HandlerFunc(handler.Create), "internal_create")) r.Handler(http.MethodPut, "/:uuid", otelhttp.NewHandler(http.HandlerFunc(handler.Update), "internal_update")) r.Handler(http.MethodPatch, "/:uuid", otelhttp.NewHandler(http.HandlerFunc(handler.Patch), "internal_patch")) + // No conflict with "/:uuid": httprouter has one tree per method, and no GET route has a wildcard. + r.Handler(http.MethodGet, "/states", otelhttp.NewHandler(http.HandlerFunc(handler.ListStates), "internal_list_states")) r.Handler(http.MethodDelete, "/:uuid", otelhttp.NewHandler(http.HandlerFunc(handler.Delete), "internal_delete")) r.NotFound = http.HandlerFunc(handlers.NotFound) @@ -101,12 +102,6 @@ func registerInternalBlacklistRoutes(r *http.ServeMux, handler handlers.Handlers r.Handle("DELETE /internal/blacklist/{owner}/{repo}", otelhttp.NewHandler(http.HandlerFunc(handler.DeleteFromBlacklist), "internal_delete_from_blacklist")) } -// registerInternalStatesRoute registers the plugin states endpoint directly on -// the ServeMux, like the blacklist endpoints. -func registerInternalStatesRoute(r *http.ServeMux, handler handlers.Handlers) { - r.Handle("GET /internal/states", otelhttp.NewHandler(http.HandlerFunc(handler.ListStates), "internal_list_states")) -} - func buildExternalRouter(handler handlers.Handlers) http.Handler { r := httprouter.New() diff --git a/pkg/handlers/state.go b/pkg/handlers/state.go index 05d2acb..3f6bba4 100644 --- a/pkg/handlers/state.go +++ b/pkg/handlers/state.go @@ -1,7 +1,6 @@ package handlers import ( - "bytes" "context" "encoding/json" "errors" @@ -12,6 +11,9 @@ import ( "github.com/traefik/plugin-service/pkg/db" ) +// maxPatchSize is the maximum size of a patch body. +const maxPatchSize = 1 << 20 + // PluginStateStorer is capable of reading and updating the plugin state used by the scraper. type PluginStateStorer interface { ListStates(ctx context.Context) ([]db.PluginState, error) @@ -60,28 +62,24 @@ func (h Handlers) Patch(rw http.ResponseWriter, req *http.Request) { logger := log.With().Str("plugin_id", id).Logger() - body, err := io.ReadAll(req.Body) - if err != nil { + var patch db.PluginPatch + + dec := json.NewDecoder(http.MaxBytesReader(rw, req.Body, maxPatchSize)) + dec.DisallowUnknownFields() + + // An empty body is an empty patch. + if err = dec.Decode(&patch); err != nil && !errors.Is(err, io.EOF) { span.RecordError(err) - logger.Error().Err(err).Msg("Unable to read body for patch") + logger.Error().Err(err).Msg("Unable to decode plugin patch") JSONError(rw, http.StatusBadRequest, err.Error()) return } - var patch db.PluginPatch - - if len(body) > 0 { - dec := json.NewDecoder(bytes.NewReader(body)) - dec.DisallowUnknownFields() - - if err = dec.Decode(&patch); err != nil { - span.RecordError(err) - logger.Error().Err(err).Msg("Unable to decode plugin patch") - JSONError(rw, http.StatusBadRequest, err.Error()) + if dec.More() { + JSONError(rw, http.StatusBadRequest, "body must contain a single JSON object") - return - } + return } if patch.IsEmpty() { @@ -90,6 +88,12 @@ func (h Handlers) Patch(rw http.ResponseWriter, req *http.Request) { return } + if patch.Stars != nil && *patch.Stars < 0 { + JSONError(rw, http.StatusBadRequest, "stars must not be negative") + + return + } + updated, err := h.store.Patch(ctx, id, patch) if err != nil { span.RecordError(err) diff --git a/pkg/handlers/state_test.go b/pkg/handlers/state_test.go index 9b10193..ef4244d 100644 --- a/pkg/handlers/state_test.go +++ b/pkg/handlers/state_test.go @@ -125,6 +125,10 @@ func TestHandlers_Patch_badRequest(t *testing.T) { {desc: "unknown field", path: "/123", body: `{"name":"other"}`}, {desc: "invalid JSON", path: "/123", body: `{`}, {desc: "missing id", path: "/", body: `{"stars":1}`}, + {desc: "two JSON objects", path: "/123", body: `{"stars":1}{"hidden":true}`}, + {desc: "trailing data", path: "/123", body: `{"stars":1} garbage`}, + {desc: "negative stars", path: "/123", body: `{"stars":-5}`}, + {desc: "too large", path: "/123", body: `{"tagsETag":"` + strings.Repeat("a", maxPatchSize) + `"}`}, } for _, test := range testCases {