diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index cd29bf9..8020bbb 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -82,6 +82,9 @@ func buildInternalRouter(handler handlers.Handlers) http.Handler { r.Handler(http.MethodGet, "/", otelhttp.NewHandler(http.HandlerFunc(handler.List), "internal_list")) r.Handler(http.MethodPost, "/", otelhttp.NewHandler(http.HandlerFunc(handler.Create), "internal_create")) r.Handler(http.MethodPut, "/:uuid", otelhttp.NewHandler(http.HandlerFunc(handler.Update), "internal_update")) + r.Handler(http.MethodPatch, "/:uuid", otelhttp.NewHandler(http.HandlerFunc(handler.Patch), "internal_patch")) + // No conflict with "/:uuid": httprouter has one tree per method, and no GET route has a wildcard. + r.Handler(http.MethodGet, "/states", otelhttp.NewHandler(http.HandlerFunc(handler.ListStates), "internal_list_states")) r.Handler(http.MethodDelete, "/:uuid", otelhttp.NewHandler(http.HandlerFunc(handler.Delete), "internal_delete")) r.NotFound = http.HandlerFunc(handlers.NotFound) diff --git a/pkg/db/db.go b/pkg/db/db.go index 54d3bf8..b0bd864 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -25,6 +25,33 @@ type Plugin struct { Disabled bool `json:"disabled,omitempty" bson:"disabled"` Hidden bool `json:"hidden,omitempty" bson:"hidden"` UseUnsafe bool `json:"useUnsafe,omitempty" bson:"useUnsafe"` + // TagsETag is the ETag of the last GitHub tags response seen by the scraper. + // omitempty: an update without the ETag (an older scraper) does not erase it. + TagsETag string `json:"tagsETag,omitempty" bson:"tagsETag,omitempty"` +} + +// PluginState is the part of a plugin the scraper needs to know if a plugin changed. +type PluginState struct { + ID string `json:"id" bson:"id"` + Name string `json:"name" bson:"name"` + LatestVersion string `json:"latestVersion" bson:"latestVersion"` + Stars int `json:"stars" bson:"stars"` + Hidden bool `json:"hidden" bson:"hidden"` + Disabled bool `json:"disabled" bson:"disabled"` + TagsETag string `json:"tagsETag" bson:"tagsETag"` +} + +// PluginPatch holds the plugin fields that can be updated alone. +// A nil field is not updated. +type PluginPatch struct { + Stars *int `json:"stars,omitempty"` + Hidden *bool `json:"hidden,omitempty"` + TagsETag *string `json:"tagsETag,omitempty"` +} + +// IsEmpty returns true when the patch updates nothing. +func (p PluginPatch) IsEmpty() bool { + return p.Stars == nil && p.Hidden == nil && p.TagsETag == nil } // BlacklistEntry represents a repository excluded from the plugin scraping. diff --git a/pkg/db/mongodb/bootstrap.go b/pkg/db/mongodb/bootstrap.go index 860abb8..4e5d8d6 100644 --- a/pkg/db/mongodb/bootstrap.go +++ b/pkg/db/mongodb/bootstrap.go @@ -30,7 +30,7 @@ func (m *MongoDB) Bootstrap() error { Options: &options.IndexOptions{ Name: new("_by_stars"), }, - Keys: bson.D{{Key: "stars", Value: 1}}, + Keys: bson.D{{Key: fieldStars, Value: 1}}, }, { Options: &options.IndexOptions{ diff --git a/pkg/db/mongodb/plugindb.go b/pkg/db/mongodb/plugindb.go index 5441002..1c35452 100644 --- a/pkg/db/mongodb/plugindb.go +++ b/pkg/db/mongodb/plugindb.go @@ -24,6 +24,8 @@ const ( fieldHashes = "hashes" fieldDisabled = "disabled" fieldHidden = "hidden" + fieldTagsETag = "tagsETag" + fieldStars = "stars" ) // MongoDB is a mongoDB client. @@ -160,7 +162,7 @@ func (m *MongoDB) List(ctx context.Context, page db.Pagination) ([]db.Plugin, st opts := &options.FindOptions{} opts.SetLimit(int64(page.Size + 1)) opts.SetProjection(bson.D{{Key: fieldHashes, Value: 0}}) - opts.SetSort(bson.D{{Key: "stars", Value: -1}}) + opts.SetSort(bson.D{{Key: fieldStars, Value: -1}}) cursor, err := m.client.Collection(m.collName).Find(ctx, criteria, opts) if err != nil { @@ -326,6 +328,86 @@ func (m *MongoDB) Update(ctx context.Context, id string, plugin db.Plugin) (db.P return updated, nil } +// ListStates returns the state of all plugins, hidden and disabled plugins included. +func (m *MongoDB) ListStates(ctx context.Context) ([]db.PluginState, error) { + ctx, span := m.tracer.Start(ctx, "db_list_states") + defer span.End() + + opts := &options.FindOptions{} + opts.SetProjection(bson.D{ + {Key: "_id", Value: 0}, + {Key: "id", Value: 1}, + {Key: "name", Value: 1}, + {Key: "latestVersion", Value: 1}, + {Key: fieldStars, Value: 1}, + {Key: fieldHidden, Value: 1}, + {Key: fieldDisabled, Value: 1}, + {Key: fieldTagsETag, Value: 1}, + }) + + cursor, err := m.client.Collection(m.collName).Find(ctx, bson.D{}, opts) + if err != nil { + span.RecordError(err) + + return nil, fmt.Errorf("unable to find plugin states: %w", err) + } + + states := make([]db.PluginState, 0) + + if err = cursor.All(ctx, &states); err != nil { + span.RecordError(err) + + return nil, fmt.Errorf("unable to unmarshal plugin states: %w", err) + } + + return states, nil +} + +// Patch updates only the non-nil fields of the patch. +func (m *MongoDB) Patch(ctx context.Context, id string, patch db.PluginPatch) (db.Plugin, error) { + ctx, span := m.tracer.Start(ctx, "db_patch") + defer span.End() + + set := bson.D{} + + if patch.Stars != nil { + set = append(set, bson.E{Key: fieldStars, Value: *patch.Stars}) + } + + if patch.Hidden != nil { + set = append(set, bson.E{Key: fieldHidden, Value: *patch.Hidden}) + } + + if patch.TagsETag != nil { + set = append(set, bson.E{Key: fieldTagsETag, Value: *patch.TagsETag}) + } + + if len(set) == 0 { + return db.Plugin{}, errors.New("nothing to update") + } + + filter := bson.D{{Key: "id", Value: id}} + update := bson.D{{Key: "$set", Value: set}} + + opts := &options.FindOneAndUpdateOptions{} + opts.SetReturnDocument(options.After) + opts.SetProjection(bson.D{{Key: fieldHashes, Value: 0}}) + + var updated db.Plugin + + if err := m.client.Collection(m.collName).FindOneAndUpdate(ctx, filter, update, opts).Decode(&updated); err != nil { + span.RecordError(err) + + if errors.Is(err, mongo.ErrNoDocuments) { + return db.Plugin{}, db.NotFoundError{Err: err} + } + + return db.Plugin{}, fmt.Errorf("unable to patch plugin: %w", err) + } + + return updated, nil +} + // CreateHash creates a new plugin hash. func (m *MongoDB) CreateHash(ctx context.Context, module, version, hash string) (db.PluginHash, error) { ctx, span := m.tracer.Start(ctx, "db_create_hash") diff --git a/pkg/db/mongodb/plugindb_test.go b/pkg/db/mongodb/plugindb_test.go index 03077fb..62498ca 100644 --- a/pkg/db/mongodb/plugindb_test.go +++ b/pkg/db/mongodb/plugindb_test.go @@ -43,6 +43,7 @@ func TestMongoDB_Create(t *testing.T) { "something": "there", }, CreatedAt: time.Now().Add(-2 * time.Hour), + TagsETag: `W/"abc"`, } got, err := store.Create(ctx, plugin) @@ -650,8 +651,9 @@ func TestMongoDB_Update(t *testing.T) { Versions: []string{ "v1.1.1", }, - Stars: 10, - Snippet: nil, + Stars: 10, + Snippet: nil, + TagsETag: `W/"old"`, }, Hashes: []db.PluginHash{ {Name: "plugin@v1.1.1", Hash: "123"}, @@ -676,14 +678,16 @@ func TestMongoDB_Update(t *testing.T) { Versions: []string{ "v1.1.1", }, - Stars: 10, - Snippet: nil, + Stars: 10, + Snippet: nil, + TagsETag: `W/"new"`, }) require.NoError(t, err) want := fixtures["plugin"].Plugin want.Name = "New Name" want.Author = "New Author" + want.TagsETag = `W/"new"` assert.Equal(t, want, got) @@ -703,6 +707,15 @@ func TestMongoDB_Update(t *testing.T) { assert.Equal(t, want, got) + // An update without the ETag (an older scraper) keeps the stored one. + withoutETag := got + withoutETag.TagsETag = "" + + got, err = store.Update(ctx, "123", withoutETag) + require.NoError(t, err) + + assert.Equal(t, want, got) + // Check that we get a db.NotFound when no plugin have the given id. _, err = store.Update(ctx, "456", got) require.ErrorAs(t, err, &db.NotFoundError{}) @@ -987,3 +1000,123 @@ func buildNextID(t *testing.T, next db.Plugin) string { return base64.RawStdEncoding.EncodeToString(b) } + +func TestMongoDB_ListStates(t *testing.T) { + ctx := context.Background() + + store, _ := createDatabase(t, []fixture{ + { + key: "active", + plugin: pluginDocument{ + Plugin: db.Plugin{ + ID: "123", + Name: "active", + Readme: "readme", + LatestVersion: "v1.0.0", + Stars: 10, + Snippet: map[string]any{"something": "there"}, + TagsETag: `W/"abc"`, + }, + Hashes: []db.PluginHash{{Name: "active@v1.0.0", Hash: "123"}}, + }, + }, + { + key: "hidden", + plugin: pluginDocument{ + Plugin: db.Plugin{ID: "234", Name: "hidden", LatestVersion: "v2.0.0", Stars: 3, Hidden: true}, + }, + }, + { + key: "disabled", + plugin: pluginDocument{ + Plugin: db.Plugin{ID: "345", Name: "disabled", LatestVersion: "v3.0.0", Disabled: true}, + }, + }, + }) + + states, err := store.ListStates(ctx) + require.NoError(t, err) + + assert.ElementsMatch(t, []db.PluginState{ + {ID: "123", Name: "active", LatestVersion: "v1.0.0", Stars: 10, TagsETag: `W/"abc"`}, + {ID: "234", Name: "hidden", LatestVersion: "v2.0.0", Stars: 3, Hidden: true}, + {ID: "345", Name: "disabled", LatestVersion: "v3.0.0", Disabled: true}, + }, states) +} + +func TestMongoDB_ListStates_empty(t *testing.T) { + store, _ := createDatabase(t, nil) + + states, err := store.ListStates(context.Background()) + require.NoError(t, err) + + assert.NotNil(t, states) + assert.Empty(t, states) +} + +func TestMongoDB_Patch(t *testing.T) { + ctx := context.Background() + + store, fixtures := createDatabase(t, []fixture{ + { + key: "plugin", + plugin: pluginDocument{ + Plugin: db.Plugin{ + ID: "123", + Name: "plugin", + DisplayName: "plugin", + Author: "author", + Readme: "readme", + LatestVersion: "v1.1.1", + Versions: []string{"v1.1.1"}, + Stars: 10, + Snippet: map[string]any{"something": "there"}, + Disabled: true, + TagsETag: `W/"old"`, + }, + Hashes: []db.PluginHash{ + {Name: "plugin@v1.1.1", Hash: "123"}, + }, + }, + }, + }) + + want := fixtures["plugin"].Plugin + + // Only stars. + got, err := store.Patch(ctx, "123", db.PluginPatch{Stars: new(42)}) + require.NoError(t, err) + + want.Stars = 42 + assert.Equal(t, want, toUTCPlugin(got)) + + // Only hidden. + got, err = store.Patch(ctx, "123", db.PluginPatch{Hidden: new(true)}) + require.NoError(t, err) + + want.Hidden = true + assert.Equal(t, want, toUTCPlugin(got)) + + // Only ETag, zero values are written. + got, err = store.Patch(ctx, "123", db.PluginPatch{Stars: new(0), TagsETag: new(`W/"new"`)}) + require.NoError(t, err) + + want.Stars = 0 + want.TagsETag = `W/"new"` + assert.Equal(t, want, toUTCPlugin(got)) + + // Check the stored document, hashes included, is not changed in other fields. + stored, ok := getPlugin(t, store, "123") + require.True(t, ok) + + assert.Equal(t, want, toUTCPlugin(stored.Plugin)) + assert.Equal(t, fixtures["plugin"].Hashes, stored.Hashes) + + // Empty patch. + _, err = store.Patch(ctx, "123", db.PluginPatch{}) + require.Error(t, err) + + // Unknown plugin. + _, err = store.Patch(ctx, "456", db.PluginPatch{Stars: new(1)}) + require.ErrorAs(t, err, &db.NotFoundError{}) +} diff --git a/pkg/handlers/dbmock_test.go b/pkg/handlers/dbmock_test.go index 0bc162b..ff0afe8 100644 --- a/pkg/handlers/dbmock_test.go +++ b/pkg/handlers/dbmock_test.go @@ -14,6 +14,8 @@ type mockDB struct { getByNameFn func(context.Context, string, bool) (db.Plugin, error) searchByNameFn func(context.Context, string, db.Pagination) ([]db.Plugin, string, error) updateFn func(context.Context, string, db.Plugin) (db.Plugin, error) + listStatesFn func(ctx context.Context) ([]db.PluginState, error) + patchFn func(ctx context.Context, id string, patch db.PluginPatch) (db.Plugin, error) deleteHashFn func(ctx context.Context, id string) error createHashFn func(ctx context.Context, module, version, hash string) (db.PluginHash, error) @@ -53,6 +55,14 @@ func (m mockDB) Update(ctx context.Context, id string, plugin db.Plugin) (db.Plu return m.updateFn(ctx, id, plugin) } +func (m mockDB) ListStates(ctx context.Context) ([]db.PluginState, error) { + return m.listStatesFn(ctx) +} + +func (m mockDB) Patch(ctx context.Context, id string, patch db.PluginPatch) (db.Plugin, error) { + return m.patchFn(ctx, id, patch) +} + func (m mockDB) DeleteHash(ctx context.Context, id string) error { return m.deleteHashFn(ctx, id) } diff --git a/pkg/handlers/handlers.go b/pkg/handlers/handlers.go index c63a147..7867973 100644 --- a/pkg/handlers/handlers.go +++ b/pkg/handlers/handlers.go @@ -43,6 +43,7 @@ type PluginStorer interface { // Storer is capable of storing plugins and the plugin blacklist. type Storer interface { PluginStorer + PluginStateStorer BlacklistStorer } diff --git a/pkg/handlers/state.go b/pkg/handlers/state.go new file mode 100644 index 0000000..3f6bba4 --- /dev/null +++ b/pkg/handlers/state.go @@ -0,0 +1,120 @@ +package handlers + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + + "github.com/rs/zerolog/log" + "github.com/traefik/plugin-service/pkg/db" +) + +// maxPatchSize is the maximum size of a patch body. +const maxPatchSize = 1 << 20 + +// PluginStateStorer is capable of reading and updating the plugin state used by the scraper. +type PluginStateStorer interface { + ListStates(ctx context.Context) ([]db.PluginState, error) + Patch(ctx context.Context, id string, patch db.PluginPatch) (db.Plugin, error) +} + +// ListStates returns the state of all plugins, hidden and disabled plugins included, without pagination. +func (h Handlers) ListStates(rw http.ResponseWriter, req *http.Request) { + ctx, span := h.tracer.Start(req.Context(), "handler_list_states") + defer span.End() + + rw.Header().Set("Content-Type", "application/json") + + states, err := h.store.ListStates(ctx) + if err != nil { + span.RecordError(err) + log.Error().Err(err).Msg("Error fetching plugin states") + JSONInternalServerError(rw) + + return + } + + if err := json.NewEncoder(rw).Encode(states); err != nil { + span.RecordError(err) + log.Error().Err(err).Msg("Failed to encode plugin states") + JSONInternalServerError(rw) + + return + } +} + +// Patch updates only the given fields of a plugin (stars, hidden, tagsETag). +func (h Handlers) Patch(rw http.ResponseWriter, req *http.Request) { + ctx, span := h.tracer.Start(req.Context(), "handler_patch") + defer span.End() + + rw.Header().Set("Content-Type", "application/json") + + id, err := getPathParam(req.URL) + if err != nil { + span.RecordError(err) + JSONError(rw, http.StatusBadRequest, "Missing plugin id") + + return + } + + logger := log.With().Str("plugin_id", id).Logger() + + var patch db.PluginPatch + + dec := json.NewDecoder(http.MaxBytesReader(rw, req.Body, maxPatchSize)) + dec.DisallowUnknownFields() + + // An empty body is an empty patch. + if err = dec.Decode(&patch); err != nil && !errors.Is(err, io.EOF) { + span.RecordError(err) + logger.Error().Err(err).Msg("Unable to decode plugin patch") + JSONError(rw, http.StatusBadRequest, err.Error()) + + return + } + + if dec.More() { + JSONError(rw, http.StatusBadRequest, "body must contain a single JSON object") + + return + } + + if patch.IsEmpty() { + JSONError(rw, http.StatusBadRequest, "nothing to update") + + return + } + + if patch.Stars != nil && *patch.Stars < 0 { + JSONError(rw, http.StatusBadRequest, "stars must not be negative") + + return + } + + updated, err := h.store.Patch(ctx, id, patch) + if err != nil { + span.RecordError(err) + + if errors.As(err, &db.NotFoundError{}) { + NotFound(rw, req) + + return + } + + logger.Error().Err(err).Msg("Error patching plugin") + JSONInternalServerError(rw) + + return + } + + if err := json.NewEncoder(rw).Encode(updated); err != nil { + span.RecordError(err) + logger.Error().Err(err).Msg("Failed to encode patched plugin") + JSONInternalServerError(rw) + + return + } +} diff --git a/pkg/handlers/state_test.go b/pkg/handlers/state_test.go new file mode 100644 index 0000000..ef4244d --- /dev/null +++ b/pkg/handlers/state_test.go @@ -0,0 +1,159 @@ +package handlers + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/traefik/plugin-service/pkg/db" +) + +func TestHandlers_ListStates(t *testing.T) { + testDB := mockDB{ + listStatesFn: func(_ context.Context) ([]db.PluginState, error) { + return []db.PluginState{ + {ID: "123", Name: "github.com/traefik/plugindemo", LatestVersion: "v0.2.1", Stars: 22, TagsETag: `W/"abc"`}, + {ID: "456", Name: "github.com/traefik/hidden", LatestVersion: "v1.0.0", Hidden: true, Disabled: true}, + }, nil + }, + } + + rw := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/internal/states", http.NoBody) + + New(testDB, nil, nil, "").ListStates(rw, req) + + require.Equal(t, http.StatusOK, rw.Code) + assert.JSONEq(t, `[ + {"id":"123","name":"github.com/traefik/plugindemo","latestVersion":"v0.2.1","stars":22,"hidden":false,"disabled":false,"tagsETag":"W/\"abc\""}, + {"id":"456","name":"github.com/traefik/hidden","latestVersion":"v1.0.0","stars":0,"hidden":true,"disabled":true,"tagsETag":""} + ]`, rw.Body.String()) +} + +func TestHandlers_ListStates_empty(t *testing.T) { + testDB := mockDB{ + listStatesFn: func(_ context.Context) ([]db.PluginState, error) { + return []db.PluginState{}, nil + }, + } + + rw := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/internal/states", http.NoBody) + + New(testDB, nil, nil, "").ListStates(rw, req) + + require.Equal(t, http.StatusOK, rw.Code) + assert.JSONEq(t, `[]`, rw.Body.String()) +} + +func TestHandlers_Patch(t *testing.T) { + testCases := []struct { + desc string + body string + wantPatch db.PluginPatch + }{ + { + desc: "stars only", + body: `{"stars":12}`, + wantPatch: db.PluginPatch{Stars: new(12)}, + }, + { + desc: "hidden only", + body: `{"hidden":false}`, + wantPatch: db.PluginPatch{Hidden: new(false)}, + }, + { + desc: "empty ETag", + body: `{"tagsETag":""}`, + wantPatch: db.PluginPatch{TagsETag: new("")}, + }, + { + desc: "all fields", + body: `{"stars":0,"hidden":true,"tagsETag":"W/\"abc\""}`, + wantPatch: db.PluginPatch{Stars: new(0), Hidden: new(true), TagsETag: new(`W/"abc"`)}, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + var ( + gotID string + gotPatch db.PluginPatch + ) + + testDB := mockDB{ + patchFn: func(_ context.Context, id string, patch db.PluginPatch) (db.Plugin, error) { + gotID = id + gotPatch = patch + + return db.Plugin{ID: id, Name: "github.com/traefik/plugindemo"}, nil + }, + } + + rw := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPatch, "/123", strings.NewReader(test.body)) + + New(testDB, nil, nil, "").Patch(rw, req) + + require.Equal(t, http.StatusOK, rw.Code) + assert.Equal(t, "123", gotID) + assert.Equal(t, test.wantPatch, gotPatch) + }) + } +} + +func TestHandlers_Patch_badRequest(t *testing.T) { + testDB := mockDB{ + patchFn: func(_ context.Context, _ string, _ db.PluginPatch) (db.Plugin, error) { + t.Fatal("store should not be called on invalid input") + return db.Plugin{}, nil + }, + } + + testCases := []struct { + desc string + path string + body string + }{ + {desc: "empty body", path: "/123", body: ""}, + {desc: "empty object", path: "/123", body: `{}`}, + {desc: "null values", path: "/123", body: `{"stars":null}`}, + {desc: "unknown field", path: "/123", body: `{"name":"other"}`}, + {desc: "invalid JSON", path: "/123", body: `{`}, + {desc: "missing id", path: "/", body: `{"stars":1}`}, + {desc: "two JSON objects", path: "/123", body: `{"stars":1}{"hidden":true}`}, + {desc: "trailing data", path: "/123", body: `{"stars":1} garbage`}, + {desc: "negative stars", path: "/123", body: `{"stars":-5}`}, + {desc: "too large", path: "/123", body: `{"tagsETag":"` + strings.Repeat("a", maxPatchSize) + `"}`}, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + rw := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPatch, test.path, strings.NewReader(test.body)) + + New(testDB, nil, nil, "").Patch(rw, req) + + assert.Equal(t, http.StatusBadRequest, rw.Code) + }) + } +} + +func TestHandlers_Patch_notFound(t *testing.T) { + testDB := mockDB{ + patchFn: func(_ context.Context, _ string, _ db.PluginPatch) (db.Plugin, error) { + return db.Plugin{}, db.NotFoundError{} + }, + } + + rw := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPatch, "/123", strings.NewReader(`{"stars":1}`)) + + New(testDB, nil, nil, "").Patch(rw, req) + + assert.Equal(t, http.StatusNotFound, rw.Code) +}