Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions cmd/serve/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,9 @@ func buildInternalRouter(handler handlers.Handlers) http.Handler {
r.Handler(http.MethodGet, "/", otelhttp.NewHandler(http.HandlerFunc(handler.List), "internal_list"))
r.Handler(http.MethodPost, "/", otelhttp.NewHandler(http.HandlerFunc(handler.Create), "internal_create"))
r.Handler(http.MethodPut, "/:uuid", otelhttp.NewHandler(http.HandlerFunc(handler.Update), "internal_update"))
r.Handler(http.MethodPatch, "/:uuid", otelhttp.NewHandler(http.HandlerFunc(handler.Patch), "internal_patch"))

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The blacklist workaround is not needed here: httprouter has one tree per method, and the GET tree only has / (no /:uuid). Keeping the route in the internal router also keeps its PanicHandler and JSON NotFound.

Suggested change
r.Handler(http.MethodPatch, "/:uuid", otelhttp.NewHandler(http.HandlerFunc(handler.Patch), "internal_patch"))
r.Handler(http.MethodPatch, "/:uuid", otelhttp.NewHandler(http.HandlerFunc(handler.Patch), "internal_patch"))
r.Handler(http.MethodGet, "/states", otelhttp.NewHandler(http.HandlerFunc(handler.ListStates), "internal_list_states"))

Then registerInternalStatesRoute and its call can be removed.

// No conflict with "/:uuid": httprouter has one tree per method, and no GET route has a wildcard.
r.Handler(http.MethodGet, "/states", otelhttp.NewHandler(http.HandlerFunc(handler.ListStates), "internal_list_states"))
r.Handler(http.MethodDelete, "/:uuid", otelhttp.NewHandler(http.HandlerFunc(handler.Delete), "internal_delete"))

r.NotFound = http.HandlerFunc(handlers.NotFound)
Expand Down
27 changes: 27 additions & 0 deletions pkg/db/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,33 @@ type Plugin struct {
Disabled bool `json:"disabled,omitempty" bson:"disabled"`
Hidden bool `json:"hidden,omitempty" bson:"hidden"`
UseUnsafe bool `json:"useUnsafe,omitempty" bson:"useUnsafe"`
// TagsETag is the ETag of the last GitHub tags response seen by the scraper.
// omitempty: an update without the ETag (an older scraper) does not erase it.
TagsETag string `json:"tagsETag,omitempty" bson:"tagsETag,omitempty"`
}

// PluginState is the part of a plugin the scraper needs to know if a plugin changed.
type PluginState struct {
ID string `json:"id" bson:"id"`
Name string `json:"name" bson:"name"`
LatestVersion string `json:"latestVersion" bson:"latestVersion"`
Stars int `json:"stars" bson:"stars"`
Hidden bool `json:"hidden" bson:"hidden"`
Disabled bool `json:"disabled" bson:"disabled"`
TagsETag string `json:"tagsETag" bson:"tagsETag"`
}

// PluginPatch holds the plugin fields that can be updated alone.
// A nil field is not updated.
type PluginPatch struct {
Stars *int `json:"stars,omitempty"`
Hidden *bool `json:"hidden,omitempty"`
TagsETag *string `json:"tagsETag,omitempty"`
}

// IsEmpty returns true when the patch updates nothing.
func (p PluginPatch) IsEmpty() bool {
return p.Stars == nil && p.Hidden == nil && p.TagsETag == nil
}

// BlacklistEntry represents a repository excluded from the plugin scraping.
Expand Down
2 changes: 1 addition & 1 deletion pkg/db/mongodb/bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func (m *MongoDB) Bootstrap() error {
Options: &options.IndexOptions{
Name: new("_by_stars"),
},
Keys: bson.D{{Key: "stars", Value: 1}},
Keys: bson.D{{Key: fieldStars, Value: 1}},
},
{
Options: &options.IndexOptions{
Expand Down
84 changes: 83 additions & 1 deletion pkg/db/mongodb/plugindb.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ const (
fieldHashes = "hashes"
fieldDisabled = "disabled"
fieldHidden = "hidden"
fieldTagsETag = "tagsETag"
fieldStars = "stars"
)

// MongoDB is a mongoDB client.
Expand Down Expand Up @@ -160,7 +162,7 @@ func (m *MongoDB) List(ctx context.Context, page db.Pagination) ([]db.Plugin, st
opts := &options.FindOptions{}
opts.SetLimit(int64(page.Size + 1))
opts.SetProjection(bson.D{{Key: fieldHashes, Value: 0}})
opts.SetSort(bson.D{{Key: "stars", Value: -1}})
opts.SetSort(bson.D{{Key: fieldStars, Value: -1}})

cursor, err := m.client.Collection(m.collName).Find(ctx, criteria, opts)
if err != nil {
Expand Down Expand Up @@ -326,6 +328,86 @@ func (m *MongoDB) Update(ctx context.Context, id string, plugin db.Plugin) (db.P
return updated, nil
}

// ListStates returns the state of all plugins, hidden and disabled plugins included.
func (m *MongoDB) ListStates(ctx context.Context) ([]db.PluginState, error) {
ctx, span := m.tracer.Start(ctx, "db_list_states")
defer span.End()

opts := &options.FindOptions{}
opts.SetProjection(bson.D{
{Key: "_id", Value: 0},
{Key: "id", Value: 1},
{Key: "name", Value: 1},
{Key: "latestVersion", Value: 1},
{Key: fieldStars, Value: 1},
{Key: fieldHidden, Value: 1},
{Key: fieldDisabled, Value: 1},
{Key: fieldTagsETag, Value: 1},
})

cursor, err := m.client.Collection(m.collName).Find(ctx, bson.D{}, opts)
if err != nil {
span.RecordError(err)

return nil, fmt.Errorf("unable to find plugin states: %w", err)
}

states := make([]db.PluginState, 0)

if err = cursor.All(ctx, &states); err != nil {
span.RecordError(err)

return nil, fmt.Errorf("unable to unmarshal plugin states: %w", err)
}

return states, nil
}

// Patch updates only the non-nil fields of the patch.
func (m *MongoDB) Patch(ctx context.Context, id string, patch db.PluginPatch) (db.Plugin, error) {
ctx, span := m.tracer.Start(ctx, "db_patch")
defer span.End()

set := bson.D{}

if patch.Stars != nil {
set = append(set, bson.E{Key: fieldStars, Value: *patch.Stars})
}

if patch.Hidden != nil {
set = append(set, bson.E{Key: fieldHidden, Value: *patch.Hidden})
}

if patch.TagsETag != nil {
set = append(set, bson.E{Key: fieldTagsETag, Value: *patch.TagsETag})
}

if len(set) == 0 {
return db.Plugin{}, errors.New("nothing to update")
}

filter := bson.D{{Key: "id", Value: id}}
update := bson.D{{Key: "$set", Value: set}}

opts := &options.FindOneAndUpdateOptions{}
opts.SetReturnDocument(options.After)
opts.SetProjection(bson.D{{Key: fieldHashes, Value: 0}})

var updated db.Plugin

if err := m.client.Collection(m.collName).FindOneAndUpdate(ctx, filter, update, opts).Decode(&updated); err != nil {
span.RecordError(err)

if errors.Is(err, mongo.ErrNoDocuments) {
return db.Plugin{}, db.NotFoundError{Err: err}
}

return db.Plugin{}, fmt.Errorf("unable to patch plugin: %w", err)
}

return updated, nil
}

// CreateHash creates a new plugin hash.
func (m *MongoDB) CreateHash(ctx context.Context, module, version, hash string) (db.PluginHash, error) {
ctx, span := m.tracer.Start(ctx, "db_create_hash")
Expand Down
141 changes: 137 additions & 4 deletions pkg/db/mongodb/plugindb_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ func TestMongoDB_Create(t *testing.T) {
"something": "there",
},
CreatedAt: time.Now().Add(-2 * time.Hour),
TagsETag: `W/"abc"`,
}

got, err := store.Create(ctx, plugin)
Expand Down Expand Up @@ -650,8 +651,9 @@ func TestMongoDB_Update(t *testing.T) {
Versions: []string{
"v1.1.1",
},
Stars: 10,
Snippet: nil,
Stars: 10,
Snippet: nil,
TagsETag: `W/"old"`,
},
Hashes: []db.PluginHash{
{Name: "plugin@v1.1.1", Hash: "123"},
Expand All @@ -676,14 +678,16 @@ func TestMongoDB_Update(t *testing.T) {
Versions: []string{
"v1.1.1",
},
Stars: 10,
Snippet: nil,
Stars: 10,
Snippet: nil,
TagsETag: `W/"new"`,
})
require.NoError(t, err)

want := fixtures["plugin"].Plugin
want.Name = "New Name"
want.Author = "New Author"
want.TagsETag = `W/"new"`

assert.Equal(t, want, got)

Expand All @@ -703,6 +707,15 @@ func TestMongoDB_Update(t *testing.T) {

assert.Equal(t, want, got)

// An update without the ETag (an older scraper) keeps the stored one.
withoutETag := got
withoutETag.TagsETag = ""

got, err = store.Update(ctx, "123", withoutETag)
require.NoError(t, err)

assert.Equal(t, want, got)

// Check that we get a db.NotFound when no plugin have the given id.
_, err = store.Update(ctx, "456", got)
require.ErrorAs(t, err, &db.NotFoundError{})
Expand Down Expand Up @@ -987,3 +1000,123 @@ func buildNextID(t *testing.T, next db.Plugin) string {

return base64.RawStdEncoding.EncodeToString(b)
}

func TestMongoDB_ListStates(t *testing.T) {
ctx := context.Background()

store, _ := createDatabase(t, []fixture{
{
key: "active",
plugin: pluginDocument{
Plugin: db.Plugin{
ID: "123",
Name: "active",
Readme: "readme",
LatestVersion: "v1.0.0",
Stars: 10,
Snippet: map[string]any{"something": "there"},
TagsETag: `W/"abc"`,
},
Hashes: []db.PluginHash{{Name: "active@v1.0.0", Hash: "123"}},
},
},
{
key: "hidden",
plugin: pluginDocument{
Plugin: db.Plugin{ID: "234", Name: "hidden", LatestVersion: "v2.0.0", Stars: 3, Hidden: true},
},
},
{
key: "disabled",
plugin: pluginDocument{
Plugin: db.Plugin{ID: "345", Name: "disabled", LatestVersion: "v3.0.0", Disabled: true},
},
},
})

states, err := store.ListStates(ctx)
require.NoError(t, err)

assert.ElementsMatch(t, []db.PluginState{
{ID: "123", Name: "active", LatestVersion: "v1.0.0", Stars: 10, TagsETag: `W/"abc"`},
{ID: "234", Name: "hidden", LatestVersion: "v2.0.0", Stars: 3, Hidden: true},
{ID: "345", Name: "disabled", LatestVersion: "v3.0.0", Disabled: true},
}, states)
}

func TestMongoDB_ListStates_empty(t *testing.T) {
store, _ := createDatabase(t, nil)

states, err := store.ListStates(context.Background())
require.NoError(t, err)

assert.NotNil(t, states)
assert.Empty(t, states)
}

func TestMongoDB_Patch(t *testing.T) {
ctx := context.Background()

store, fixtures := createDatabase(t, []fixture{
{
key: "plugin",
plugin: pluginDocument{
Plugin: db.Plugin{
ID: "123",
Name: "plugin",
DisplayName: "plugin",
Author: "author",
Readme: "readme",
LatestVersion: "v1.1.1",
Versions: []string{"v1.1.1"},
Stars: 10,
Snippet: map[string]any{"something": "there"},
Disabled: true,
TagsETag: `W/"old"`,
},
Hashes: []db.PluginHash{
{Name: "plugin@v1.1.1", Hash: "123"},
},
},
},
})

want := fixtures["plugin"].Plugin

// Only stars.
got, err := store.Patch(ctx, "123", db.PluginPatch{Stars: new(42)})
require.NoError(t, err)

want.Stars = 42
assert.Equal(t, want, toUTCPlugin(got))

// Only hidden.
got, err = store.Patch(ctx, "123", db.PluginPatch{Hidden: new(true)})
require.NoError(t, err)

want.Hidden = true
assert.Equal(t, want, toUTCPlugin(got))

// Only ETag, zero values are written.
got, err = store.Patch(ctx, "123", db.PluginPatch{Stars: new(0), TagsETag: new(`W/"new"`)})
require.NoError(t, err)

want.Stars = 0
want.TagsETag = `W/"new"`
assert.Equal(t, want, toUTCPlugin(got))

// Check the stored document, hashes included, is not changed in other fields.
stored, ok := getPlugin(t, store, "123")
require.True(t, ok)

assert.Equal(t, want, toUTCPlugin(stored.Plugin))
assert.Equal(t, fixtures["plugin"].Hashes, stored.Hashes)

// Empty patch.
_, err = store.Patch(ctx, "123", db.PluginPatch{})
require.Error(t, err)

// Unknown plugin.
_, err = store.Patch(ctx, "456", db.PluginPatch{Stars: new(1)})
require.ErrorAs(t, err, &db.NotFoundError{})
}
10 changes: 10 additions & 0 deletions pkg/handlers/dbmock_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ type mockDB struct {
getByNameFn func(context.Context, string, bool) (db.Plugin, error)
searchByNameFn func(context.Context, string, db.Pagination) ([]db.Plugin, string, error)
updateFn func(context.Context, string, db.Plugin) (db.Plugin, error)
listStatesFn func(ctx context.Context) ([]db.PluginState, error)
patchFn func(ctx context.Context, id string, patch db.PluginPatch) (db.Plugin, error)

deleteHashFn func(ctx context.Context, id string) error
createHashFn func(ctx context.Context, module, version, hash string) (db.PluginHash, error)
Expand Down Expand Up @@ -53,6 +55,14 @@ func (m mockDB) Update(ctx context.Context, id string, plugin db.Plugin) (db.Plu
return m.updateFn(ctx, id, plugin)
}

func (m mockDB) ListStates(ctx context.Context) ([]db.PluginState, error) {
return m.listStatesFn(ctx)
}

func (m mockDB) Patch(ctx context.Context, id string, patch db.PluginPatch) (db.Plugin, error) {
return m.patchFn(ctx, id, patch)
}

func (m mockDB) DeleteHash(ctx context.Context, id string) error {
return m.deleteHashFn(ctx, id)
}
Expand Down
1 change: 1 addition & 0 deletions pkg/handlers/handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ type PluginStorer interface {
// Storer is capable of storing plugins and the plugin blacklist.
type Storer interface {
PluginStorer
PluginStateStorer
BlacklistStorer
}

Expand Down
Loading
Loading