From d2de22442bb638e161a8d7915c5f4276e40113bd Mon Sep 17 00:00:00 2001 From: Clifford Tawiah Date: Mon, 5 Oct 2026 20:54:56 -0400 Subject: [PATCH] feat(sync): persist manifests in LaunchDarkly --- cmd/sync/prompt.go | 2 +- internal/sync/api/client.go | 130 ++++++++ internal/sync/api/client_test.go | 64 ++++ internal/sync/bootstrap/bootstrap.go | 18 +- internal/sync/bootstrap/bootstrap_test.go | 36 ++- internal/sync/detach/detach.go | 43 +-- internal/sync/detach/detach_test.go | 63 ++-- internal/sync/local/attachment.go | 4 + internal/sync/manifest/model.go | 24 +- internal/sync/manifest/store.go | 345 ++++++++++++++++------ internal/sync/manifest/store_test.go | 155 ++++++---- internal/sync/prompt/acceptance_test.go | 186 ++++++++++-- internal/sync/prompt/conflict_test.go | 33 ++- internal/sync/prompt/execute.go | 3 + internal/sync/prompt/execute_test.go | 1 - internal/sync/prompt/plan_test.go | 1 - internal/sync/prompt/runner.go | 83 +++++- internal/sync/prompt/watch_test.go | 2 +- internal/sync/repository/git.go | 48 ++- internal/sync/repository/git_test.go | 35 ++- internal/sync/source/resolver.go | 62 +++- internal/sync/source/resolver_test.go | 45 ++- 22 files changed, 1107 insertions(+), 276 deletions(-) diff --git a/cmd/sync/prompt.go b/cmd/sync/prompt.go index 3c9e0a3a..b5aa3b1f 100644 --- a/cmd/sync/prompt.go +++ b/cmd/sync/prompt.go @@ -37,7 +37,7 @@ func NewPromptCmd(client resources.Client) *cobra.Command { cmd := &cobra.Command{ Use: "prompt", Short: "Synchronize local prompt variations with LaunchDarkly", - Long: "Bootstrap local prompt variations from LaunchDarkly, add more variations, or synchronize changes using the committed local manifest. " + + Long: "Bootstrap local prompt variations from LaunchDarkly, add more variations, or synchronize changes using the LaunchDarkly manifest. " + "Sync rechecks state before every write, rerun sync after a change.", Args: func(cmd *cobra.Command, args []string) error { if err := cobra.NoArgs(cmd, args); err != nil { diff --git a/internal/sync/api/client.go b/internal/sync/api/client.go index 41d89b6d..31e70cc9 100644 --- a/internal/sync/api/client.go +++ b/internal/sync/api/client.go @@ -26,6 +26,41 @@ type VariationState struct { ConfigMode syncdomain.VariationMode } +// SyncManifest is the complete synchronization baseline for one project and source. +type SyncManifest struct { + Source string `json:"source"` + Items []SyncManifestResource `json:"items"` +} + +// SyncManifestResource is one versioned baseline entry returned by LaunchDarkly. +type SyncManifestResource struct { + ResourceKind syncdomain.Kind `json:"resourceKind"` + ResourceLookupKey string `json:"resourceLookupKey"` + Fingerprint string `json:"fingerprint"` + Version int `json:"version"` +} + +// SyncManifestUpsert creates or updates one baseline entry. +type SyncManifestUpsert struct { + ResourceKind syncdomain.Kind `json:"resourceKind"` + ResourceLookupKey string `json:"resourceLookupKey"` + Fingerprint string `json:"fingerprint"` + Version int `json:"version"` +} + +// SyncManifestDeletion removes one baseline entry at its expected version. +type SyncManifestDeletion struct { + ResourceKind syncdomain.Kind `json:"resourceKind"` + ResourceLookupKey string `json:"resourceLookupKey"` + Version int `json:"version"` +} + +type patchSyncManifestRequest struct { + Source string `json:"source"` + Upserts []SyncManifestUpsert `json:"upserts"` + Deletions []SyncManifestDeletion `json:"deletions"` +} + type createVariationRequest struct { Key string `json:"key"` Name string `json:"name"` @@ -81,6 +116,82 @@ func NewClient(transport resources.Client, accessToken, baseURI string) Client { } } +// GetSyncManifest returns the synchronization baseline for one project and source. +func (client Client) GetSyncManifest(projectKey, source string) (SyncManifest, error) { + endpoint, err := client.syncManifestEndpoint(projectKey) + if err != nil { + return SyncManifest{}, err + } + + response, err := client.transport.MakeRequest( + client.accessToken, + http.MethodGet, + endpoint, + "", + url.Values{"source": []string{source}}, + nil, + false, + ) + if err != nil { + return SyncManifest{}, fmt.Errorf("get sync manifest for project %q: %w", projectKey, err) + } + + var manifest SyncManifest + if err := json.Unmarshal(response, &manifest); err != nil { + return SyncManifest{}, fmt.Errorf("decode sync manifest for project %q: %w", projectKey, err) + } + return manifest, nil +} + +// PatchSyncManifest applies versioned baseline changes and returns the refreshed manifest. +func (client Client) PatchSyncManifest( + projectKey string, + source string, + upserts []SyncManifestUpsert, + deletions []SyncManifestDeletion, +) (SyncManifest, error) { + endpoint, err := client.syncManifestEndpoint(projectKey) + if err != nil { + return SyncManifest{}, err + } + if upserts == nil { + upserts = []SyncManifestUpsert{} + } + if deletions == nil { + deletions = []SyncManifestDeletion{} + } + body, err := json.Marshal(patchSyncManifestRequest{ + Source: source, Upserts: upserts, Deletions: deletions, + }) + if err != nil { + return SyncManifest{}, fmt.Errorf("encode sync manifest changes for project %q: %w", projectKey, err) + } + + response, err := client.transport.MakeRequest( + client.accessToken, + http.MethodPatch, + endpoint, + "application/json", + nil, + body, + false, + ) + if err != nil { + return SyncManifest{}, newResourceMutationError("update", "sync manifest for project", projectKey, err) + } + + var manifest SyncManifest + if err := json.Unmarshal(response, &manifest); err != nil { + return SyncManifest{}, newResourceMutationError( + "decode updated", + "sync manifest for project", + projectKey, + err, + ) + } + return manifest, nil +} + // ModelConfig returns the latest version of one model config. func (client Client) ModelConfig(projectKey, modelConfigKey string) (ModelConfig, error) { endpoint, err := url.JoinPath(client.baseURI, "api/v2/projects", projectKey, "ai-configs/model-configs", modelConfigKey) @@ -274,6 +385,19 @@ func (client Client) variationEndpoint(projectKey, configKey string, path ...str return endpoint, nil } +func (client Client) syncManifestEndpoint(projectKey string) (string, error) { + endpoint, err := url.JoinPath( + client.baseURI, + "api/v2/projects", + projectKey, + "configs/sync/manifests", + ) + if err != nil { + return "", fmt.Errorf("build sync manifest endpoint: %w", err) + } + return endpoint, nil +} + // newMutationError records whether a failed request received a definitive API // response. Errors without a status code may represent a committed write whose // response was lost, so the reconciliation layer verifies those with a read. @@ -302,3 +426,9 @@ func responseStatusCode(err error) (int, bool) { } return 0, false } + +// IsConflict reports whether a wrapped LaunchDarkly API error has a 409 status. +func IsConflict(err error) bool { + status, ok := responseStatusCode(err) + return ok && status == http.StatusConflict +} diff --git a/internal/sync/api/client_test.go b/internal/sync/api/client_test.go index 5a45de60..3e39b100 100644 --- a/internal/sync/api/client_test.go +++ b/internal/sync/api/client_test.go @@ -2,6 +2,7 @@ package api import ( "errors" + "net/http" "net/url" "testing" @@ -498,6 +499,69 @@ func TestClientArchiveVariationReturnsTransportError(t *testing.T) { require.ErrorContains(t, err, `archive config variation "helpful": in use`) } +func TestClientGetsSyncManifest(t *testing.T) { + transport := &recordingClient{Responses: [][]byte{[]byte(`{ + "source": "git:example.com/acme/repo", + "items": [{ + "resourceKind": "variation", + "resourceLookupKey": "support/helpful", + "fingerprint": "sha256:abc", + "version": 3 + }] + }`)}} + client := NewClient(transport, "token", "https://example.com") + + manifest, err := client.GetSyncManifest("project", "git:example.com/acme/repo") + + require.NoError(t, err) + require.Len(t, manifest.Items, 1) + assert.Equal(t, 3, manifest.Items[0].Version) + require.Len(t, transport.Requests, 1) + request := transport.Requests[0] + assert.Equal(t, http.MethodGet, request.Method) + assert.Equal(t, "https://example.com/api/v2/projects/project/configs/sync/manifests", request.Path) + assert.Equal(t, "git:example.com/acme/repo", request.Query.Get("source")) +} + +func TestClientPatchesSyncManifest(t *testing.T) { + transport := &recordingClient{Responses: [][]byte{[]byte(`{ + "source": "git:example.com/acme/repo", + "items": [] + }`)}} + client := NewClient(transport, "token", "https://example.com") + + _, err := client.PatchSyncManifest( + "project", + "git:example.com/acme/repo", + []SyncManifestUpsert{{ + ResourceKind: syncdomain.KindVariation, ResourceLookupKey: "support/helpful", + Fingerprint: "sha256:abc", Version: 2, + }}, + nil, + ) + + require.NoError(t, err) + require.Len(t, transport.Requests, 1) + request := transport.Requests[0] + assert.Equal(t, http.MethodPatch, request.Method) + assert.Equal(t, "application/json", request.ContentType) + assert.JSONEq(t, `{ + "source": "git:example.com/acme/repo", + "upserts": [{ + "resourceKind": "variation", + "resourceLookupKey": "support/helpful", + "fingerprint": "sha256:abc", + "version": 2 + }], + "deletions": [] + }`, string(request.Body)) +} + +func TestIsConflictRecognizesAPIStatus(t *testing.T) { + assert.True(t, IsConflict(errors.New(`{"code":"conflict","statusCode":409}`))) + assert.False(t, IsConflict(errors.New(`{"code":"invalid_request","statusCode":400}`))) +} + func testVariation(mode syncdomain.VariationMode) syncdomain.Variation { return syncdomain.Variation{ Mode: mode, diff --git a/internal/sync/bootstrap/bootstrap.go b/internal/sync/bootstrap/bootstrap.go index e06e99ab..4ae7d65e 100644 --- a/internal/sync/bootstrap/bootstrap.go +++ b/internal/sync/bootstrap/bootstrap.go @@ -29,8 +29,8 @@ type AttachmentReader interface { // ManifestStore persists the synchronization baseline after local files are written. type ManifestStore interface { - Load() (syncmanifest.Manifest, bool, error) - Write(syncmanifest.Manifest) error + Load(projectKeys []string) (syncmanifest.Manifest, error) + Update(previous, next syncmanifest.Manifest) (syncmanifest.Manifest, error) } // Options contains the dependencies and streams for one bootstrap flow. @@ -236,20 +236,26 @@ func finishSelection(options Options, files []synclocal.VariationFile) error { return nil } - manifest, _, err := options.Manifest.Load() + projectKeys := make([]string, 0, len(files)) + for _, file := range files { + projectKeys = append(projectKeys, file.ProjectKey) + } + manifest, err := options.Manifest.Load(projectKeys) if err != nil { return err } + updatedManifest := manifest + updatedManifest.Resources = append([]syncmanifest.Resource(nil), manifest.Resources...) for _, file := range files { lookupKey := file.ConfigKey + "/" + file.Variation.Key fingerprint, err := syncdomain.FingerprintVariation(file.ProjectKey, lookupKey, file.Variation) if err != nil { return err } - manifest.SetFingerprint(syncdomain.ResourceID{ + updatedManifest.SetFingerprint(syncdomain.ResourceID{ Kind: syncdomain.KindVariation, ProjectKey: file.ProjectKey, LookupKey: lookupKey, }, fingerprint) - if err := manifest.SetAttachmentsIfMissing(file.ProjectKey, file.Variation.Attachments); err != nil { + if err := updatedManifest.SetAttachmentsIfMissing(file.ProjectKey, file.Variation.Attachments); err != nil { return err } } @@ -266,7 +272,7 @@ func finishSelection(options Options, files []synclocal.VariationFile) error { // The manifest is written last so it never claims a wrapper exists before // that wrapper reaches disk. Roll back every file this operation created if // persistence fails, including shared dependencies that did not exist before. - if err := options.Manifest.Write(manifest); err != nil { + if _, err := options.Manifest.Update(manifest, updatedManifest); err != nil { return errors.Join(err, options.Store.RollbackCreation(creation)) } diff --git a/internal/sync/bootstrap/bootstrap_test.go b/internal/sync/bootstrap/bootstrap_test.go index 3d354bfa..ae73bb46 100644 --- a/internal/sync/bootstrap/bootstrap_test.go +++ b/internal/sync/bootstrap/bootstrap_test.go @@ -135,6 +135,7 @@ Be helpful. func TestFinishSelectionWritesInitialManifest(t *testing.T) { root := t.TempDir() var output bytes.Buffer + manifestStore := &memoryManifestStore{manifest: syncmanifest.New()} variation := syncdomain.Variation{ Mode: syncdomain.VariationModeAgent, Key: "variation", Name: "Variation", Instructions: "Be helpful.", Tools: []syncdomain.AttachmentRef{{Key: "search"}}, @@ -149,7 +150,7 @@ func TestFinishSelectionWritesInitialManifest(t *testing.T) { err := finishSelection(Options{ Store: synclocal.NewStore(root), - Manifest: syncmanifest.NewStore(root), + Manifest: manifestStore, Output: &output, Initial: true, }, []synclocal.VariationFile{ @@ -158,9 +159,8 @@ func TestFinishSelectionWritesInitialManifest(t *testing.T) { }) require.NoError(t, err) - manifest, exists, err := syncmanifest.NewStore(root).Load() + manifest, err := manifestStore.Load([]string{"project"}) require.NoError(t, err) - require.True(t, exists) require.Len(t, manifest.Resources, 3) attachmentFingerprint, err := syncdomain.FingerprintAttachment("project", variation.Attachments[0]) require.NoError(t, err) @@ -178,7 +178,7 @@ func TestFinishSelectionWritesInitialManifest(t *testing.T) { func TestFinishSelectionAddsMultipleVersionedVariationsToExistingManifest(t *testing.T) { root := t.TempDir() store := synclocal.NewStore(root) - manifestStore := syncmanifest.NewStore(root) + manifestStore := &memoryManifestStore{manifest: syncmanifest.New()} first := syncdomain.Variation{ Mode: syncdomain.VariationModeAgent, Key: "first", Name: "First", Instructions: "First.", } @@ -203,9 +203,8 @@ func TestFinishSelectionAddsMultipleVersionedVariationsToExistingManifest(t *tes {ProjectKey: "project", ConfigKey: "config", Variation: third}, })) - manifest, exists, err := manifestStore.Load() + manifest, err := manifestStore.Load([]string{"project"}) require.NoError(t, err) - require.True(t, exists) require.Len(t, manifest.Resources, 3) assert.Equal(t, "config/first", manifest.Resources[0].LookupKey) assert.Equal(t, "config/second", manifest.Resources[1].LookupKey) @@ -330,10 +329,27 @@ func (*fakeCatalog) SearchConfigs(string, string, []syncdomain.VariationMode, in type failingManifestStore struct{} -func (failingManifestStore) Load() (syncmanifest.Manifest, bool, error) { - return syncmanifest.New(), false, nil +func (failingManifestStore) Load([]string) (syncmanifest.Manifest, error) { + return syncmanifest.New(), nil } -func (failingManifestStore) Write(syncmanifest.Manifest) error { - return errors.New("write manifest") +func (failingManifestStore) Update(syncmanifest.Manifest, syncmanifest.Manifest) (syncmanifest.Manifest, error) { + return syncmanifest.Manifest{}, errors.New("write manifest") +} + +type memoryManifestStore struct { + manifest syncmanifest.Manifest +} + +func (store *memoryManifestStore) Load([]string) (syncmanifest.Manifest, error) { + return store.manifest, nil +} + +func (store *memoryManifestStore) Update( + _ syncmanifest.Manifest, + next syncmanifest.Manifest, +) (syncmanifest.Manifest, error) { + next.Sort() + store.manifest = next + return next, nil } diff --git a/internal/sync/detach/detach.go b/internal/sync/detach/detach.go index f54f07db..3a721a09 100644 --- a/internal/sync/detach/detach.go +++ b/internal/sync/detach/detach.go @@ -18,18 +18,24 @@ import ( // Resource identifies one local or manifested resource that can be detached. type Resource = syncdomain.ResourceID +type ManifestStore interface { + Load(projectKeys []string) (syncmanifest.Manifest, error) + Update(previous, next syncmanifest.Manifest) (syncmanifest.Manifest, error) +} + // Options contains the local stores and streams used by detach. type Options struct { RepositoryRoot string Store synclocal.Store - Manifest syncmanifest.Store + Manifest ManifestStore + ProjectKeys []string Input io.Reader Output io.Writer } // Run lets the user select resources and removes their local sync state. func Run(options Options) error { - resources, manifest, manifestExists, err := loadResources(options.RepositoryRoot, options.Manifest) + resources, manifest, err := loadResources(options.RepositoryRoot, options.Manifest, options.ProjectKeys) if err != nil { return err } @@ -62,7 +68,7 @@ func Run(options Options) error { if canceled { return nil } - if err := detachResources(options, manifest, manifestExists, selected); err != nil { + if err := detachResources(options, manifest, selected); err != nil { return err } @@ -75,10 +81,14 @@ func Run(options Options) error { } // loadResources returns the union of local wrappers and manifested resources. -func loadResources(repositoryRoot string, manifestStore syncmanifest.Store) ([]Resource, syncmanifest.Manifest, bool, error) { - manifest, manifestExists, err := manifestStore.Load() +func loadResources( + repositoryRoot string, + manifestStore ManifestStore, + projectKeys []string, +) ([]Resource, syncmanifest.Manifest, error) { + manifest, err := manifestStore.Load(projectKeys) if err != nil { - return nil, syncmanifest.Manifest{}, false, err + return nil, syncmanifest.Manifest{}, err } resources := make(map[Resource]struct{}, len(manifest.Resources)) @@ -90,7 +100,7 @@ func loadResources(repositoryRoot string, manifestStore syncmanifest.Store) ([]R files, err := synclocal.SourceFiles(repositoryRoot) if err != nil { - return nil, syncmanifest.Manifest{}, false, err + return nil, syncmanifest.Manifest{}, err } for _, file := range files { resource, ok := resourceFromWrapperPath(file) @@ -104,7 +114,7 @@ func loadResources(repositoryRoot string, manifestStore syncmanifest.Store) ([]R result = append(result, resource) } slices.SortFunc(result, syncdomain.CompareResourceIDs) - return result, manifest, manifestExists, nil + return result, manifest, nil } // resourceFromWrapperPath derives a variation identity without parsing its contents. @@ -121,7 +131,7 @@ func resourceFromWrapperPath(file string) (Resource, bool) { } // detachResources removes selected resources from the manifest before deleting local wrappers. -func detachResources(options Options, original syncmanifest.Manifest, manifestExists bool, selected []Resource) error { +func detachResources(options Options, original syncmanifest.Manifest, selected []Resource) error { selectedSet := make(map[Resource]struct{}, len(selected)) for _, resource := range selected { selectedSet[resource] = struct{}{} @@ -149,7 +159,8 @@ func detachResources(options Options, original syncmanifest.Manifest, manifestEx } updated.RemoveUnreferencedAttachments(referenced) } - if err := options.Manifest.Write(updated); err != nil { + persisted, err := options.Manifest.Update(original, updated) + if err != nil { return err } @@ -164,7 +175,7 @@ func detachResources(options Options, original syncmanifest.Manifest, manifestEx } exists, err := options.Store.VariationExists(resource.ProjectKey, configKey, variationKey) if err != nil { - return errors.Join(err, restoreManifest(options.Manifest, original, manifestExists)) + return errors.Join(err, restoreManifest(options.Manifest, persisted, original)) } if exists { deletions = append(deletions, synclocal.VariationDeletion{ @@ -174,7 +185,7 @@ func detachResources(options Options, original syncmanifest.Manifest, manifestEx } if _, err := options.Store.DeleteVariations(deletions); err != nil { - return errors.Join(err, restoreManifest(options.Manifest, original, manifestExists)) + return errors.Join(err, restoreManifest(options.Manifest, persisted, original)) } if err := options.Store.RemoveEmptyDirectories(); err != nil { return err @@ -183,9 +194,7 @@ func detachResources(options Options, original syncmanifest.Manifest, manifestEx } // restoreManifest restores the manifest when local wrapper deletion fails. -func restoreManifest(store syncmanifest.Store, manifest syncmanifest.Manifest, existed bool) error { - if existed { - return store.Write(manifest) - } - return store.Remove() +func restoreManifest(store ManifestStore, current, original syncmanifest.Manifest) error { + _, err := store.Update(current, original) + return err } diff --git a/internal/sync/detach/detach_test.go b/internal/sync/detach/detach_test.go index b17ca002..6d5ec356 100644 --- a/internal/sync/detach/detach_test.go +++ b/internal/sync/detach/detach_test.go @@ -24,9 +24,8 @@ func TestLoadResourcesUnionsLocalAndManifestResources(t *testing.T) { }}) require.NoError(t, err) - manifestStore := syncmanifest.NewStore(root) + manifestStore := newMemoryManifestStore() require.NoError(t, manifestStore.Write(syncmanifest.Manifest{ - FormatVersion: syncmanifest.FormatVersion, Resources: []syncmanifest.Resource{ { ResourceKind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/manifest-only", @@ -39,10 +38,9 @@ func TestLoadResourcesUnionsLocalAndManifestResources(t *testing.T) { }, })) - resources, _, exists, err := loadResources(root, manifestStore) + resources, _, err := loadResources(root, manifestStore, []string{"project"}) require.NoError(t, err) - require.True(t, exists) assert.Equal(t, []Resource{ {Kind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/local"}, {Kind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/manifest-only"}, @@ -62,9 +60,8 @@ func TestDetachResourcesPrunesUnreferencedAttachmentManifestEntries(t *testing.T _, err := store.Add([]synclocal.VariationFile{{ProjectKey: "project", ConfigKey: "config", Variation: variation}}) require.NoError(t, err) - manifestStore := syncmanifest.NewStore(root) + manifestStore := newMemoryManifestStore() original := syncmanifest.Manifest{ - FormatVersion: syncmanifest.FormatVersion, Resources: []syncmanifest.Resource{ { ResourceKind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/prompt", @@ -82,12 +79,11 @@ func TestDetachResourcesPrunesUnreferencedAttachmentManifestEntries(t *testing.T err = detachResources( Options{RepositoryRoot: root, Store: store, Manifest: manifestStore}, original, - true, []Resource{resource}, ) require.NoError(t, err) - manifest, _, err := manifestStore.Load() + manifest, err := manifestStore.Load([]string{"project"}) require.NoError(t, err) assert.Empty(t, manifest.Resources) } @@ -106,9 +102,8 @@ func TestDetachResourcesRemovesWrapperAndManifestButKeepsReferencedFile(t *testi }}) require.NoError(t, err) - manifestStore := syncmanifest.NewStore(root) + manifestStore := newMemoryManifestStore() original := syncmanifest.Manifest{ - FormatVersion: syncmanifest.FormatVersion, Resources: []syncmanifest.Resource{{ ResourceKind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/prompt", Fingerprint: testFingerprint(), @@ -117,7 +112,7 @@ func TestDetachResourcesRemovesWrapperAndManifestButKeepsReferencedFile(t *testi require.NoError(t, manifestStore.Write(original)) resource := Resource{Kind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/prompt"} - err = detachResources(Options{Store: store, Manifest: manifestStore}, original, true, []Resource{resource}) + err = detachResources(Options{Store: store, Manifest: manifestStore}, original, []Resource{resource}) require.NoError(t, err) exists, err := store.VariationExists("project", "config", "prompt") @@ -125,18 +120,16 @@ func TestDetachResourcesRemovesWrapperAndManifestButKeepsReferencedFile(t *testi assert.False(t, exists) _, err = os.Stat(referencePath) require.NoError(t, err) - manifest, exists, err := manifestStore.Load() + manifest, err := manifestStore.Load([]string{"project"}) require.NoError(t, err) - require.True(t, exists) assert.Empty(t, manifest.Resources) } func TestDetachResourcesRemovesManifestEntryWhenWrapperWasAlreadyDeleted(t *testing.T) { root := t.TempDir() store := synclocal.NewStore(root) - manifestStore := syncmanifest.NewStore(root) + manifestStore := newMemoryManifestStore() original := syncmanifest.Manifest{ - FormatVersion: syncmanifest.FormatVersion, Resources: []syncmanifest.Resource{{ ResourceKind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/deleted", Fingerprint: testFingerprint(), @@ -145,12 +138,11 @@ func TestDetachResourcesRemovesManifestEntryWhenWrapperWasAlreadyDeleted(t *test require.NoError(t, manifestStore.Write(original)) resource := Resource{Kind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/deleted"} - err := detachResources(Options{Store: store, Manifest: manifestStore}, original, true, []Resource{resource}) + err := detachResources(Options{Store: store, Manifest: manifestStore}, original, []Resource{resource}) require.NoError(t, err) - manifest, exists, err := manifestStore.Load() + manifest, err := manifestStore.Load([]string{"project"}) require.NoError(t, err) - require.True(t, exists) assert.Empty(t, manifest.Resources) } @@ -161,20 +153,18 @@ func TestDetachResourcesDeletesUnreadableWrapper(t *testing.T) { require.NoError(t, os.WriteFile(wrapper, []byte("not front matter"), 0o644)) store := synclocal.NewStore(root) - manifestStore := syncmanifest.NewStore(root) + manifestStore := newMemoryManifestStore() resource := Resource{Kind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/broken"} err := detachResources( Options{Store: store, Manifest: manifestStore}, syncmanifest.New(), - false, []Resource{resource}, ) require.NoError(t, err) - manifest, exists, loadErr := manifestStore.Load() + manifest, loadErr := manifestStore.Load([]string{"project"}) require.NoError(t, loadErr) - require.True(t, exists) assert.Empty(t, manifest.Resources) _, statErr := os.Stat(wrapper) require.ErrorIs(t, statErr, os.ErrNotExist) @@ -191,7 +181,7 @@ func TestRunRequiresTerminalWhenResourcesExist(t *testing.T) { err = Run(Options{ RepositoryRoot: root, Store: store, - Manifest: syncmanifest.NewStore(root), + Manifest: newMemoryManifestStore(), Input: bytes.NewBuffer(nil), Output: bytes.NewBuffer(nil), }) @@ -206,7 +196,7 @@ func TestRunReportsWhenNoResourcesAreSynced(t *testing.T) { err := Run(Options{ RepositoryRoot: root, Store: synclocal.NewStore(root), - Manifest: syncmanifest.NewStore(root), + Manifest: newMemoryManifestStore(), Input: bytes.NewBuffer(nil), Output: &output, }) @@ -215,6 +205,31 @@ func TestRunReportsWhenNoResourcesAreSynced(t *testing.T) { assert.Equal(t, "No resources are currently synced.\n", output.String()) } +type memoryManifestStore struct { + manifest syncmanifest.Manifest +} + +func newMemoryManifestStore() *memoryManifestStore { + return &memoryManifestStore{manifest: syncmanifest.New()} +} + +func (store *memoryManifestStore) Load([]string) (syncmanifest.Manifest, error) { + return store.manifest, nil +} + +func (store *memoryManifestStore) Update( + _ syncmanifest.Manifest, + next syncmanifest.Manifest, +) (syncmanifest.Manifest, error) { + store.manifest = next + return next, nil +} + +func (store *memoryManifestStore) Write(manifest syncmanifest.Manifest) error { + store.manifest = manifest + return nil +} + func testVariation(key string) syncdomain.Variation { return syncdomain.Variation{Mode: syncdomain.VariationModeAgent, Key: key, Name: key, Instructions: "Help."} } diff --git a/internal/sync/local/attachment.go b/internal/sync/local/attachment.go index 76a89ae9..d4c4a878 100644 --- a/internal/sync/local/attachment.go +++ b/internal/sync/local/attachment.go @@ -296,6 +296,10 @@ func (store Store) AttachVariation(projectKey, configKey string, variation syncd // referenced by any local variation. func (store Store) OrphanedAttachments() ([]OrphanedAttachment, error) { resources, err := CompileWorkspace(store.repositoryRoot) + if errors.Is(err, ErrNoDirectory) { + resources = nil + err = nil + } if err != nil { return nil, err } diff --git a/internal/sync/manifest/model.go b/internal/sync/manifest/model.go index d46b17ab..85a493c5 100644 --- a/internal/sync/manifest/model.go +++ b/internal/sync/manifest/model.go @@ -9,25 +9,22 @@ import ( syncdomain "github.com/launchdarkly/ldcli/internal/sync" ) -// FormatVersion is the current manifest schema version. -const FormatVersion = 1 - var fingerprintPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) // Manifest records the common resource state accepted by the last successful // synchronization. type Manifest struct { - FormatVersion int `yaml:"formatVersion"` - Resources []Resource `yaml:"resources"` + Resources []Resource } // Resource identifies one tracked resource and its last synchronized // fingerprint. type Resource struct { - ResourceKind syncdomain.Kind `yaml:"resourceKind"` - ProjectKey string `yaml:"projectKey"` - LookupKey string `yaml:"lookupKey"` - Fingerprint string `yaml:"fingerprint"` + ResourceKind syncdomain.Kind + ProjectKey string + LookupKey string + Fingerprint string + Version int } // ID returns the common identity represented by this manifest entry. @@ -37,7 +34,7 @@ func (resource Resource) ID() syncdomain.ResourceID { // New returns an empty current-version manifest. func New() Manifest { - return Manifest{FormatVersion: FormatVersion, Resources: []Resource{}} + return Manifest{Resources: []Resource{}} } // SetFingerprint records the last synchronized state for one resource. @@ -117,10 +114,6 @@ func (manifest *Manifest) Remove(id syncdomain.ResourceID) { // Validate checks the manifest schema and resource identities. func (manifest Manifest) Validate() error { - if manifest.FormatVersion != FormatVersion { - return fmt.Errorf("unsupported manifest formatVersion %d", manifest.FormatVersion) - } - seen := make(map[syncdomain.ResourceID]struct{}, len(manifest.Resources)) for _, resource := range manifest.Resources { if err := validatePathSegment("resource kind", string(resource.ResourceKind)); err != nil { @@ -135,6 +128,9 @@ func (manifest Manifest) Validate() error { if !fingerprintPattern.MatchString(resource.Fingerprint) { return fmt.Errorf("invalid fingerprint for %s/%s", resource.ProjectKey, resource.LookupKey) } + if resource.Version < 0 { + return fmt.Errorf("invalid version for %s/%s", resource.ProjectKey, resource.LookupKey) + } identity := resource.ID() if _, exists := seen[identity]; exists { diff --git a/internal/sync/manifest/store.go b/internal/sync/manifest/store.go index 8e1c97ee..3d62e5c7 100644 --- a/internal/sync/manifest/store.go +++ b/internal/sync/manifest/store.go @@ -1,129 +1,298 @@ package manifest import ( - "bytes" "errors" "fmt" - "io" - "os" - "path/filepath" + "slices" + "strings" - syncdomain "github.com/launchdarkly/ldcli/internal/sync" - "gopkg.in/yaml.v3" + syncapi "github.com/launchdarkly/ldcli/internal/sync/api" ) -// FileName is the repository-local sync manifest filename. -const FileName = "manifest.yaml" +const manifestBatchLimit = 100 -// Store reads and atomically writes the committed synchronization manifest. +type client interface { + GetSyncManifest(projectKey, source string) (syncapi.SyncManifest, error) + PatchSyncManifest( + projectKey string, + source string, + upserts []syncapi.SyncManifestUpsert, + deletions []syncapi.SyncManifestDeletion, + ) (syncapi.SyncManifest, error) +} + +// Store persists synchronization baselines through LaunchDarkly. type Store struct { - path string + client client + source string } -// NewStore creates a manifest store rooted at the Git repository. -func NewStore(repositoryRoot string) Store { - return Store{path: filepath.Join(repositoryRoot, syncdomain.RootDir, FileName)} +// NewStore creates a remote manifest store for one repository source. +func NewStore(client client, source string) Store { + return Store{client: client, source: source} } -// Load returns the manifest and whether it already exists. -func (store Store) Load() (Manifest, bool, error) { - data, err := os.ReadFile(store.path) - if errors.Is(err, os.ErrNotExist) { - return New(), false, nil - } - if err != nil { - return Manifest{}, false, fmt.Errorf("read sync manifest: %w", err) - } +// Load combines the project-scoped remote manifests used by the workspace. +func (store Store) Load(projectKeys []string) (Manifest, error) { + projectKeys = append([]string(nil), projectKeys...) + slices.Sort(projectKeys) + projectKeys = slices.Compact(projectKeys) - decoder := yaml.NewDecoder(bytes.NewReader(data)) - // A committed manifest is an API between CLI versions. Reject unknown - // fields instead of silently discarding data written by a newer schema. - decoder.KnownFields(true) - var manifest Manifest - if err := decoder.Decode(&manifest); err != nil { - return Manifest{}, false, fmt.Errorf("decode sync manifest: %w", err) - } - var trailing any - if err := decoder.Decode(&trailing); err != io.EOF { - if err == nil { - err = fmt.Errorf("multiple YAML documents are not supported") + manifest := New() + for _, projectKey := range projectKeys { + remote, err := store.client.GetSyncManifest(projectKey, store.source) + if err != nil { + return Manifest{}, err + } + resources, err := resourcesFromRemote(projectKey, store.source, remote) + if err != nil { + return Manifest{}, err } - return Manifest{}, false, fmt.Errorf("decode sync manifest: %w", err) + manifest.Resources = append(manifest.Resources, resources...) } + manifest.Sort() if err := manifest.Validate(); err != nil { - return Manifest{}, false, fmt.Errorf("validate sync manifest: %w", err) + return Manifest{}, fmt.Errorf("validate sync manifest: %w", err) } - manifest.Sort() - return manifest, true, nil + return manifest, nil } -// Write atomically replaces the manifest with deterministic YAML. -func (store Store) Write(manifest Manifest) error { - manifest.FormatVersion = FormatVersion - if manifest.Resources == nil { - manifest.Resources = []Resource{} +// Update applies the difference between two aggregate manifest states. +func (store Store) Update(previous, next Manifest) (Manifest, error) { + if err := previous.Validate(); err != nil { + return Manifest{}, fmt.Errorf("validate previous sync manifest: %w", err) } - manifest.Sort() - if err := manifest.Validate(); err != nil { - return fmt.Errorf("validate sync manifest: %w", err) + for _, resource := range previous.Resources { + if resource.Version == 0 { + return Manifest{}, fmt.Errorf( + "sync manifest resource %s/%s is missing its remote version", + resource.ProjectKey, + resource.LookupKey, + ) + } + } + if err := next.Validate(); err != nil { + return Manifest{}, fmt.Errorf("validate updated sync manifest: %w", err) + } + + result := New() + for _, projectKey := range manifestProjects(previous, next) { + before := projectResources(previous, projectKey) + after := projectResources(next, projectKey) + upserts, deletions := manifestChanges(before, after) + + if len(upserts) == 0 && len(deletions) == 0 { + result.Resources = append(result.Resources, retainVersions(before, after)...) + continue + } + + remote, err := store.apply(projectKey, upserts, deletions) + if err != nil { + return Manifest{}, err + } + resources, err := resourcesFromRemote(projectKey, store.source, remote) + if err != nil { + return Manifest{}, err + } + result.Resources = append(result.Resources, resources...) } - var data bytes.Buffer - encoder := yaml.NewEncoder(&data) - encoder.SetIndent(2) - if err := encoder.Encode(manifest); err != nil { - return fmt.Errorf("encode sync manifest: %w", err) + result.Sort() + return result, nil +} + +func (store Store) apply( + projectKey string, + upserts []syncapi.SyncManifestUpsert, + deletions []syncapi.SyncManifestDeletion, +) (syncapi.SyncManifest, error) { + var latest syncapi.SyncManifest + for len(upserts) != 0 || len(deletions) != 0 { + upsertCount := min(len(upserts), manifestBatchLimit) + deletionCount := min(len(deletions), manifestBatchLimit) + batchUpserts := upserts[:upsertCount] + batchDeletions := deletions[:deletionCount] + + remote, err := store.client.PatchSyncManifest( + projectKey, + store.source, + batchUpserts, + batchDeletions, + ) + if err != nil { + if syncapi.IsConflict(err) { + return syncapi.SyncManifest{}, fmt.Errorf( + "sync manifest for project %q changed in LaunchDarkly; run sync again: %w", + projectKey, + err, + ) + } + if !syncapi.MutationMayHaveSucceeded(err) { + return syncapi.SyncManifest{}, err + } + + remote, readErr := store.client.GetSyncManifest(projectKey, store.source) + if readErr != nil || !changesApplied(remote, batchUpserts, batchDeletions) { + return syncapi.SyncManifest{}, errors.Join(err, readErr) + } + } + + latest = remote + upserts = upserts[upsertCount:] + deletions = deletions[deletionCount:] } - if err := encoder.Close(); err != nil { - return fmt.Errorf("encode sync manifest: %w", err) + return latest, nil +} + +func resourcesFromRemote(projectKey, source string, remote syncapi.SyncManifest) ([]Resource, error) { + if remote.Source != "" && remote.Source != source { + return nil, fmt.Errorf( + "sync manifest for project %q returned source %q instead of %q", + projectKey, + remote.Source, + source, + ) } - directory := filepath.Dir(store.path) - if err := os.MkdirAll(directory, 0o755); err != nil { - return fmt.Errorf("create sync directory: %w", err) + resources := make([]Resource, 0, len(remote.Items)) + for _, item := range remote.Items { + if item.Version < 1 { + return nil, fmt.Errorf( + "sync manifest resource %s/%s returned invalid version %d", + projectKey, + item.ResourceLookupKey, + item.Version, + ) + } + resources = append(resources, Resource{ + ResourceKind: item.ResourceKind, + ProjectKey: projectKey, + LookupKey: item.ResourceLookupKey, + Fingerprint: item.Fingerprint, + Version: item.Version, + }) } - temporary, err := os.CreateTemp(directory, ".manifest-*.yaml") - if err != nil { - return fmt.Errorf("create temporary sync manifest: %w", err) + return resources, nil +} + +func manifestProjects(manifests ...Manifest) []string { + var projectKeys []string + for _, manifest := range manifests { + for _, resource := range manifest.Resources { + projectKeys = append(projectKeys, resource.ProjectKey) + } } - temporaryPath := temporary.Name() - defer func() { _ = os.Remove(temporaryPath) }() + slices.Sort(projectKeys) + return slices.Compact(projectKeys) +} - // Flush and close the complete temporary file before the single rename - // commit point, so readers observe either the old or the new manifest. - if err := temporary.Chmod(0o644); err != nil { - _ = temporary.Close() - return fmt.Errorf("set sync manifest permissions: %w", err) +func projectResources(manifest Manifest, projectKey string) []Resource { + var resources []Resource + for _, resource := range manifest.Resources { + if resource.ProjectKey == projectKey { + resources = append(resources, resource) + } } - if _, err := temporary.Write(data.Bytes()); err != nil { - _ = temporary.Close() - return fmt.Errorf("write sync manifest: %w", err) + return resources +} + +func manifestChanges( + previous []Resource, + next []Resource, +) ([]syncapi.SyncManifestUpsert, []syncapi.SyncManifestDeletion) { + before := make(map[identity]Resource, len(previous)) + for _, resource := range previous { + before[resourceIdentity(resource)] = resource } - if err := temporary.Sync(); err != nil { - _ = temporary.Close() - return fmt.Errorf("sync manifest contents: %w", err) + + var upserts []syncapi.SyncManifestUpsert + remaining := make(map[identity]Resource, len(previous)) + for id, resource := range before { + remaining[id] = resource } - if err := temporary.Close(); err != nil { - return fmt.Errorf("close sync manifest: %w", err) + + for _, resource := range next { + id := resourceIdentity(resource) + current, exists := before[id] + delete(remaining, id) + if exists && current.Fingerprint == resource.Fingerprint { + continue + } + version := 0 + if exists { + version = current.Version + } + upserts = append(upserts, syncapi.SyncManifestUpsert{ + ResourceKind: resource.ResourceKind, + ResourceLookupKey: resource.LookupKey, + Fingerprint: resource.Fingerprint, + Version: version, + }) } - if err := os.Rename(temporaryPath, store.path); err != nil { - return fmt.Errorf("replace sync manifest: %w", err) + + var deletions []syncapi.SyncManifestDeletion + for _, resource := range remaining { + deletions = append(deletions, syncapi.SyncManifestDeletion{ + ResourceKind: resource.ResourceKind, + ResourceLookupKey: resource.LookupKey, + Version: resource.Version, + }) } + slices.SortFunc(upserts, func(left, right syncapi.SyncManifestUpsert) int { + if left.ResourceKind != right.ResourceKind { + return strings.Compare(string(left.ResourceKind), string(right.ResourceKind)) + } + return strings.Compare(left.ResourceLookupKey, right.ResourceLookupKey) + }) + slices.SortFunc(deletions, func(left, right syncapi.SyncManifestDeletion) int { + if left.ResourceKind != right.ResourceKind { + return strings.Compare(string(left.ResourceKind), string(right.ResourceKind)) + } + return strings.Compare(left.ResourceLookupKey, right.ResourceLookupKey) + }) + return upserts, deletions +} - // The rename above is the commit point. Directory syncing improves crash - // durability where the platform supports it, but must not turn a committed - // replacement into a reported failure. - if directoryHandle, err := os.Open(directory); err == nil { - _ = directoryHandle.Sync() - _ = directoryHandle.Close() +type identity struct { + kind string + lookupKey string +} + +func resourceIdentity(resource Resource) identity { + return identity{kind: string(resource.ResourceKind), lookupKey: resource.LookupKey} +} + +func retainVersions(previous, next []Resource) []Resource { + versions := make(map[identity]int, len(previous)) + for _, resource := range previous { + versions[resourceIdentity(resource)] = resource.Version + } + result := append([]Resource(nil), next...) + for index := range result { + result[index].Version = versions[resourceIdentity(result[index])] } - return nil + return result } -// Remove deletes the manifest when rolling back creation of a new manifest. -func (store Store) Remove() error { - if err := os.Remove(store.path); err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("remove sync manifest: %w", err) +func changesApplied( + remote syncapi.SyncManifest, + upserts []syncapi.SyncManifestUpsert, + deletions []syncapi.SyncManifestDeletion, +) bool { + current := make(map[identity]syncapi.SyncManifestResource, len(remote.Items)) + for _, item := range remote.Items { + current[identity{kind: string(item.ResourceKind), lookupKey: item.ResourceLookupKey}] = item + } + for _, upsert := range upserts { + item, ok := current[identity{kind: string(upsert.ResourceKind), lookupKey: upsert.ResourceLookupKey}] + if !ok || item.Fingerprint != upsert.Fingerprint || item.Version <= upsert.Version { + return false + } + } + for _, deletion := range deletions { + if _, ok := current[identity{kind: string(deletion.ResourceKind), lookupKey: deletion.ResourceLookupKey}]; ok { + return false + } } - return nil + return true } diff --git a/internal/sync/manifest/store_test.go b/internal/sync/manifest/store_test.go index 394c8ef0..97d153dd 100644 --- a/internal/sync/manifest/store_test.go +++ b/internal/sync/manifest/store_test.go @@ -1,76 +1,126 @@ package manifest import ( - "os" - "path/filepath" + "errors" "strings" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" syncdomain "github.com/launchdarkly/ldcli/internal/sync" + syncapi "github.com/launchdarkly/ldcli/internal/sync/api" ) -func TestStoreRoundTripIsDeterministic(t *testing.T) { - root := t.TempDir() - store := NewStore(root) - input := Manifest{ - Resources: []Resource{ - {ResourceKind: syncdomain.KindVariation, ProjectKey: "zeta", LookupKey: "config/b", Fingerprint: fingerprint("b")}, - {ResourceKind: syncdomain.KindVariation, ProjectKey: "alpha", LookupKey: "config/a", Fingerprint: fingerprint("a")}, - }, +type patchCall struct { + projectKey string + upserts []syncapi.SyncManifestUpsert + deletions []syncapi.SyncManifestDeletion +} + +type manifestClient struct { + manifests map[string]syncapi.SyncManifest + patchResponses []syncapi.SyncManifest + patchErr error + patches []patchCall +} + +func (client *manifestClient) GetSyncManifest(projectKey, _ string) (syncapi.SyncManifest, error) { + return client.manifests[projectKey], nil +} + +func (client *manifestClient) PatchSyncManifest( + projectKey string, + _ string, + upserts []syncapi.SyncManifestUpsert, + deletions []syncapi.SyncManifestDeletion, +) (syncapi.SyncManifest, error) { + client.patches = append(client.patches, patchCall{ + projectKey: projectKey, + upserts: append([]syncapi.SyncManifestUpsert(nil), upserts...), + deletions: append([]syncapi.SyncManifestDeletion(nil), deletions...), + }) + if client.patchErr != nil { + return syncapi.SyncManifest{}, client.patchErr } + response := client.patchResponses[0] + client.patchResponses = client.patchResponses[1:] + return response, nil +} + +func TestStoreLoadsProjectManifests(t *testing.T) { + client := &manifestClient{manifests: map[string]syncapi.SyncManifest{ + "zeta": { + Source: "git:example/repo", + Items: []syncapi.SyncManifestResource{{ + ResourceKind: syncdomain.KindVariation, ResourceLookupKey: "config/b", + Fingerprint: fingerprint("b"), Version: 4, + }}, + }, + "alpha": { + Source: "git:example/repo", + Items: []syncapi.SyncManifestResource{{ + ResourceKind: syncdomain.KindVariation, ResourceLookupKey: "config/a", + Fingerprint: fingerprint("a"), Version: 2, + }}, + }, + }} + + loaded, err := NewStore(client, "git:example/repo").Load([]string{"zeta", "alpha", "alpha"}) - require.NoError(t, store.Write(input)) - data, err := os.ReadFile(filepath.Join(root, syncdomain.RootDir, FileName)) - require.NoError(t, err) - require.Equal(t, `formatVersion: 1 -resources: - - resourceKind: variation - projectKey: alpha - lookupKey: config/a - fingerprint: `+fingerprint("a")+` - - resourceKind: variation - projectKey: zeta - lookupKey: config/b - fingerprint: `+fingerprint("b")+` -`, string(data)) - - loaded, exists, err := store.Load() require.NoError(t, err) - require.True(t, exists) require.Equal(t, []Resource{ - {ResourceKind: syncdomain.KindVariation, ProjectKey: "alpha", LookupKey: "config/a", Fingerprint: fingerprint("a")}, - {ResourceKind: syncdomain.KindVariation, ProjectKey: "zeta", LookupKey: "config/b", Fingerprint: fingerprint("b")}, + {ResourceKind: syncdomain.KindVariation, ProjectKey: "alpha", LookupKey: "config/a", Fingerprint: fingerprint("a"), Version: 2}, + {ResourceKind: syncdomain.KindVariation, ProjectKey: "zeta", LookupKey: "config/b", Fingerprint: fingerprint("b"), Version: 4}, }, loaded.Resources) } -func TestStoreLoadsMissingManifestAsEmpty(t *testing.T) { - loaded, exists, err := NewStore(t.TempDir()).Load() +func TestStoreUpdatesWithRemoteVersions(t *testing.T) { + client := &manifestClient{patchResponses: []syncapi.SyncManifest{{ + Source: "git:example/repo", + Items: []syncapi.SyncManifestResource{ + {ResourceKind: syncdomain.KindVariation, ResourceLookupKey: "config/existing", Fingerprint: fingerprint("c"), Version: 4}, + {ResourceKind: syncdomain.KindTool, ResourceLookupKey: "search", Fingerprint: fingerprint("d"), Version: 1}, + }, + }}} + store := NewStore(client, "git:example/repo") + previous := Manifest{Resources: []Resource{ + {ResourceKind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/existing", Fingerprint: fingerprint("a"), Version: 3}, + {ResourceKind: syncdomain.KindSkill, ProjectKey: "project", LookupKey: "old", Fingerprint: fingerprint("b"), Version: 2}, + }} + next := Manifest{Resources: []Resource{ + {ResourceKind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/existing", Fingerprint: fingerprint("c"), Version: 3}, + {ResourceKind: syncdomain.KindTool, ProjectKey: "project", LookupKey: "search", Fingerprint: fingerprint("d")}, + }} + + updated, err := store.Update(previous, next) require.NoError(t, err) - require.False(t, exists) - require.Equal(t, New(), loaded) + require.Len(t, client.patches, 1) + assert.Equal(t, []syncapi.SyncManifestUpsert{ + {ResourceKind: syncdomain.KindTool, ResourceLookupKey: "search", Fingerprint: fingerprint("d"), Version: 0}, + {ResourceKind: syncdomain.KindVariation, ResourceLookupKey: "config/existing", Fingerprint: fingerprint("c"), Version: 3}, + }, client.patches[0].upserts) + assert.Equal(t, []syncapi.SyncManifestDeletion{{ + ResourceKind: syncdomain.KindSkill, ResourceLookupKey: "old", Version: 2, + }}, client.patches[0].deletions) + assert.Equal(t, 4, updated.Resources[1].Version) } -func TestStoreRejectsUnknownFields(t *testing.T) { - root := t.TempDir() - path := filepath.Join(root, syncdomain.RootDir, FileName) - require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) - require.NoError(t, os.WriteFile(path, []byte("formatVersion: 1\nunknown: true\nresources: []\n"), 0o644)) - - _, _, err := NewStore(root).Load() - require.ErrorContains(t, err, "field unknown not found") -} +func TestStoreReportsOptimisticConflict(t *testing.T) { + client := &manifestClient{patchErr: errors.New(`{"code":"conflict","statusCode":409}`)} + store := NewStore(client, "git:example/repo") + previous := Manifest{Resources: []Resource{{ + ResourceKind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/variation", + Fingerprint: fingerprint("a"), Version: 1, + }}} + next := previous + next.Resources = append([]Resource(nil), previous.Resources...) + next.Resources[0].Fingerprint = fingerprint("b") -func TestStoreRejectsMultipleYAMLDocuments(t *testing.T) { - root := t.TempDir() - path := filepath.Join(root, syncdomain.RootDir, FileName) - require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) - require.NoError(t, os.WriteFile(path, []byte("formatVersion: 1\nresources: []\n---\nformatVersion: 1\nresources: []\n"), 0o644)) + _, err := store.Update(previous, next) - _, _, err := NewStore(root).Load() - require.ErrorContains(t, err, "multiple YAML documents") + require.ErrorContains(t, err, `sync manifest for project "project" changed in LaunchDarkly`) } func TestManifestValidation(t *testing.T) { @@ -85,10 +135,6 @@ func TestManifestValidation(t *testing.T) { mutate func(*Manifest) error string }{ - "format": { - mutate: func(manifest *Manifest) { manifest.FormatVersion = 2 }, - error: "unsupported manifest formatVersion", - }, "kind": { mutate: func(manifest *Manifest) { manifest.Resources[0].ResourceKind = "" }, error: "invalid resource kind", @@ -117,7 +163,7 @@ func TestManifestValidation(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - manifest := Manifest{FormatVersion: FormatVersion, Resources: []Resource{valid}} + manifest := Manifest{Resources: []Resource{valid}} test.mutate(&manifest) require.ErrorContains(t, manifest.Validate(), test.error) }) @@ -126,7 +172,6 @@ func TestManifestValidation(t *testing.T) { func TestManifestSupportsDifferentResourceIdentities(t *testing.T) { manifest := Manifest{ - FormatVersion: FormatVersion, Resources: []Resource{ {ResourceKind: "tool", ProjectKey: "project", LookupKey: "weather", Fingerprint: fingerprint("a")}, {ResourceKind: "skill", ProjectKey: "project", LookupKey: "support/summarize/v2", Fingerprint: fingerprint("b")}, diff --git a/internal/sync/prompt/acceptance_test.go b/internal/sync/prompt/acceptance_test.go index d046848b..be22bce1 100644 --- a/internal/sync/prompt/acceptance_test.go +++ b/internal/sync/prompt/acceptance_test.go @@ -2,12 +2,14 @@ package prompt_test import ( "encoding/json" + "errors" "fmt" "net/http" "net/url" "os" "os/exec" "path/filepath" + "slices" "strings" "testing" @@ -30,6 +32,7 @@ type directAPI struct { modelConfigs []syncapi.ModelConfig tools map[string]versionedTool skills map[string]versionedSkill + manifest *syncapi.SyncManifest requests []string } @@ -55,11 +58,14 @@ func (api *directAPI) MakeRequest( method string, path string, _ string, - _ url.Values, + query url.Values, body []byte, _ bool, ) ([]byte, error) { api.requests = append(api.requests, method+" "+path) + if strings.HasSuffix(path, "/configs/sync/manifests") { + return handleManifestRequest(api.manifest, method, query, body) + } if strings.HasSuffix(path, "/ai-tools") && method == http.MethodPost { var tool versionedTool if err := json.Unmarshal(body, &tool.Tool); err != nil { @@ -200,7 +206,6 @@ func TestPromptDryRunUsesOnlyExistingReadAPI(t *testing.T) { require.NotEmpty(t, api.requests) for _, request := range api.requests { assert.True(t, strings.HasPrefix(request, "GET "), request) - assert.NotContains(t, request, "/sync/") } } @@ -640,6 +645,10 @@ func TestPromptServerDeletionLeavesReferencedFile(t *testing.T) { require.NoError(t, err) assert.Equal(t, "Keep this file\n", string(content)) resources, err := synclocal.CompileWorkspace(root) + if errors.Is(err, synclocal.ErrNoDirectory) { + resources = nil + err = nil + } require.NoError(t, err) assert.Empty(t, resources) } @@ -649,7 +658,11 @@ func TestPromptPropagatesTrackedLocalDeletion(t *testing.T) { baseline := variation("Baseline") writeVariation(t, root, baseline, false) writeManifest(t, root, baseline) - _, err := synclocal.NewStore(root).DeleteVariations([]synclocal.VariationDeletion{{ + command := exec.Command("git", "add", ".launchdarkly") + command.Dir = root + output, err := command.CombinedOutput() + require.NoError(t, err, "%s", output) + _, err = synclocal.NewStore(root).DeleteVariations([]synclocal.VariationDeletion{{ ProjectKey: "production", ConfigKey: "support", VariationKey: "default", }}) require.NoError(t, err) @@ -660,10 +673,7 @@ func TestPromptPropagatesTrackedLocalDeletion(t *testing.T) { require.NoError(t, err) require.NotNil(t, api.variation) assert.Equal(t, "archived", api.variationState) - manifest, exists, err := syncmanifest.NewStore(root).Load() - require.NoError(t, err) - require.True(t, exists) - assert.Empty(t, manifest.Resources) + assert.Empty(t, manifestsByRoot[root].Items) } func TestPromptPropagatesTrackedServerDeletion(t *testing.T) { @@ -677,12 +687,13 @@ func TestPromptPropagatesTrackedServerDeletion(t *testing.T) { require.NoError(t, err) resources, err := synclocal.Compile(os.DirFS(root)) + if errors.Is(err, synclocal.ErrNoDirectory) { + resources = nil + err = nil + } require.NoError(t, err) assert.Empty(t, resources) - manifest, exists, err := syncmanifest.NewStore(root).Load() - require.NoError(t, err) - require.True(t, exists) - assert.Empty(t, manifest.Resources) + assert.Empty(t, manifestsByRoot[root].Items) } func TestPromptRejectsDivergentChangesWithoutMutation(t *testing.T) { @@ -759,7 +770,7 @@ func (api *changingReadAPI) MakeRequest( body []byte, beta bool, ) ([]byte, error) { - if method == "GET" { + if method == "GET" && !strings.Contains(path, "/configs/sync/manifests") { api.reads++ if api.reads == 2 { api.variation = pointer(variation("Concurrent")) @@ -788,11 +799,10 @@ func TestPromptRecordsPartialSuccessAndConvergesOnNextRun(t *testing.T) { require.ErrorContains(t, err, "second") assert.Contains(t, stdout, `"status": "succeeded"`) assert.Contains(t, stdout, `"status": "failed"`) - manifest, _, loadErr := syncmanifest.NewStore(root).Load() - require.NoError(t, loadErr) - require.Len(t, manifest.Resources, 2) - assert.Equal(t, fingerprint(t, firstLocal), manifest.Resources[0].Fingerprint) - assert.Equal(t, fingerprint(t, secondBaseline), manifest.Resources[1].Fingerprint) + manifest := manifestsByRoot[root] + require.Len(t, manifest.Items, 2) + assert.Equal(t, fingerprint(t, firstLocal), manifest.Items[0].Fingerprint) + assert.Equal(t, fingerprint(t, secondBaseline), manifest.Items[1].Fingerprint) api.failKey = "" _, _, err = runPrompt(t, root, api, "--yes") @@ -804,6 +814,7 @@ func TestPromptRecordsPartialSuccessAndConvergesOnNextRun(t *testing.T) { type multiDirectAPI struct { variations map[string]syncdomain.Variation failKey string + manifest *syncapi.SyncManifest requests []string } @@ -812,11 +823,14 @@ func (api *multiDirectAPI) MakeRequest( method string, path string, _ string, - _ url.Values, + query url.Values, body []byte, _ bool, ) ([]byte, error) { api.requests = append(api.requests, method+" "+path) + if strings.HasSuffix(path, "/configs/sync/manifests") { + return handleManifestRequest(api.manifest, method, query, body) + } if method == "GET" { variations := make([]syncdomain.Variation, 0, len(api.variations)) for _, variation := range api.variations { @@ -856,11 +870,32 @@ func (*multiDirectAPI) MakeUnauthenticatedRequest(string, string, []byte) ([]byt return nil, nil } +var manifestsByRoot = map[string]*syncapi.SyncManifest{} + func runPrompt(t *testing.T, root string, client resources.Client, arguments ...string) (string, string, error) { t.Helper() t.Chdir(root) t.Setenv("XDG_CONFIG_HOME", t.TempDir()) - args := []string{"sync", "prompt", "--access-token", "token", "--base-uri", "https://example.test"} + manifest := manifestsByRoot[root] + if manifest == nil { + manifest = &syncapi.SyncManifest{Source: "git:example/repo", Items: []syncapi.SyncManifestResource{}} + manifestsByRoot[root] = manifest + } + switch api := client.(type) { + case *directAPI: + api.manifest = manifest + case *multiDirectAPI: + api.manifest = manifest + case *ambiguousWriteAPI: + api.manifest = manifest + case *changingReadAPI: + api.manifest = manifest + } + args := []string{ + "sync", "prompt", + "--access-token", "token", + "--base-uri", "https://example.test", + } args = append(args, arguments...) stdout, stderr, err := cmd.CallCmdCapturingStderr( t, @@ -871,13 +906,84 @@ func runPrompt(t *testing.T, root string, client resources.Client, arguments ... return string(stdout), string(stderr), err } +func handleManifestRequest( + manifest *syncapi.SyncManifest, + method string, + query url.Values, + body []byte, +) ([]byte, error) { + if method == http.MethodGet { + if manifest.Source == "" { + manifest.Source = query.Get("source") + } + return json.Marshal(manifest) + } + + var request struct { + Source string `json:"source"` + Upserts []syncapi.SyncManifestUpsert `json:"upserts"` + Deletions []syncapi.SyncManifestDeletion `json:"deletions"` + } + if err := json.Unmarshal(body, &request); err != nil { + return nil, err + } + items := make(map[syncdomain.ResourceID]syncapi.SyncManifestResource, len(manifest.Items)) + for _, item := range manifest.Items { + items[syncdomain.ResourceID{Kind: item.ResourceKind, LookupKey: item.ResourceLookupKey}] = item + } + for _, upsert := range request.Upserts { + id := syncdomain.ResourceID{Kind: upsert.ResourceKind, LookupKey: upsert.ResourceLookupKey} + version := 1 + if current, ok := items[id]; ok { + if current.Version != upsert.Version { + return nil, fmt.Errorf(`{"code":"conflict","statusCode":409}`) + } + version = current.Version + 1 + } else if upsert.Version != 0 { + return nil, fmt.Errorf(`{"code":"conflict","statusCode":409}`) + } + items[id] = syncapi.SyncManifestResource{ + ResourceKind: upsert.ResourceKind, + ResourceLookupKey: upsert.ResourceLookupKey, + Fingerprint: upsert.Fingerprint, + Version: version, + } + } + for _, deletion := range request.Deletions { + id := syncdomain.ResourceID{Kind: deletion.ResourceKind, LookupKey: deletion.ResourceLookupKey} + current, ok := items[id] + if !ok || current.Version != deletion.Version { + return nil, fmt.Errorf(`{"code":"conflict","statusCode":409}`) + } + delete(items, id) + } + + manifest.Source = request.Source + manifest.Items = manifest.Items[:0] + for _, item := range items { + manifest.Items = append(manifest.Items, item) + } + slices.SortFunc(manifest.Items, func(left, right syncapi.SyncManifestResource) int { + return syncdomain.CompareResourceIDs( + syncdomain.ResourceID{Kind: left.ResourceKind, LookupKey: left.ResourceLookupKey}, + syncdomain.ResourceID{Kind: right.ResourceKind, LookupKey: right.ResourceLookupKey}, + ) + }) + return json.Marshal(manifest) +} + func initRepository(t *testing.T) string { t.Helper() root := t.TempDir() - command := exec.Command("git", "init", "--quiet") - command.Dir = root - output, err := command.CombinedOutput() - require.NoError(t, err, "%s", output) + for _, args := range [][]string{ + {"init", "--quiet"}, + {"remote", "add", "origin", "git@example:repo.git"}, + } { + command := exec.Command("git", args...) + command.Dir = root + output, err := command.CombinedOutput() + require.NoError(t, err, "%s", output) + } return root } @@ -942,7 +1048,16 @@ func writeManifestResources(t *testing.T, root string, values ...syncdomain.Vari Kind: syncdomain.KindVariation, ProjectKey: "production", LookupKey: "support/" + value.Key, }, fingerprint(t, value)) } - require.NoError(t, syncmanifest.NewStore(root).Write(manifest)) + remote := &syncapi.SyncManifest{Source: "git:example/repo"} + for _, resource := range manifest.Resources { + remote.Items = append(remote.Items, syncapi.SyncManifestResource{ + ResourceKind: resource.ResourceKind, + ResourceLookupKey: resource.LookupKey, + Fingerprint: resource.Fingerprint, + Version: 1, + }) + } + manifestsByRoot[root] = remote } func assertManifestFingerprint(t *testing.T, root string, value syncdomain.Variation) { @@ -951,20 +1066,24 @@ func assertManifestFingerprint(t *testing.T, root string, value syncdomain.Varia func assertManifestFingerprints(t *testing.T, root string, values ...syncdomain.Variation) { t.Helper() - manifest, exists, err := syncmanifest.NewStore(root).Load() - require.NoError(t, err) - require.True(t, exists) + manifest := manifestsByRoot[root] + require.NotNil(t, manifest) variations := make(map[string]string) attachments := make(map[syncdomain.ResourceID]string) - for _, resource := range manifest.Resources { + for _, resource := range manifest.Items { if resource.ResourceKind == syncdomain.KindVariation { - variations[resource.LookupKey] = resource.Fingerprint + variations[resource.ResourceLookupKey] = resource.Fingerprint } else { - attachments[resource.ID()] = resource.Fingerprint + attachments[syncdomain.ResourceID{ + Kind: resource.ResourceKind, + ProjectKey: "production", + LookupKey: resource.ResourceLookupKey, + }] = resource.Fingerprint } } require.Len(t, variations, len(values)) expectedAttachments := make(map[syncdomain.ResourceID]string) + var err error for _, value := range values { assert.Equal(t, fingerprint(t, value), variations["support/"+value.Key]) for _, attachment := range value.Attachments { @@ -993,7 +1112,12 @@ func requireOnlyReads(t *testing.T, requests []string) { t.Helper() require.NotEmpty(t, requests) for _, request := range requests { - require.True(t, strings.HasPrefix(request, "GET "), request) + require.True( + t, + strings.HasPrefix(request, "GET ") || + (strings.HasPrefix(request, "PATCH ") && strings.HasSuffix(request, "/configs/sync/manifests")), + request, + ) } } diff --git a/internal/sync/prompt/conflict_test.go b/internal/sync/prompt/conflict_test.go index 7bc473d7..80c1469f 100644 --- a/internal/sync/prompt/conflict_test.go +++ b/internal/sync/prompt/conflict_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "io" "net/url" + "os/exec" "strings" "testing" @@ -324,7 +325,6 @@ func divergentPlan(t *testing.T) Plan { require.NoError(t, err) manifest := syncmanifest.Manifest{ - FormatVersion: syncmanifest.FormatVersion, Resources: []syncmanifest.Resource{{ ResourceKind: id.Kind, ProjectKey: id.ProjectKey, @@ -337,11 +337,16 @@ func divergentPlan(t *testing.T) Plan { }) } -func writeConflictWorkspace(t *testing.T, root string, baseline, local syncdomain.Variation) (synclocal.Store, syncmanifest.Store) { +func writeConflictWorkspace(t *testing.T, root string, baseline, local syncdomain.Variation) (synclocal.Store, manifestStore) { t.Helper() + command := exec.Command("git", "init", "--quiet") + command.Dir = root + output, err := command.CombinedOutput() + require.NoError(t, err, "%s", output) + localStore := synclocal.NewStore(root) - _, err := localStore.Add([]synclocal.VariationFile{{ + _, err = localStore.Add([]synclocal.VariationFile{{ ProjectKey: "production", ConfigKey: "support", Upsert: true, Variation: local, }}) require.NoError(t, err) @@ -349,16 +354,30 @@ func writeConflictWorkspace(t *testing.T, root string, baseline, local syncdomai id := testResourceID() fingerprint, err := syncdomain.FingerprintVariation(id.ProjectKey, id.LookupKey, baseline) require.NoError(t, err) - manifestStore := syncmanifest.NewStore(root) - require.NoError(t, manifestStore.Write(syncmanifest.Manifest{ - FormatVersion: syncmanifest.FormatVersion, + manifestStore := &memoryManifestStore{manifest: syncmanifest.Manifest{ Resources: []syncmanifest.Resource{{ ResourceKind: id.Kind, ProjectKey: id.ProjectKey, LookupKey: id.LookupKey, Fingerprint: fingerprint, }}, - })) + }} return localStore, manifestStore } +type memoryManifestStore struct { + manifest syncmanifest.Manifest +} + +func (store *memoryManifestStore) Load([]string) (syncmanifest.Manifest, error) { + return store.manifest, nil +} + +func (store *memoryManifestStore) Update( + _ syncmanifest.Manifest, + next syncmanifest.Manifest, +) (syncmanifest.Manifest, error) { + store.manifest = next + return next, nil +} + type conflictAPI struct { variation *syncdomain.Variation tool syncdomain.Attachment diff --git a/internal/sync/prompt/execute.go b/internal/sync/prompt/execute.go index 9da8624b..3a2d12d7 100644 --- a/internal/sync/prompt/execute.go +++ b/internal/sync/prompt/execute.go @@ -393,6 +393,9 @@ func verifyLocalResult(repositoryRoot string, resource PlannedResource) error { // or an empty fingerprint when the resource does not exist. func readLocalFingerprint(repositoryRoot string, id ResourceID) (string, error) { localResources, err := synclocal.CompileWorkspace(repositoryRoot) + if errors.Is(err, synclocal.ErrNoDirectory) { + return "", nil + } if err != nil { return "", err } diff --git a/internal/sync/prompt/execute_test.go b/internal/sync/prompt/execute_test.go index de1738d7..5368195e 100644 --- a/internal/sync/prompt/execute_test.go +++ b/internal/sync/prompt/execute_test.go @@ -18,7 +18,6 @@ import ( func TestExecutePlanDoesNotMutateReviewedManifest(t *testing.T) { id := ResourceID{Kind: syncdomain.KindVariation, ProjectKey: "project", LookupKey: "config/variation"} reviewedManifest := syncmanifest.Manifest{ - FormatVersion: syncmanifest.FormatVersion, Resources: []syncmanifest.Resource{{ ResourceKind: id.Kind, ProjectKey: id.ProjectKey, diff --git a/internal/sync/prompt/plan_test.go b/internal/sync/prompt/plan_test.go index d21c1ca8..ef94d7ed 100644 --- a/internal/sync/prompt/plan_test.go +++ b/internal/sync/prompt/plan_test.go @@ -39,7 +39,6 @@ func TestBuildPlanThreeWayMatrix(t *testing.T) { baselineFingerprint, err := syncdomain.FingerprintVariation(id.ProjectKey, id.LookupKey, baseline) require.NoError(t, err) manifest := syncmanifest.Manifest{ - FormatVersion: syncmanifest.FormatVersion, Resources: []syncmanifest.Resource{{ ResourceKind: id.Kind, ProjectKey: id.ProjectKey, diff --git a/internal/sync/prompt/runner.go b/internal/sync/prompt/runner.go index 1e7b8feb..95e8c144 100644 --- a/internal/sync/prompt/runner.go +++ b/internal/sync/prompt/runner.go @@ -8,6 +8,7 @@ import ( "os" "os/signal" "slices" + "strings" "syscall" "github.com/launchdarkly/ldcli/internal/resources" @@ -21,6 +22,7 @@ import ( synclocal "github.com/launchdarkly/ldcli/internal/sync/local" syncmanifest "github.com/launchdarkly/ldcli/internal/sync/manifest" syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" + syncrepository "github.com/launchdarkly/ldcli/internal/sync/repository" syncsource "github.com/launchdarkly/ldcli/internal/sync/source" ) @@ -56,10 +58,15 @@ type bootstrapRunner func(syncbootstrap.Options) error type detachRunner func(syncdetach.Options) error type linkRunner func(synclink.Options) (string, error) +type manifestStore interface { + Load(projectKeys []string) (syncmanifest.Manifest, error) + Update(previous, next syncmanifest.Manifest) (syncmanifest.Manifest, error) +} + type syncWorkspace struct { root string local synclocal.Store - manifest syncmanifest.Store + manifest manifestStore } type attachmentID struct { @@ -97,18 +104,23 @@ func (runner Runner) Run(options Options) error { if err != nil { return err } + apiClient := syncapi.NewClient(runner.client, options.AccessToken, options.BaseURI) workspace := syncWorkspace{ root: resolvedWorkspace.Root, local: synclocal.NewStore(resolvedWorkspace.Root), - manifest: syncmanifest.NewStore(resolvedWorkspace.Root), + manifest: syncmanifest.NewStore(apiClient, resolvedWorkspace.Source), } - apiClient := syncapi.NewClient(runner.client, options.AccessToken, options.BaseURI) if options.Detach { + projectKeys, err := discoverProjectKeys(workspace.root) + if err != nil { + return err + } return runner.detach(syncdetach.Options{ RepositoryRoot: workspace.root, Store: workspace.local, Manifest: workspace.manifest, + ProjectKeys: projectKeys, Input: options.Input, Output: options.Output, }) @@ -162,7 +174,11 @@ func (runner Runner) Run(options Options) error { } } - if !localDirectoryExists || options.Add { + projectKeys, err := discoverProjectKeys(workspace.root) + if err != nil { + return err + } + if (!localDirectoryExists && len(projectKeys) == 0) || options.Add { if err := runner.bootstrap(syncbootstrap.Options{ Catalog: apiClient, Attachments: apiClient, @@ -233,9 +249,13 @@ func (runner Runner) runWorkspaceSync(options Options, workspace syncWorkspace, } apiClient := syncapi.NewClient(runner.client, options.AccessToken, options.BaseURI) + reviewedProjectKeys, err := discoverProjectKeys(workspace.root) + if err != nil { + return err + } // The manifest is the common ancestor in a three-way comparison between // current local files and current LaunchDarkly state. - baseline, _, err := workspace.manifest.Load() + baseline, err := workspace.manifest.Load(reviewedProjectKeys) if err != nil { return err } @@ -276,7 +296,17 @@ func (runner Runner) runWorkspaceSync(options Options, workspace syncWorkspace, } // Re-read both sides after review so no action uses stale state. - currentManifest, _, err := workspace.manifest.Load() + currentProjectKeys, err := discoverProjectKeys(workspace.root) + if err != nil { + return err + } + if !slices.Equal(reviewedProjectKeys, currentProjectKeys) { + if options.Watch { + return errRefreshWatchPlan + } + return fmt.Errorf("sync projects changed after review; run sync again") + } + currentManifest, err := workspace.manifest.Load(currentProjectKeys) if err != nil { return err } @@ -295,7 +325,7 @@ func (runner Runner) runWorkspaceSync(options Options, workspace syncWorkspace, // potentially stale objects that were rendered during review. currentPlan = applyConflictResolutions(currentPlan, conflictResult.resolutions) outcomes, updatedManifest, executionErr := executePlan(workspace.root, workspace.local, apiClient, currentManifest, currentPlan) - if err := workspace.manifest.Write(updatedManifest); err != nil { + if _, err := workspace.manifest.Update(currentManifest, updatedManifest); err != nil { executionErr = errors.Join(executionErr, err) } if err := workspace.local.RemoveEmptyDirectories(); err != nil { @@ -340,6 +370,10 @@ func validateOptions(options Options) error { // loadWorkspacePlan reads local and server state before building a three-way plan. func loadWorkspacePlan(repositoryRoot string, baseline syncmanifest.Manifest, client syncapi.Client) (Plan, error) { localResources, err := synclocal.CompileWorkspace(repositoryRoot) + if errors.Is(err, synclocal.ErrNoDirectory) { + localResources = nil + err = nil + } if err != nil { return Plan{}, err } @@ -374,6 +408,41 @@ func loadWorkspacePlan(repositoryRoot string, baseline syncmanifest.Manifest, cl return plan, nil } +func discoverProjectKeys(repositoryRoot string) ([]string, error) { + files, err := synclocal.SourceFiles(repositoryRoot) + if err != nil { + return nil, err + } + deleted, err := syncrepository.DeletedPaths(repositoryRoot) + if err != nil { + return nil, err + } + + var projectKeys []string + for _, file := range append(files, deleted...) { + if projectKey, ok := projectKeyFromManagedPath(file); ok { + projectKeys = append(projectKeys, projectKey) + } + } + slices.Sort(projectKeys) + return slices.Compact(projectKeys), nil +} + +func projectKeyFromManagedPath(file string) (string, bool) { + parts := strings.Split(file, "/") + if len(parts) < 4 || parts[0] != syncdomain.RootDir || parts[1] == "" { + return "", false + } + switch { + case len(parts) == 5 && parts[2] == "configs" && strings.HasSuffix(parts[4], ".prompt.md"): + case len(parts) == 4 && parts[2] == "tools" && strings.HasSuffix(parts[3], ".json"): + case len(parts) == 4 && parts[2] == "skills" && strings.HasSuffix(parts[3], ".md"): + default: + return "", false + } + return parts[1], true +} + // attachmentHydrator reads each shared dependency once while building a plan. type attachmentHydrator struct { client syncapi.Client diff --git a/internal/sync/prompt/watch_test.go b/internal/sync/prompt/watch_test.go index 0dfe2fb4..1da1dc21 100644 --- a/internal/sync/prompt/watch_test.go +++ b/internal/sync/prompt/watch_test.go @@ -428,7 +428,7 @@ func TestSourceWatcherRecognizesNewManagedResourceKinds(t *testing.T) { require.True(t, watcher.relevant(fsnotify.Event{Name: resourceFile, Op: fsnotify.Create})) require.False(t, watcher.relevant(fsnotify.Event{ - Name: filepath.Join(managedRoot, "manifest.yaml"), + Name: filepath.Join(managedRoot, "README.md"), Op: fsnotify.Create, })) } diff --git a/internal/sync/repository/git.go b/internal/sync/repository/git.go index 3fa0df4f..b508eeb6 100644 --- a/internal/sync/repository/git.go +++ b/internal/sync/repository/git.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "os/exec" + "slices" "strings" ) @@ -13,7 +14,8 @@ var ErrGitUnavailable = errors.New("git executable is unavailable") // GitRepository identifies a repository discovered through Git. type GitRepository struct { - Root string + Root string + Origin string } type gitRunner interface { @@ -54,6 +56,11 @@ func FindGitRepository(dir string) (GitRepository, bool, error) { return findGitRepository(execGit{}, dir) } +// DeletedPaths returns staged and unstaged deleted paths relative to the repository. +func DeletedPaths(repositoryRoot string) ([]string, error) { + return deletedPaths(execGit{}, repositoryRoot) +} + // findGitRepository contains the injectable repository-discovery workflow used // by the real command and focused tests. func findGitRepository(git gitRunner, dir string) (GitRepository, bool, error) { @@ -72,5 +79,42 @@ func findGitRepository(git gitRunner, dir string) (GitRepository, bool, error) { return GitRepository{}, false, fmt.Errorf("find Git repository: %w", err) } - return GitRepository{Root: root}, true, nil + origin, stderr, err := git.output(root, "config", "--get", "remote.origin.url") + if err != nil { + if stderr == "" { + return GitRepository{}, false, errors.New("Git origin is not configured") + } + return GitRepository{}, false, fmt.Errorf("read Git origin: %s: %w", stderr, err) + } + return GitRepository{Root: root, Origin: origin}, true, nil +} + +func deletedPaths(git gitRunner, repositoryRoot string) ([]string, error) { + var paths []string + commands := [][]string{ + {"diff", "--name-only", "--diff-filter=D", "-z", "--", ".launchdarkly"}, + {"diff", "--cached", "--name-only", "--diff-filter=D", "-z", "--", ".launchdarkly"}, + } + for _, command := range commands { + output, stderr, err := git.output(repositoryRoot, command...) + if err != nil { + if stderr != "" { + return nil, fmt.Errorf("find deleted sync files: %s: %w", stderr, err) + } + return nil, fmt.Errorf("find deleted sync files: %w", err) + } + paths = append(paths, splitNullTerminated(output)...) + } + slices.Sort(paths) + return slices.Compact(paths), nil +} + +func splitNullTerminated(value string) []string { + var values []string + for _, item := range strings.Split(value, "\x00") { + if item != "" { + values = append(values, item) + } + } + return values } diff --git a/internal/sync/repository/git_test.go b/internal/sync/repository/git_test.go index a97df213..f3b829d9 100644 --- a/internal/sync/repository/git_test.go +++ b/internal/sync/repository/git_test.go @@ -11,7 +11,8 @@ func TestFindGitRepository(t *testing.T) { git := &fakeGit{ path: "/usr/bin/git", outputs: map[string]gitResult{ - "rev-parse --show-toplevel": {output: "/tmp/example"}, + "rev-parse --show-toplevel": {output: "/tmp/example"}, + "config --get remote.origin.url": {output: "git@github.com:launchdarkly/ldcli.git"}, }, } @@ -20,21 +21,22 @@ func TestFindGitRepository(t *testing.T) { require.NoError(t, err) require.True(t, found) require.Equal(t, "/tmp/example", repository.Root) + require.Equal(t, "git@github.com:launchdarkly/ldcli.git", repository.Origin) } -func TestFindGitRepositoryDoesNotRequireOrigin(t *testing.T) { +func TestFindGitRepositoryRequiresOrigin(t *testing.T) { git := &fakeGit{ path: "/usr/bin/git", outputs: map[string]gitResult{ - "rev-parse --show-toplevel": {output: "/tmp/local-only"}, + "rev-parse --show-toplevel": {output: "/tmp/local-only"}, + "config --get remote.origin.url": {err: errors.New("exit status 1")}, }, } - repository, found, err := findGitRepository(git, "/tmp/local-only") + _, found, err := findGitRepository(git, "/tmp/local-only") - require.NoError(t, err) - require.True(t, found) - require.Equal(t, "/tmp/local-only", repository.Root) + require.False(t, found) + require.EqualError(t, err, "Git origin is not configured") } func TestFindGitRepositoryRequiresGitAndInitializedRepository(t *testing.T) { @@ -97,6 +99,25 @@ func TestFindGitRepositoryReturnsOperationalError(t *testing.T) { }) } +func TestDeletedPathsCombinesStagedAndUnstagedChanges(t *testing.T) { + git := &fakeGit{outputs: map[string]gitResult{ + "diff --name-only --diff-filter=D -z -- .launchdarkly": { + output: ".launchdarkly/project/configs/config/unstaged.prompt.md\x00", + }, + "diff --cached --name-only --diff-filter=D -z -- .launchdarkly": { + output: ".launchdarkly/project/configs/config/staged.prompt.md\x00", + }, + }} + + paths, err := deletedPaths(git, "/tmp/example") + + require.NoError(t, err) + require.Equal(t, []string{ + ".launchdarkly/project/configs/config/staged.prompt.md", + ".launchdarkly/project/configs/config/unstaged.prompt.md", + }, paths) +} + type gitResult struct { output string stderr string diff --git a/internal/sync/source/resolver.go b/internal/sync/source/resolver.go index 8b8151f4..844a1432 100644 --- a/internal/sync/source/resolver.go +++ b/internal/sync/source/resolver.go @@ -3,7 +3,9 @@ package source import ( "errors" "fmt" + "net/url" "path/filepath" + "strings" "github.com/launchdarkly/ldcli/internal/sync/repository" ) @@ -13,7 +15,8 @@ var ErrGitRequired = errors.New("sync must run inside an initialized Git reposit // Workspace identifies the repository root used by sync. type Workspace struct { - Root string + Root string + Source string } // Resolver finds the Git workspace containing a requested directory. @@ -43,7 +46,62 @@ func (resolver Resolver) Resolve(dir string) (Workspace, error) { return Workspace{}, err } - return Workspace{Root: root}, nil + source, err := sourceFromOrigin(gitRepository.Origin) + if err != nil { + return Workspace{}, fmt.Errorf("derive sync source from Git origin: %w", err) + } + return Workspace{Root: root, Source: source}, nil +} + +func sourceFromOrigin(origin string) (string, error) { + origin = strings.TrimSpace(origin) + if origin == "" { + return "", errors.New("origin is empty") + } + + host, repositoryPath, ok := scpOrigin(origin) + if !ok { + parsed, err := url.Parse(origin) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("unsupported origin %q", origin) + } + host = normalizedHost(parsed) + repositoryPath = parsed.Path + } + + repositoryPath = strings.Trim(strings.TrimSuffix(repositoryPath, ".git"), "/") + if host == "" || repositoryPath == "" { + return "", fmt.Errorf("unsupported origin %q", origin) + } + return "git:" + strings.ToLower(host) + "/" + repositoryPath, nil +} + +func scpOrigin(origin string) (string, string, bool) { + if strings.Contains(origin, "://") { + return "", "", false + } + userAndHost, repositoryPath, ok := strings.Cut(origin, ":") + if !ok || repositoryPath == "" { + return "", "", false + } + _, host, hasUser := strings.Cut(userAndHost, "@") + if !hasUser { + host = userAndHost + } + return host, repositoryPath, host != "" +} + +func normalizedHost(origin *url.URL) string { + host := origin.Host + switch { + case origin.Scheme == "ssh" && origin.Port() == "22": + host = origin.Hostname() + case origin.Scheme == "https" && origin.Port() == "443": + host = origin.Hostname() + case origin.Scheme == "http" && origin.Port() == "80": + host = origin.Hostname() + } + return host } // canonicalPath resolves symlinks and returns an absolute, clean path so every diff --git a/internal/sync/source/resolver_test.go b/internal/sync/source/resolver_test.go index 5d40dd45..91bacab0 100644 --- a/internal/sync/source/resolver_test.go +++ b/internal/sync/source/resolver_test.go @@ -15,7 +15,9 @@ func TestResolverReturnsCanonicalRepositoryRoot(t *testing.T) { root := t.TempDir() resolver := Resolver{ findGitRepository: func(string) (repository.GitRepository, bool, error) { - return repository.GitRepository{Root: root}, true, nil + return repository.GitRepository{ + Root: root, Origin: "git@github.com:launchdarkly/ldcli.git", + }, true, nil }, } @@ -25,6 +27,7 @@ func TestResolverReturnsCanonicalRepositoryRoot(t *testing.T) { expected, err := canonicalPath(root) require.NoError(t, err) require.Equal(t, expected, workspace.Root) + require.Equal(t, "git:github.com/launchdarkly/ldcli", workspace.Source) } func TestResolverResolvesRepositorySymlink(t *testing.T) { @@ -33,7 +36,9 @@ func TestResolverResolvesRepositorySymlink(t *testing.T) { require.NoError(t, os.Symlink(root, link)) resolver := Resolver{ findGitRepository: func(string) (repository.GitRepository, bool, error) { - return repository.GitRepository{Root: link}, true, nil + return repository.GitRepository{ + Root: link, Origin: "git@github.com:launchdarkly/ldcli.git", + }, true, nil }, } @@ -70,3 +75,39 @@ func TestResolverReturnsRepositoryError(t *testing.T) { require.ErrorIs(t, err, expected) } + +func TestResolverNormalizesGitOrigin(t *testing.T) { + tests := map[string]string{ + "SSH shorthand": "git@GitHub.com:launchdarkly/ldcli.git", + "SSH URL": "ssh://git@github.com:22/launchdarkly/ldcli.git", + "HTTPS URL": "https://token@github.com:443/launchdarkly/ldcli.git", + } + for name, origin := range tests { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + resolver := Resolver{ + findGitRepository: func(string) (repository.GitRepository, bool, error) { + return repository.GitRepository{Root: root, Origin: origin}, true, nil + }, + } + + workspace, err := resolver.Resolve(root) + + require.NoError(t, err) + require.Equal(t, "git:github.com/launchdarkly/ldcli", workspace.Source) + }) + } +} + +func TestResolverRejectsInvalidOrigin(t *testing.T) { + root := t.TempDir() + resolver := Resolver{ + findGitRepository: func(string) (repository.GitRepository, bool, error) { + return repository.GitRepository{Root: root}, true, nil + }, + } + + _, err := resolver.Resolve(root) + + require.ErrorContains(t, err, "derive sync source from Git origin") +}