feat: Re-land local flag overrides using dev server, with filter reset fix

internal/dev_server/db/sqlite.go

@@ -91,6 +91,14 @@ func (s *Sqlite) UpdateProject(ctx context.Context, project model.Project) (bool
 			_ = tx.Rollback()
 		}
 	}()
+	previousFlagsStateJson := "{}"
+	err = tx.QueryRowContext(ctx, `SELECT flag_state FROM projects WHERE key = ?`, project.Key).Scan(&previousFlagsStateJson)
+	if errors.Is(err, sql.ErrNoRows) {
+		err = nil
+	}
+	if err != nil {
+		return false, errors.Wrap(err, "unable to read flag state when updating project")
+	}
 	result, err := tx.ExecContext(ctx, `
 		UPDATE projects
 		SET flag_state = ?, last_sync_time = ?, context=?, source_environment_key=?
@@ -117,8 +125,10 @@ func (s *Sqlite) UpdateProject(ctx context.Context, project model.Project) (bool
 	// Prune overrides for flags no longer in the project. Key off flag_state (always fully populated from the SDK), not available_variations, which lags the background fill in streaming mode and would wrongly wipe overrides.
 	_, err = tx.ExecContext(ctx, `
 		DELETE FROM overrides
-		WHERE project_key = ? AND flag_key NOT IN (SELECT key FROM json_each(?))
-	`, project.Key, string(flagsStateJson))
+		WHERE project_key = ?
+			AND flag_key IN (SELECT key FROM json_each(?))
+			AND flag_key NOT IN (SELECT key FROM json_each(?))
+	`, project.Key, previousFlagsStateJson, string(flagsStateJson))
 	if err != nil {
 		return false, err
 	}

internal/dev_server/db/sqlite_test.go

@@ -452,4 +452,33 @@ func TestDBFunctions(t *testing.T) {
 		require.Len(t, overrides, 1)
 		assert.Equal(t, override, overrides[0])
 	})
+
+	t.Run("UpdateProject keeps overrides for flags that were never in the project", func(t *testing.T) {
+		project := projects[2]
+
+		override, err := store.UpsertOverride(ctx, model.Override{
+			ProjectKey: project.Key,
+			FlagKey:    "local-only-flag",
+			Value:      ldvalue.String("on"),
+			Active:     true,
+			Version:    1,
+		})
+		require.NoError(t, err)
+
+		project.AllFlagsState = model.FlagsState{
+			"flag-2": model.FlagState{Value: ldvalue.Bool(true), Version: 1},
+		}
+		project.AvailableVariations = []model.FlagVariation{
+			{
+				FlagKey: "flag-2",
+			},
+		}
+		updated, err := store.UpdateProject(ctx, project)
+		require.NoError(t, err)
+		require.True(t, updated)
+
+		overrides, err := store.GetOverridesForProject(ctx, project.Key)
+		require.NoError(t, err)
+		assert.Contains(t, overrides, override)
+	})
 }

internal/dev_server/model/override.go

@@ -27,16 +27,6 @@ func getFlagStateForFlagAndProject(ctx context.Context, projectKey, flagKey stri
 		return FlagState{}, err
 	}
 
-	var flagExists bool
-	for flag := range project.AllFlagsState {
-		if flagKey == flag {
-			flagExists = true
-			break
-		}
-	}
-	if !flagExists {
-		return FlagState{}, NewErrNotFound("flag", flagKey)
-	}
 	return project.AllFlagsState[flagKey], nil
 }
 
@@ -75,11 +65,12 @@ func UpsertOverride(ctx context.Context, projectKey, flagKey string, value ldval
 }
 
 func DeleteOverride(ctx context.Context, projectKey, flagKey string) error {
-	flagState, err := getFlagStateForFlagAndProject(ctx, projectKey, flagKey)
+	store := StoreFromContext(ctx)
+	project, err := store.GetDevProject(ctx, projectKey)
 	if err != nil {
 		return err
 	}
-	store := StoreFromContext(ctx)
+	flagState, synced := project.AllFlagsState[flagKey]
 	version, err := store.DeactivateOverride(ctx, projectKey, flagKey)
 	if err != nil {
 		return err
@@ -90,6 +81,19 @@ func DeleteOverride(ctx context.Context, projectKey, flagKey string) error {
 		return errors.Wrap(err, "unable to increment payload version")
 	}
 
+	if !synced {
+		allFlags, err := project.GetFlagStateWithOverridesForProject(ctx)
+		if err != nil {
+			return err
+		}
+		GetObserversFromContext(ctx).Notify(SyncEvent{
+			ProjectKey:     projectKey,
+			AllFlagsState:  allFlags,
+			PayloadVersion: newPayloadVersion,
+		})
+		return nil
+	}
+
 	override := Override{
 		ProjectKey: projectKey,
 		FlagKey:    flagKey,

internal/dev_server/model/override_test.go

@@ -49,16 +49,26 @@ func TestUpsertOverride(t *testing.T) {
 		assert.Error(t, err)
 	})
 
-	t.Run("Returns error if flag does not exist in project", func(t *testing.T) {
-		badProj := model.Project{
+	t.Run("flag not in project is overridden as a new flag", func(t *testing.T) {
+		emptyProj := model.Project{
 			Key:           projKey,
 			AllFlagsState: model.FlagsState{},
 		}
-		store.EXPECT().GetDevProject(gomock.Any(), projKey).Return(&badProj, nil)
+		store.EXPECT().GetDevProject(gomock.Any(), projKey).Return(&emptyProj, nil)
+		store.EXPECT().UpsertOverride(gomock.Any(), override).Return(override, nil)
+		store.EXPECT().IncrementProjectPayloadVersion(gomock.Any(), projKey).Return(1, nil)
+		observer.
+			EXPECT().
+			Handle(model.OverrideEvent{
+				FlagKey:        flagKey,
+				ProjectKey:     projKey,
+				FlagState:      model.FlagState{Value: ldvalue.Bool(true), Version: 1, TrackEvents: true},
+				PayloadVersion: 1,
+			})
 
-		_, err := model.UpsertOverride(ctx, projKey, flagKey, ldValue)
-		assert.Error(t, err)
-		assert.ErrorAs(t, err, &model.ErrNotFound{})
+		o, err := model.UpsertOverride(ctx, projKey, flagKey, ldValue)
+		assert.Nil(t, err)
+		assert.Equal(t, override, o)
 	})
 
 	t.Run("store fails to upsert, returns error", func(t *testing.T) {
@@ -146,6 +156,26 @@ func TestDeleteOverride(t *testing.T) {
 		err := model.DeleteOverride(ctx, projKey, flagKey)
 		assert.Nil(t, err)
 	})
+
+	t.Run("flag not in project is removed with a full sync", func(t *testing.T) {
+		localFlagKey := "local-only"
+		store.EXPECT().GetDevProject(gomock.Any(), projKey).Return(project, nil)
+		store.EXPECT().DeactivateOverride(gomock.Any(), projKey, localFlagKey).Return(2, nil)
+		store.EXPECT().IncrementProjectPayloadVersion(gomock.Any(), projKey).Return(3, nil)
+		store.EXPECT().GetOverridesForProject(gomock.Any(), projKey).Return(model.Overrides{
+			{ProjectKey: projKey, FlagKey: localFlagKey, Value: ldValue, Active: false, Version: 2},
+		}, nil)
+		observer.
+			EXPECT().
+			Handle(model.SyncEvent{
+				ProjectKey:     projKey,
+				AllFlagsState:  project.AllFlagsState,
+				PayloadVersion: 3,
+			})
+
+		err := model.DeleteOverride(ctx, projKey, localFlagKey)
+		assert.Nil(t, err)
+	})
 }
 
 func TestDeleteOverrides(t *testing.T) {

internal/dev_server/model/project.go

@@ -144,6 +144,11 @@ func (project Project) GetFlagStateWithOverridesForProject(ctx context.Context)
 		}
 		withOverrides[flagKey] = flagState
 	}
+	for _, override := range overrides {
+		if _, synced := project.AllFlagsState[override.FlagKey]; override.Active && !synced {
+			withOverrides[override.FlagKey] = override.Apply(FlagState{})
+		}
+	}
 	return withOverrides, nil
 }
 

internal/dev_server/model/project_test.go

@@ -242,4 +242,21 @@ func TestGetFlagStateWithOverridesForProject(t *testing.T) {
 		assert.True(t, overriddenFlag.Value.BoolValue())
 		assert.Equal(t, 2, overriddenFlag.Version)
 	})
+
+	t.Run("Includes active overrides for flags not in the project", func(t *testing.T) {
+		overrides := model.Overrides{
+			{ProjectKey: proj.Key, FlagKey: "local-only", Value: ldvalue.String("on"), Active: true, Version: 3},
+			{ProjectKey: proj.Key, FlagKey: "removed", Value: ldvalue.Bool(true), Active: false, Version: 2},
+		}
+
+		store.EXPECT().GetOverridesForProject(gomock.Any(), proj.Key).Return(overrides, nil)
+
+		withOverrides, err := proj.GetFlagStateWithOverridesForProject(ctx)
+		assert.Nil(t, err)
+
+		assert.Equal(t, model.FlagsState{
+			flagKey:      model.FlagState{Value: ldvalue.Bool(false), Version: 1},
+			"local-only": model.FlagState{Value: ldvalue.String("on"), Version: 3, TrackEvents: true},
+		}, withOverrides)
+	})
 }

internal/dev_server/ui/src/App.css

@@ -28,6 +28,11 @@ body,
   align-items: center;
 }
 
+.only-show-overrides-label.disabled {
+  color: var(--lp-color-text-interactive-disabled);
+  cursor: not-allowed;
+}
+
 ul.flags-list {
   margin: 0;
   padding: 0;

internal/dev_server/ui/src/Flags.tsx

@@ -16,9 +16,9 @@ import {
   Stack,
 } from '@launchpad-ui/core';
 import Theme from '@launchpad-ui/tokens';
-import { useState, useCallback, useMemo, useEffect } from 'react';
+import { useState, useCallback, useEffect, useMemo } from 'react';
 import { Icon } from '@launchpad-ui/icons';
-import { apiRoute } from './util.ts';
+import { apiRoute, sortFlags } from './util.ts';
 import { FlagVariation } from './api.ts';
 import VariationValues from './Flag.tsx';
 import fuzzysort from 'fuzzysort';
@@ -41,6 +41,7 @@ function Flags({
   setOverrides,
 }: FlagProps) {
   const [onlyShowOverrides, setOnlyShowOverrides] = useState(false);
+  const [onlyShowLocal, setOnlyShowLocal] = useState(false);
   const [searchTerm, setSearchTerm] = useState('');
   const [currentPage, setCurrentPage] = useState(0);
   const flagsPerPage = 20;
@@ -50,15 +51,40 @@ function Flags({
     [overrides],
   );
 
+  const localFlagsPresent = useMemo(
+    () =>
+      !!flags && Object.keys(overrides).some((flagKey) => !(flagKey in flags)),
+    [flags, overrides],
+  );
+
   useEffect(() => {
-    if (!overridesPresent && onlyShowOverrides) {
+    if (!overridesPresent) {
       setOnlyShowOverrides(false);
     }
-  }, [overridesPresent, onlyShowOverrides]);
+  }, [overridesPresent]);
+
+  useEffect(() => {
+    if (flags && !localFlagsPresent) {
+      setOnlyShowLocal(false);
+    }
+  }, [flags, localFlagsPresent]);
+
+  const showOnlyOverrides = onlyShowOverrides && !!overridesPresent;
+  const showOnlyLocal = onlyShowLocal && localFlagsPresent;
+
+  const allFlags = useMemo(() => {
+    if (!flags) return null;
+    const localFlags = Object.fromEntries(
+      Object.entries(overrides)
+        .filter(([flagKey]) => !(flagKey in flags))
+        .map(([flagKey, { value }]) => [flagKey, { value }]),
+    );
+    return sortFlags({ ...flags, ...localFlags });
+  }, [flags, overrides]);
 
   const filteredFlags = useMemo(() => {
-    if (!flags) return [];
-    const flagEntries = Object.entries(flags);
+    if (!allFlags) return [];
+    const flagEntries = Object.entries(allFlags);
     return flagEntries
       .filter((entry) => {
         if (!searchTerm) return true;
@@ -79,13 +105,24 @@ function Flags({
         const [flagKey] = entry;
         const hasOverride = flagKey in overrides;
 
-        if (onlyShowOverrides && !hasOverride) {
+        if (showOnlyOverrides && !hasOverride) {
+          return false;
+        }
+
+        if (showOnlyLocal && (!flags || flagKey in flags)) {
           return false;
         }
 
         return true;
       });
-  }, [flags, searchTerm, onlyShowOverrides, overrides]);
+  }, [
+    allFlags,
+    flags,
+    searchTerm,
+    showOnlyOverrides,
+    showOnlyLocal,
+    overrides,
+  ]);
 
   const paginatedFlags = useMemo(() => {
     const startIndex = currentPage * flagsPerPage;
@@ -213,11 +250,11 @@ function Flags({
       >
         <Label
           htmlFor="only-show-overrides"
-          className="only-show-overrides-label"
+          className={`only-show-overrides-label${overridesPresent ? '' : ' disabled'}`}
         >
           <Checkbox
             id="only-show-overrides"
-            isSelected={onlyShowOverrides}
+            isSelected={showOnlyOverrides}
             onChange={(newValue) => {
               setOnlyShowOverrides(newValue);
             }}
@@ -229,6 +266,26 @@ function Flags({
           />
           Only show flags with overrides
         </Label>
+        {localFlagsPresent && (
+          <Label
+            htmlFor="only-show-local"
+            className="only-show-overrides-label"
+            style={{ marginLeft: '1.5rem', marginRight: 'auto' }}
+          >
+            <Checkbox
+              id="only-show-local"
+              isSelected={showOnlyLocal}
+              onChange={(newValue) => {
+                setOnlyShowLocal(newValue);
+              }}
+              style={{
+                display: 'inline-block',
+                marginRight: '.25rem',
+              }}
+            />
+            Only show local flags
+          </Label>
+        )}
         <Button
           variant="destructive"
           isDisabled={!overridesPresent}