From 39c181b3b60d980327fc05a8b8531b10e086c909 Mon Sep 17 00:00:00 2001 From: Cam Reeves Date: Tue, 4 Aug 2026 13:40:37 +1000 Subject: [PATCH 1/6] fix(tenant_consent): retry Graph calls that race directory replication (PPT-2000) Microsoft Graph is eventually consistent: the service principal created for a just-registered application is not always visible to the appRoleAssignments POST issued one second later. Graph returns 404 Request_ResourceNotFound and the admin-consent callback dies with a 500, leaving partial state (orphaned app registration, no auth strategy). Observed live on placeos-dev 2026-08-04 - two consecutive runs failed at the identical point: POST /v1.0/servicePrincipals -> 634b1fd1 created POST /v1.0/servicePrincipals/634b1fd1/appRoleAssignments -> Request_ResourceNotFound Add GraphReplicationRetry: retries exactly this case (404 + code Request_ResourceNotFound) with 1/2/4/8/8s backoff, everything else propagates untouched. Applied to every Graph call in the consent callback that references an object created moments earlier. --- spec/graph_replication_retry_spec.cr | 61 +++++++++++++++++++ .../controllers/tenant_consent.cr | 16 +++-- .../utilities/graph-replication-retry.cr | 41 +++++++++++++ 3 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 spec/graph_replication_retry_spec.cr create mode 100644 src/placeos-rest-api/utilities/graph-replication-retry.cr diff --git a/spec/graph_replication_retry_spec.cr b/spec/graph_replication_retry_spec.cr new file mode 100644 index 00000000..d16446bc --- /dev/null +++ b/spec/graph_replication_retry_spec.cr @@ -0,0 +1,61 @@ +require "./helper" + +module PlaceOS::Api + describe GraphReplicationRetry do + replication_lag_error = -> do + Office365::Exception.new( + HTTP::Status::NOT_FOUND, + {error: {code: "Request_ResourceNotFound", message: "Resource 'x' does not exist or one of its queried reference-property objects are not present."}}.to_json, + "Not Found" + ) + end + + it "returns the block value when the call succeeds" do + GraphReplicationRetry.run { 42 }.should eq 42 + end + + it "retries replication lag until the object materialises" do + attempts = 0 + value = GraphReplicationRetry.run(backoff: {0, 0, 0}) do + attempts += 1 + raise replication_lag_error.call if attempts < 3 + :materialised + end + value.should eq :materialised + attempts.should eq 3 + end + + it "gives up once the backoff schedule is exhausted" do + attempts = 0 + expect_raises(Office365::Exception, /Request_ResourceNotFound/) do + GraphReplicationRetry.run(backoff: {0, 0}) do + attempts += 1 + raise replication_lag_error.call + end + end + attempts.should eq 3 + end + + it "does not retry other graph errors" do + attempts = 0 + expect_raises(Office365::Exception) do + GraphReplicationRetry.run(backoff: {0, 0}) do + attempts += 1 + raise Office365::Exception.new(HTTP::Status::FORBIDDEN, {error: {code: "Authorization_RequestDenied", message: "denied"}}.to_json, "Forbidden") + end + end + attempts.should eq 1 + end + + it "does not retry 404s that are not replication lag" do + attempts = 0 + expect_raises(Office365::Exception) do + GraphReplicationRetry.run(backoff: {0, 0}) do + attempts += 1 + raise Office365::Exception.new(HTTP::Status::NOT_FOUND, "gone", "Not Found") + end + end + attempts.should eq 1 + end + end +end diff --git a/src/placeos-rest-api/controllers/tenant_consent.cr b/src/placeos-rest-api/controllers/tenant_consent.cr index a2287f60..37658f82 100644 --- a/src/placeos-rest-api/controllers/tenant_consent.cr +++ b/src/placeos-rest-api/controllers/tenant_consent.cr @@ -85,7 +85,9 @@ module PlaceOS::Api Log.debug { {message: "App registerd with Application permissions", tenant: tenant_id, client_id: created_app.app_id.as(String)} } ra.each do |resource| - client.application_add_app_role_assignment(created_app.app_id.as(String), resource["id"]) + GraphReplicationRetry.run do + client.application_add_app_role_assignment(created_app.app_id.as(String), resource["id"]) + end end created_app.app_id.as(String) @@ -110,8 +112,10 @@ module PlaceOS::Api created_app = client.create_application(app) Log.debug { {message: "App registerd with Delegated permissions", tenant: tenant_id, client_id: created_app.app_id.as(String)} } - client.application_add_oauth2_permission_grant(created_app.app_id.as(String), "Calendars.ReadWrite Calendars.ReadWrite.Shared Group.Read.All User.Read.All offline_access openid profile") - secret = client.application_add_pwd(created_app.app_id.as(String), "PlaceOS User Auth Secret") + GraphReplicationRetry.run do + client.application_add_oauth2_permission_grant(created_app.app_id.as(String), "Calendars.ReadWrite Calendars.ReadWrite.Shared Group.Read.All User.Read.All offline_access openid profile") + end + secret = GraphReplicationRetry.run { client.application_add_pwd(created_app.app_id.as(String), "PlaceOS User Auth Secret") } {client_id: created_app.app_id.as(String), client_secret: secret.secret_text.as(String)} end @@ -144,7 +148,7 @@ module PlaceOS::Api private def add_outlook_plugin_auth(app_id : String) : Nil client = get_client - app = client.get_application(app_id) + app = GraphReplicationRetry.run { client.get_application(app_id) } app_redirect_uris = app.web.try &.redirect_uris || [] of String app_redirect_uris.push("#{domain_url}/outlook/#/book/spaces") @@ -171,7 +175,7 @@ module PlaceOS::Api ], }, } - client.update_application(app_id, updated.to_json) + GraphReplicationRetry.run { client.update_application(app_id, updated.to_json) } updated = { "api": { @@ -185,7 +189,7 @@ module PlaceOS::Api ], }, } - client.update_application(app_id, updated.to_json) + GraphReplicationRetry.run { client.update_application(app_id, updated.to_json) } end private def create_outlook_repo : Nil diff --git a/src/placeos-rest-api/utilities/graph-replication-retry.cr b/src/placeos-rest-api/utilities/graph-replication-retry.cr new file mode 100644 index 00000000..c5210af2 --- /dev/null +++ b/src/placeos-rest-api/utilities/graph-replication-retry.cr @@ -0,0 +1,41 @@ +require "json" +require "office365" + +module PlaceOS::Api + # Microsoft Graph is eventually consistent: a directory object that was just + # created (an application registration or service principal) is not always + # visible to an immediately following request that references it. Graph + # signals this as a 404 with the error code `Request_ResourceNotFound`. + # + # Retry exactly that case with a backoff; every other error propagates + # untouched. + module GraphReplicationRetry + Log = ::Log.for(self) + + # seconds between attempts, ~23s in total. Replication typically settles + # within a few seconds. + BACKOFF = {1, 2, 4, 8, 8} + + def self.run(backoff = BACKOFF, & : -> T) : T forall T + attempt = 0 + loop do + begin + return yield + rescue error : ::Office365::Exception + delay = backoff[attempt]? + raise error unless delay && replication_lag?(error) + attempt += 1 + Log.warn { "graph resource not replicated yet (attempt #{attempt}), retrying in #{delay}s" } + sleep delay.seconds + end + end + end + + def self.replication_lag?(error : ::Office365::Exception) : Bool + return false unless error.http_status.not_found? + JSON.parse(error.http_body).dig?("error", "code").try(&.as_s?) == "Request_ResourceNotFound" + rescue JSON::ParseException + false + end + end +end From 30579aa5f951f97d4972f382304bd226fbd7642d Mon Sep 17 00:00:00 2001 From: Cam Reeves Date: Tue, 4 Aug 2026 14:14:49 +1000 Subject: [PATCH 2/6] fix(tenant_consent): retry the second face of Graph replication lag The lag has (at least) two presentations depending on which side of the race loses: - 404 Request_ResourceNotFound: a just-created service principal is not visible to the appRoleAssignments POST (observed when calls are slow) - 400 Request_BadRequest / NoBackingApplicationObject: the service principal POST cannot see the application object registered moments earlier (observed on a fast box - the flow raced further ahead) Both observed live on placeos-dev 2026-08-04 within the same hour, selected purely by host load. Match both. --- spec/graph_replication_retry_spec.cr | 19 +++++++++++++++++++ .../utilities/graph-replication-retry.cr | 15 +++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/spec/graph_replication_retry_spec.cr b/spec/graph_replication_retry_spec.cr index d16446bc..6b107e41 100644 --- a/spec/graph_replication_retry_spec.cr +++ b/spec/graph_replication_retry_spec.cr @@ -10,10 +10,29 @@ module PlaceOS::Api ) end + app_not_backed_error = -> do + Office365::Exception.new( + HTTP::Status::BAD_REQUEST, + {error: {code: "Request_BadRequest", message: "The appId 'x' of the service principal does not reference a valid application object.", details: [{code: "NoBackingApplicationObject", message: "The appId 'x' of the service principal does not reference a valid application object.", target: "appId"}]}}.to_json, + "Bad Request" + ) + end + it "returns the block value when the call succeeds" do GraphReplicationRetry.run { 42 }.should eq 42 end + it "retries service principal creation racing the application object" do + attempts = 0 + value = GraphReplicationRetry.run(backoff: {0, 0, 0}) do + attempts += 1 + raise app_not_backed_error.call if attempts < 3 + :materialised + end + value.should eq :materialised + attempts.should eq 3 + end + it "retries replication lag until the object materialises" do attempts = 0 value = GraphReplicationRetry.run(backoff: {0, 0, 0}) do diff --git a/src/placeos-rest-api/utilities/graph-replication-retry.cr b/src/placeos-rest-api/utilities/graph-replication-retry.cr index c5210af2..892f21f6 100644 --- a/src/placeos-rest-api/utilities/graph-replication-retry.cr +++ b/src/placeos-rest-api/utilities/graph-replication-retry.cr @@ -32,8 +32,19 @@ module PlaceOS::Api end def self.replication_lag?(error : ::Office365::Exception) : Bool - return false unless error.http_status.not_found? - JSON.parse(error.http_body).dig?("error", "code").try(&.as_s?) == "Request_ResourceNotFound" + body = JSON.parse(error.http_body) + case error.http_status + when .not_found? + # a just-created object is not yet visible to a request referencing it + body.dig?("error", "code").try(&.as_s?) == "Request_ResourceNotFound" + when .bad_request? + # a service principal cannot be created because the application object + # registered moments earlier has not replicated yet + details = body.dig?("error", "details").try(&.as_a?) || [] of JSON::Any + details.any? { |detail| detail["code"]?.try(&.as_s?) == "NoBackingApplicationObject" } + else + false + end rescue JSON::ParseException false end From c8d1e2e462f4de23d46ae750d84745c8d294bf0c Mon Sep 17 00:00:00 2001 From: Cam Reeves Date: Tue, 4 Aug 2026 14:36:54 +1000 Subject: [PATCH 3/6] fix(tenant_consent): third face of Graph replication lag + bigger budget Live verification hit a third presentation: 404 Directory_ObjectNotFound ("Unable to read the company information from the directory") thrown by the SECOND create_application call while the tenant was mid-replication - a documented-transient directory read failure. Also observed a single object taking 25s+ to replicate, exceeding the previous 23s budget. - treat Directory_ObjectNotFound as replication lag - extend backoff to 1/2/4/8/8/12 (~35s) - wrap create_application + the management-app read as well --- spec/graph_replication_retry_spec.cr | 19 +++++++++++++++++++ .../controllers/tenant_consent.cr | 6 +++--- .../utilities/graph-replication-retry.cr | 15 ++++++++++----- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/spec/graph_replication_retry_spec.cr b/spec/graph_replication_retry_spec.cr index 6b107e41..9f3d82de 100644 --- a/spec/graph_replication_retry_spec.cr +++ b/spec/graph_replication_retry_spec.cr @@ -18,10 +18,29 @@ module PlaceOS::Api ) end + directory_busy_error = -> do + Office365::Exception.new( + HTTP::Status::NOT_FOUND, + {error: {code: "Directory_ObjectNotFound", message: "Unable to read the company information from the directory."}}.to_json, + "Not Found" + ) + end + it "returns the block value when the call succeeds" do GraphReplicationRetry.run { 42 }.should eq 42 end + it "retries transient directory reads while the tenant replicates" do + attempts = 0 + value = GraphReplicationRetry.run(backoff: {0, 0, 0}) do + attempts += 1 + raise directory_busy_error.call if attempts < 2 + :materialised + end + value.should eq :materialised + attempts.should eq 2 + end + it "retries service principal creation racing the application object" do attempts = 0 value = GraphReplicationRetry.run(backoff: {0, 0, 0}) do diff --git a/src/placeos-rest-api/controllers/tenant_consent.cr b/src/placeos-rest-api/controllers/tenant_consent.cr index 37658f82..90292eda 100644 --- a/src/placeos-rest-api/controllers/tenant_consent.cr +++ b/src/placeos-rest-api/controllers/tenant_consent.cr @@ -81,7 +81,7 @@ module PlaceOS::Api app = Office365::Application.single_tenant_app("PlaceOS Bookings Visualiser") .add_required_resource(ra) - created_app = client.create_application(app) + created_app = GraphReplicationRetry.run { client.create_application(app) } Log.debug { {message: "App registerd with Application permissions", tenant: tenant_id, client_id: created_app.app_id.as(String)} } ra.each do |resource| @@ -109,7 +109,7 @@ module PlaceOS::Api .add_web_redirect_uri("https://#{domain}/auth/oauth2/callback?id=#{strat_id}") .add_required_resource(ra) - created_app = client.create_application(app) + created_app = GraphReplicationRetry.run { client.create_application(app) } Log.debug { {message: "App registerd with Delegated permissions", tenant: tenant_id, client_id: created_app.app_id.as(String)} } GraphReplicationRetry.run do @@ -125,7 +125,7 @@ module PlaceOS::Api private def update_app_redirect_uri(add : Bool = true) : Nil client = get_client - app = client.get_application(PLACE_APP_CLIENT_ID, "id,web") + app = GraphReplicationRetry.run { client.get_application(PLACE_APP_CLIENT_ID, "id,web") } app_redirect_uris = app.web.try &.redirect_uris || [] of String return nil if add && app_redirect_uris.includes?(redirect_url) diff --git a/src/placeos-rest-api/utilities/graph-replication-retry.cr b/src/placeos-rest-api/utilities/graph-replication-retry.cr index 892f21f6..8cc81b92 100644 --- a/src/placeos-rest-api/utilities/graph-replication-retry.cr +++ b/src/placeos-rest-api/utilities/graph-replication-retry.cr @@ -12,9 +12,10 @@ module PlaceOS::Api module GraphReplicationRetry Log = ::Log.for(self) - # seconds between attempts, ~23s in total. Replication typically settles - # within a few seconds. - BACKOFF = {1, 2, 4, 8, 8} + # seconds between attempts, ~35s in total. Replication typically settles + # within a few seconds but has been observed (sandbox tenant, 2026-08-04) + # taking 25s+. + BACKOFF = {1, 2, 4, 8, 8, 12} def self.run(backoff = BACKOFF, & : -> T) : T forall T attempt = 0 @@ -35,8 +36,12 @@ module PlaceOS::Api body = JSON.parse(error.http_body) case error.http_status when .not_found? - # a just-created object is not yet visible to a request referencing it - body.dig?("error", "code").try(&.as_s?) == "Request_ResourceNotFound" + # Request_ResourceNotFound: a just-created object is not yet visible + # to a request referencing it. + # Directory_ObjectNotFound ("Unable to read the company information + # from the directory"): transient directory read failure while the + # tenant is busy replicating - documented by Microsoft as retryable. + body.dig?("error", "code").try(&.as_s?).in?("Request_ResourceNotFound", "Directory_ObjectNotFound") when .bad_request? # a service principal cannot be created because the application object # registered moments earlier has not replicated yet From 4754675cf4b8c8804bdd5e0e9d536634f93c9984 Mon Sep 17 00:00:00 2001 From: Cam Reeves Date: Tue, 4 Aug 2026 14:56:08 +1000 Subject: [PATCH 4/6] fix(tenant_consent): fourth face - pre-authorization races its own scope The two sequential PATCHes in add_outlook_plugin_auth race each other: the second (preAuthorizedApplications) validates against a replica that has not yet seen the scope added by the first, failing with 400 InvalidValue on api.preAuthorizedApplications.delegatedPermissionIds. Matcher is deliberately narrow (that exact target only) so genuine InvalidValue validation errors are never retried; spec pins both sides. Every run now dies one step later than the previous - this was the last Graph write in the flow. --- spec/graph_replication_retry_spec.cr | 34 +++++++++++++++++++ .../controllers/tenant_consent.cr | 2 +- .../utilities/graph-replication-retry.cr | 17 ++++++++-- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/spec/graph_replication_retry_spec.cr b/spec/graph_replication_retry_spec.cr index 9f3d82de..d743a04f 100644 --- a/spec/graph_replication_retry_spec.cr +++ b/spec/graph_replication_retry_spec.cr @@ -26,10 +26,44 @@ module PlaceOS::Api ) end + stale_scope_error = -> do + Office365::Exception.new( + HTTP::Status::BAD_REQUEST, + {error: {code: "Request_BadRequest", message: "Property api.preAuthorizedApplications.delegatedPermissionIds has a Permission Id that cannot be found in the AppPermissions sets.", details: [{code: "InvalidValue", message: "Property api.preAuthorizedApplications.delegatedPermissionIds has a Permission Id that cannot be found in the AppPermissions sets.", target: "api.preAuthorizedApplications.delegatedPermissionIds"}]}}.to_json, + "Bad Request" + ) + end + it "returns the block value when the call succeeds" do GraphReplicationRetry.run { 42 }.should eq 42 end + it "retries pre-authorization racing the scope it references" do + attempts = 0 + value = GraphReplicationRetry.run(backoff: {0, 0, 0}) do + attempts += 1 + raise stale_scope_error.call if attempts < 2 + :materialised + end + value.should eq :materialised + attempts.should eq 2 + end + + it "does not retry InvalidValue errors on other properties" do + attempts = 0 + expect_raises(Office365::Exception) do + GraphReplicationRetry.run(backoff: {0, 0}) do + attempts += 1 + raise Office365::Exception.new( + HTTP::Status::BAD_REQUEST, + {error: {code: "Request_BadRequest", message: "bad", details: [{code: "InvalidValue", message: "bad", target: "identifierUris"}]}}.to_json, + "Bad Request" + ) + end + end + attempts.should eq 1 + end + it "retries transient directory reads while the tenant replicates" do attempts = 0 value = GraphReplicationRetry.run(backoff: {0, 0, 0}) do diff --git a/src/placeos-rest-api/controllers/tenant_consent.cr b/src/placeos-rest-api/controllers/tenant_consent.cr index 90292eda..3214de83 100644 --- a/src/placeos-rest-api/controllers/tenant_consent.cr +++ b/src/placeos-rest-api/controllers/tenant_consent.cr @@ -139,7 +139,7 @@ module PlaceOS::Api app.web.not_nil!.redirect_uris = app_redirect_uris web = {"web" => app.web} begin - client.update_application(PLACE_APP_CLIENT_ID, web.to_json) + GraphReplicationRetry.run { client.update_application(PLACE_APP_CLIENT_ID, web.to_json) } rescue ex : Office365::Exception return nil if already_exists_error?(ex.http_body) raise ex diff --git a/src/placeos-rest-api/utilities/graph-replication-retry.cr b/src/placeos-rest-api/utilities/graph-replication-retry.cr index 8cc81b92..042136fa 100644 --- a/src/placeos-rest-api/utilities/graph-replication-retry.cr +++ b/src/placeos-rest-api/utilities/graph-replication-retry.cr @@ -43,10 +43,21 @@ module PlaceOS::Api # tenant is busy replicating - documented by Microsoft as retryable. body.dig?("error", "code").try(&.as_s?).in?("Request_ResourceNotFound", "Directory_ObjectNotFound") when .bad_request? - # a service principal cannot be created because the application object - # registered moments earlier has not replicated yet + # NoBackingApplicationObject: a service principal cannot be created + # because the application object registered moments earlier has not + # replicated yet. + # InvalidValue on preAuthorizedApplications.delegatedPermissionIds: + # the PATCH that added the permission scope moments earlier has not + # replicated, so the follow-up PATCH fails validation against a stale + # copy of the application. Deliberately narrow - a generic InvalidValue + # retry would mask real validation errors. details = body.dig?("error", "details").try(&.as_a?) || [] of JSON::Any - details.any? { |detail| detail["code"]?.try(&.as_s?) == "NoBackingApplicationObject" } + details.any? do |detail| + code = detail["code"]?.try(&.as_s?) + target = detail["target"]?.try(&.as_s?) + code == "NoBackingApplicationObject" || + (code == "InvalidValue" && target == "api.preAuthorizedApplications.delegatedPermissionIds") + end else false end From cc9cb3b645fc6b32010f0ce927d5560d8146552d Mon Sep 17 00:00:00 2001 From: Cam Reeves Date: Tue, 4 Aug 2026 15:33:50 +1000 Subject: [PATCH 5/6] fix(tenant_consent): stop removing the callback redirect URI after each run The ensure block deleted the management app's redirect URI whenever a consent run completed, which set a trap for the NEXT run: the index endpoint re-adds the URI seconds before consent, and login.microsoftonline.com's app-metadata cache does not pick the change up in time - the admin clicks Accept and gets AADSTS500113 'No reply address is registered for the application'. Observed live minutes after the first fully successful run (whose cleanup removed the URI that had been registered for hours). The churn defeats itself in the other direction too: the add path's already-present check can read a stale replica that still lists the just-removed URI and skip the re-add entirely. And two concurrent consent flows would have the first finisher delete the URI mid-flight for the second. A PlaceOS host's callback URI is stable - register it and leave it. The index endpoint still adds it when missing (first run on a new host), it just never gets torn down. --- src/placeos-rest-api/controllers/tenant_consent.cr | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/placeos-rest-api/controllers/tenant_consent.cr b/src/placeos-rest-api/controllers/tenant_consent.cr index 3214de83..d91562cc 100644 --- a/src/placeos-rest-api/controllers/tenant_consent.cr +++ b/src/placeos-rest-api/controllers/tenant_consent.cr @@ -61,8 +61,6 @@ module PlaceOS::Api create_outlook_config(auth_app[:client_id]) strat.update!(client_id: auth_app[:client_id], client_secret: auth_app[:client_secret]) update_auth(authority, strat.id.as(String)) - ensure - update_app_redirect_uri(false) end else Log.warn { {message: "Admin declined consent", error: error.to_s, description: error_description.to_s} } From 1eb8ef85b5d2bc3d2639c49a00ff00d77f47993d Mon Sep 17 00:00:00 2001 From: Cam Reeves Date: Tue, 4 Aug 2026 15:37:16 +1000 Subject: [PATCH 6/6] fix(tenant_consent): sixth face - get-or-create double-creates off a stale read Observed: SP created successfully -> role assignment fails on lag -> block retries -> the existence check reads a replica that does not yet list the SP created one second earlier -> create runs again -> 409 Request_MultipleObjectsWithSameKeyValue (the uniqueness constraint sees the truth even when reads do not). Retrying converges: the next read eventually sees the object and the get-or-create skips creation, proceeding to the assignment. --- spec/graph_replication_retry_spec.cr | 19 +++++++++++++++++++ .../utilities/graph-replication-retry.cr | 7 +++++++ 2 files changed, 26 insertions(+) diff --git a/spec/graph_replication_retry_spec.cr b/spec/graph_replication_retry_spec.cr index d743a04f..98f73888 100644 --- a/spec/graph_replication_retry_spec.cr +++ b/spec/graph_replication_retry_spec.cr @@ -34,10 +34,29 @@ module PlaceOS::Api ) end + duplicate_create_error = -> do + Office365::Exception.new( + HTTP::Status::CONFLICT, + {error: {code: "Request_MultipleObjectsWithSameKeyValue", message: "The service principal cannot be created, updated, or restored because the service principal name x is already in use.", details: [{code: "ObjectConflict", message: "already in use", target: "servicePrincipalNames"}]}}.to_json, + "Conflict" + ) + end + it "returns the block value when the call succeeds" do GraphReplicationRetry.run { 42 }.should eq 42 end + it "retries a get-or-create that double-created off a stale read" do + attempts = 0 + value = GraphReplicationRetry.run(backoff: {0, 0, 0}) do + attempts += 1 + raise duplicate_create_error.call if attempts < 2 + :converged + end + value.should eq :converged + attempts.should eq 2 + end + it "retries pre-authorization racing the scope it references" do attempts = 0 value = GraphReplicationRetry.run(backoff: {0, 0, 0}) do diff --git a/src/placeos-rest-api/utilities/graph-replication-retry.cr b/src/placeos-rest-api/utilities/graph-replication-retry.cr index 042136fa..a81ba234 100644 --- a/src/placeos-rest-api/utilities/graph-replication-retry.cr +++ b/src/placeos-rest-api/utilities/graph-replication-retry.cr @@ -58,6 +58,13 @@ module PlaceOS::Api code == "NoBackingApplicationObject" || (code == "InvalidValue" && target == "api.preAuthorizedApplications.delegatedPermissionIds") end + when .conflict? + # Request_MultipleObjectsWithSameKeyValue on a service principal + # create: the preceding existence check read a stale replica that did + # not yet list the service principal created moments earlier, so the + # get-or-create tried to create it twice. Retrying converges - the + # next read eventually sees the object and the create is skipped. + body.dig?("error", "code").try(&.as_s?) == "Request_MultipleObjectsWithSameKeyValue" else false end