Improve ZooKeeper reconnection strategy

Author: jdhoek

pom.xml

@@ -43,7 +43,7 @@
   <properties>
     <java.language.level>1.8</java.language.level>
 
-    <dynamicconfig.version>2.9-beta2</dynamicconfig.version>
+    <dynamicconfig.version>2.9-beta3</dynamicconfig.version>
     <zookeeper.version>3.4.6</zookeeper.version>
     <slf4j.version>1.7.21</slf4j.version>
 

uniqueid-zookeeper/src/main/java/org/lable/oss/uniqueid/zookeeper/SynchronizedGeneratorIdentity.java

@@ -19,6 +19,8 @@
 import org.lable.oss.uniqueid.GeneratorException;
 import org.lable.oss.uniqueid.GeneratorIdentityHolder;
 import org.lable.oss.uniqueid.bytes.Blueprint;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.time.Duration;
@@ -34,6 +36,8 @@
  * @see #basedOn(String, String, Supplier, Supplier)
  */
 public class SynchronizedGeneratorIdentity implements GeneratorIdentityHolder {
+    private static final Logger logger = LoggerFactory.getLogger(SynchronizedGeneratorIdentity.class);
+
     final int clusterId;
 
     final Supplier<Duration> claimDurationSupplier;
@@ -135,6 +139,10 @@ public void relinquishGeneratorIdClaim() {
     }
 
     private ResourceClaim acquireResourceClaim() throws GeneratorException {
+        return acquireResourceClaim(0);
+    }
+
+    private ResourceClaim acquireResourceClaim(int retries) throws GeneratorException {
         try {
             return ExpiringResourceClaim.claimExpiring(
                     zooKeeperConnection,
@@ -144,7 +152,15 @@ private ResourceClaim acquireResourceClaim() throws GeneratorException {
                     acquisitionTimeoutSupplier.get()
             );
         } catch (IOException e) {
-            throw new GeneratorException(e);
+            logger.warn(
+                    "Connection to ZooKeeper quorum failed, retrying resource claim acquisition, attempt {}.",
+                    retries + 1
+            );
+            if (retries < 3) {
+                return acquireResourceClaim(retries + 1);
+            } else {
+                throw new GeneratorException(e);
+            }
         }
     }