diff --git a/core/src/main/java/org/testcontainers/containers/GenericContainer.java b/core/src/main/java/org/testcontainers/containers/GenericContainer.java index 4d3778c63d1..cefdbfcd426 100644 --- a/core/src/main/java/org/testcontainers/containers/GenericContainer.java +++ b/core/src/main/java/org/testcontainers/containers/GenericContainer.java @@ -6,6 +6,7 @@ import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.CreateContainerCmd; import com.github.dockerjava.api.command.InspectContainerResponse; +import com.github.dockerjava.api.exception.ConflictException; import com.github.dockerjava.api.exception.NotFoundException; import com.github.dockerjava.api.model.Bind; import com.github.dockerjava.api.model.ContainerNetwork; @@ -69,6 +70,8 @@ import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -88,6 +91,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -112,6 +116,11 @@ public class GenericContainer> static final String COPIED_FILES_HASH_LABEL = "org.testcontainers.copied_files.hash"; + /** + * How long a reusable container may stay in "created" before it counts as left behind by a killed start + */ + private static final Duration REUSE_ORPHAN_AGE = Duration.ofMinutes(1); + /* * Default settings */ @@ -378,6 +387,7 @@ private void tryStart() { createCommand.getLabels().putAll(DockerClientFactory.DEFAULT_LABELS); boolean reused = false; + String reuseHash = null; final boolean reusable; if (shouldBeReused) { if (!canBeReused()) { @@ -400,6 +410,11 @@ private void tryStart() { logger().debug("Can't find a reusable running container with hash: {}", hash); createCommand.getLabels().put(HASH_LABEL, hash); + if (createCommand.getName() == null) { + // Docker enforces unique names, so only one of several concurrent starts can create it + createCommand.withName("testcontainers-reuse-" + hash.substring(0, 12)); + } + reuseHash = hash; } reusable = true; } else { @@ -421,9 +436,13 @@ private void tryStart() { createCommand = ResourceReaper.instance().register(this, createCommand); } - if (!reused) { + if (reuseHash != null) { + reused = createOrAwaitReusableContainer(createCommand, reuseHash); + } else if (!reused) { containerId = createCommand.exec().getId(); + } + if (!reused) { // TODO use single "copy" invocation (and calculate an hash of the resulting tar archive) copyToFileContainerPathMap.forEach(this::copyFileToContainer); @@ -599,6 +618,77 @@ Optional findContainerForReuse(String hash) { .map(it -> it.getId()); } + /** + * Creates the reusable container, or waits for a concurrent start that already claimed its name to run it. + * + * @return {@code true} if an existing container is reused, {@code false} if this call created it + */ + private boolean createOrAwaitReusableContainer(CreateContainerCmd createCommand, String hash) { + AtomicBoolean created = new AtomicBoolean(false); + containerId = + await() + .atMost(REUSE_ORPHAN_AGE.multipliedBy(2)) + .pollDelay(Duration.ZERO) + .pollInterval(DynamicPollInterval.ofMillis(100)) + .pollInSameThread() + .until( + () -> { + try { + String id = createCommand.exec().getId(); + created.set(true); + return id; + } catch (ConflictException e) { + logger() + .debug("Container {} already exists, waiting for it to run", createCommand.getName()); + } + String runningId = findContainerForReuse(hash).orElse(null); + if (runningId == null) { + removeStoppedContainer(createCommand.getName(), hash); + } + return runningId; + }, + Objects::nonNull + ); + return !created.get(); + } + + /** + * Removes a stopped or orphaned leftover of this reusable container, so that the name can be claimed again. + */ + private void removeStoppedContainer(String name, String hash) { + try { + InspectContainerResponse existing = dockerClient.inspectContainerCmd(name).exec(); + Map labels = existing.getConfig().getLabels(); + if (labels == null || !hash.equals(labels.get(HASH_LABEL))) { + throw new IllegalStateException( + "Container name " + name + " is already used by a container with a different configuration" + ); + } + String status = existing.getState().getStatus(); + if ("paused".equals(status)) { + throw new IllegalStateException("Reusable container " + name + " is paused, unpause or remove it"); + } + // A start that dies between create and start leaves the container in "created" for good + boolean orphaned = "created".equals(status) && isOlderThan(existing.getCreated(), REUSE_ORPHAN_AGE); + if ("exited".equals(status) || "dead".equals(status) || orphaned) { + logger().info("Removing leftover reusable container {} ({}, {})", name, existing.getId(), status); + // Remove by ID, so a container that was re-created in the meantime is not affected + dockerClient.removeContainerCmd(existing.getId()).withForce(true).exec(); + } + } catch (NotFoundException | ConflictException e) { + // Removed by someone else in the meantime + } + } + + private static boolean isOlderThan(String timestamp, Duration age) { + try { + return OffsetDateTime.parse(timestamp).toInstant().isBefore(Instant.now().minus(age)); + } catch (DateTimeParseException | NullPointerException e) { + // Unknown age, so never treat it as orphaned + return false; + } + } + /** * Set any custom settings for the create command such as shared memory size. */ diff --git a/core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java b/core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java index 2c70a3f894f..f2cba2c21b5 100644 --- a/core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java +++ b/core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java @@ -8,7 +8,9 @@ import com.github.dockerjava.api.command.InspectContainerCmd; import com.github.dockerjava.api.command.InspectContainerResponse; import com.github.dockerjava.api.command.ListContainersCmd; +import com.github.dockerjava.api.command.RemoveContainerCmd; import com.github.dockerjava.api.command.StartContainerCmd; +import com.github.dockerjava.api.exception.ConflictException; import com.github.dockerjava.api.model.Container; import com.github.dockerjava.core.command.CreateContainerCmdImpl; import com.github.dockerjava.core.command.InspectContainerCmdImpl; @@ -21,6 +23,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Answers; +import org.mockito.ArgumentMatchers; import org.mockito.Mockito; import org.mockito.stubbing.Answer; import org.testcontainers.DockerClientFactory; @@ -34,12 +37,15 @@ import java.io.File; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; @@ -252,6 +258,93 @@ void shouldSetCopiedFilesHashLabel() { assertThat(commandRef.get().getLabels()).containsKeys(GenericContainer.COPIED_FILES_HASH_LABEL); } + @Test + void shouldNameReusableContainerAfterHash() { + Mockito.doReturn(true).when(TestcontainersConfiguration.getInstance()).environmentSupportsReuse(); + AtomicReference commandRef = new AtomicReference<>(); + String containerId = randomContainerId(); + when(client.createContainerCmd(any())).then(createContainerAnswer(containerId, commandRef::set)); + when(client.listContainersCmd()).then(listContainersAnswer()); + when(client.startContainerCmd(containerId)).then(startContainerAnswer()); + when(client.inspectContainerCmd(containerId)).then(inspectContainerAnswer()); + + container.start(); + + String hash = commandRef.get().getLabels().get(GenericContainer.HASH_LABEL); + assertThat(commandRef.get().getName()).isEqualTo("testcontainers-reuse-" + hash.substring(0, 12)); + } + + @Test + void shouldReuseContainerCreatedConcurrently() { + Mockito.doReturn(true).when(TestcontainersConfiguration.getInstance()).environmentSupportsReuse(); + when(client.createContainerCmd(any())) + .then(invocation -> { + CreateContainerCmd.Exec exec = command -> { + throw new ConflictException("name is already in use"); + }; + return new CreateContainerCmdImpl(exec, null, "image:latest"); + }); + String existingContainerId = randomContainerId(); + // Not running yet when first looked up, running once the concurrent start is done + when(client.listContainersCmd()) + .then(listContainersAnswer()) + .then(listContainersAnswer(existingContainerId)); + when(client.inspectContainerCmd(existingContainerId)).then(inspectContainerAnswer()); + + container.start(); + + assertThat(container.getContainerId()).isEqualTo(existingContainerId); + Mockito.verify(client, Mockito.never()).startContainerCmd(any()); + } + + @Test + void shouldReplaceContainerOrphanedInCreatedState() { + Mockito.doReturn(true).when(TestcontainersConfiguration.getInstance()).environmentSupportsReuse(); + AtomicReference commandRef = new AtomicReference<>(); + String containerId = randomContainerId(); + // The name is taken by a container whose start was killed before it ran + AtomicBoolean orphanRemoved = new AtomicBoolean(false); + when(client.createContainerCmd(any())) + .then(invocation -> { + CreateContainerCmd.Exec exec = command -> { + commandRef.set(command); + if (!orphanRemoved.get()) { + throw new ConflictException("name is already in use"); + } + CreateContainerResponse response = new CreateContainerResponse(); + response.setId(containerId); + return response; + }; + return new CreateContainerCmdImpl(exec, null, "image:latest"); + }); + when(client.listContainersCmd()).then(listContainersAnswer()); + String orphanId = randomContainerId(); + InspectContainerResponse orphan = Mockito.mock(InspectContainerResponse.class, Answers.RETURNS_DEEP_STUBS); + when(orphan.getId()).thenReturn(orphanId); + when(orphan.getState().getStatus()).thenReturn("created"); + when(orphan.getCreated()).thenReturn(Instant.now().minus(Duration.ofMinutes(5)).toString()); + when(client.inspectContainerCmd(ArgumentMatchers.startsWith("testcontainers-reuse-"))) + .then(invocation -> { + when(orphan.getConfig().getLabels()).thenReturn(commandRef.get().getLabels()); + return new InspectContainerCmdImpl(command -> orphan, invocation.getArgument(0)); + }); + RemoveContainerCmd removeCmd = Mockito.mock(RemoveContainerCmd.class, Answers.RETURNS_SELF); + when(removeCmd.exec()) + .then(invocation -> { + orphanRemoved.set(true); + return null; + }); + when(client.removeContainerCmd(orphanId)).thenReturn(removeCmd); + when(client.startContainerCmd(containerId)).then(startContainerAnswer()); + when(client.inspectContainerCmd(containerId)).then(inspectContainerAnswer()); + + container.start(); + + Mockito.verify(removeCmd).exec(); + assertThat(container.getContainerId()).isEqualTo(containerId); + Mockito.verify(client).startContainerCmd(containerId); + } + @Test void shouldHashCopiedFiles() { Mockito.doReturn(true).when(TestcontainersConfiguration.getInstance()).environmentSupportsReuse(); diff --git a/core/src/test/java/org/testcontainers/containers/ReusableContainerConcurrencyTest.java b/core/src/test/java/org/testcontainers/containers/ReusableContainerConcurrencyTest.java new file mode 100644 index 00000000000..47a025e2876 --- /dev/null +++ b/core/src/test/java/org/testcontainers/containers/ReusableContainerConcurrencyTest.java @@ -0,0 +1,116 @@ +package org.testcontainers.containers; + +import com.github.dockerjava.api.DockerClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.TestImages; +import org.testcontainers.utility.MockTestcontainersConfigurationExtension; +import org.testcontainers.utility.TestcontainersConfiguration; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(MockTestcontainersConfigurationExtension.class) +class ReusableContainerConcurrencyTest { + + private final DockerClient dockerClient = DockerClientFactory.instance().client(); + + // Makes the hash unique per test run, so containers from earlier runs are not reused + private final String runId = UUID.randomUUID().toString(); + + @BeforeEach + void enableReuse() { + Mockito.doReturn(true).when(TestcontainersConfiguration.getInstance()).environmentSupportsReuse(); + } + + @AfterEach + void removeContainers() { + dockerClient + .listContainersCmd() + .withShowAll(true) + .withLabelFilter(Collections.singletonMap("test-run-id", runId)) + .exec() + .forEach(it -> dockerClient.removeContainerCmd(it.getId()).withForce(true).exec()); + } + + @Test + void concurrentStartsShareOneContainer() throws Exception { + int starts = 8; + ExecutorService executor = Executors.newFixedThreadPool(starts); + CountDownLatch go = new CountDownLatch(1); + List> futures = new ArrayList<>(); + try { + for (int i = 0; i < starts; i++) { + futures.add( + CompletableFuture.supplyAsync( + () -> { + GenericContainer container = reusableContainer(); + try { + go.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + container.start(); + return container.getContainerId(); + }, + executor + ) + ); + } + go.countDown(); + + Set containerIds = futures.stream().map(CompletableFuture::join).collect(Collectors.toSet()); + + assertThat(containerIds).hasSize(1); + assertThat(containersOfThisRun()).hasSize(1); + } finally { + executor.shutdownNow(); + } + } + + @Test + void replacesStoppedContainer() { + GenericContainer first = reusableContainer(); + first.start(); + dockerClient.stopContainerCmd(first.getContainerId()).withTimeout(0).exec(); + + GenericContainer second = reusableContainer(); + second.start(); + + assertThat(second.getContainerId()).isNotEqualTo(first.getContainerId()); + assertThat(second.isRunning()).isTrue(); + assertThat(containersOfThisRun()).hasSize(1); + } + + private GenericContainer reusableContainer() { + return new GenericContainer<>(TestImages.TINY_IMAGE) + .withCommand("sleep", "infinity") + .withLabel("test-run-id", runId) + .withReuse(true); + } + + private List containersOfThisRun() { + return dockerClient + .listContainersCmd() + .withShowAll(true) + .withLabelFilter(Collections.singletonMap("test-run-id", runId)) + .exec() + .stream() + .map(it -> it.getId()) + .collect(Collectors.toList()); + } +}