diff --git a/modules/binary/api/src/main/java/org/apache/ignite/binary/BinaryRawWriter.java b/modules/binary/api/src/main/java/org/apache/ignite/binary/BinaryRawWriter.java index ffa89ce274ceb..a2c408af6c919 100644 --- a/modules/binary/api/src/main/java/org/apache/ignite/binary/BinaryRawWriter.java +++ b/modules/binary/api/src/main/java/org/apache/ignite/binary/BinaryRawWriter.java @@ -128,6 +128,14 @@ public interface BinaryRawWriter { */ public void writeByteArray(@Nullable byte[] val) throws BinaryObjectException; + /** + * @param val Value to write. + * @param off Offset to start write from. + * @param len Count of bytes to write. + * @throws BinaryObjectException In case of error. + */ + public void writeByteArray(@Nullable byte[] val, int off, int len) throws BinaryObjectException; + /** * @param val Value to write. * @throws BinaryObjectException In case of error. diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java index 033be969abcf7..b3e1042f6d380 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java @@ -904,6 +904,21 @@ void writeBooleanField(@Nullable Boolean val) { } } + /** {@inheritDoc} */ + @Override public void writeByteArray(@Nullable byte[] val, int pos, int len) throws BinaryObjectException { + if (val == null) + out.writeByte(GridBinaryMarshaller.NULL); + else { + if (len > val.length) + throw new IllegalArgumentException("len must be less then val.length"); + + out.unsafeEnsure(1 + 4); + out.unsafeWriteByte(GridBinaryMarshaller.BYTE_ARR); + out.unsafeWriteInt(len); + out.writeByteArray(val, pos, len); + } + } + /** {@inheritDoc} */ @Override public void writeShortArray(String fieldName, @Nullable short[] val) throws BinaryObjectException { writeFieldId(fieldName); diff --git a/modules/commons/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java b/modules/commons/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java index 8bf794062ef45..6c38c62040684 100755 --- a/modules/commons/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java +++ b/modules/commons/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java @@ -2153,4 +2153,18 @@ public static Collection emptyIfNull(@Nullable Collection col) { public static Map emptyIfNull(@Nullable Map map) { return map == null ? Collections.emptyMap() : map; } + + /** + * @param arr Array. + * @param el Element. + * @return Element index or {@code -1} if not found. + */ + public static int indexOf(T[] arr, T el) { + for (int i = 0; i < arr.length; i++) { + if (Objects.equals(arr[i], el)) + return i; + } + + return -1; + } } diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/SecurityCommandHandlerPermissionsTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/SecurityCommandHandlerPermissionsTest.java index 902bc9797d750..2eea8d8b0f745 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/SecurityCommandHandlerPermissionsTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/SecurityCommandHandlerPermissionsTest.java @@ -39,6 +39,7 @@ import org.apache.ignite.compute.ComputeTaskFuture; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.classpath.ClassPathTestUtils; import org.apache.ignite.internal.client.thin.ServicesTest; import org.apache.ignite.internal.managers.systemview.GridSystemViewManager; import org.apache.ignite.internal.processors.security.impl.TestSecurityData; @@ -67,6 +68,7 @@ import static org.apache.ignite.internal.processors.job.GridJobProcessor.JOBS_VIEW; import static org.apache.ignite.internal.processors.task.GridTaskProcessor.TASKS_VIEW; import static org.apache.ignite.internal.util.IgniteUtils.resolveIgnitePath; +import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_OPS; import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_CREATE; import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_DESTROY; import static org.apache.ignite.plugin.security.SecurityPermission.CACHE_READ; @@ -150,6 +152,17 @@ public void testCacheClear() throws Exception { checkCommandPermissions(asList("--cache", "clear", "--caches", DEFAULT_CACHE_NAME), cachePermission(CACHE_REMOVE)); } + /** */ + @Test + public void testClassPathCreate() throws Exception { + String files = String.join(",", ClassPathTestUtils.fileArg(ClassPathTestUtils.files())); + + checkCommandPermissions( + asList("--class-path", "create", "--name", "secured_app", "--files", files), + systemPermissions(ADMIN_OPS) + ); + } + /** */ @Test public void testCacheCreate() throws Exception { diff --git a/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite2.java b/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite2.java index eae0f1e86dcbb..2fad7652666c1 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite2.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/testsuites/IgniteControlUtilityTestSuite2.java @@ -22,6 +22,7 @@ import org.apache.ignite.util.CacheMetricsCommandTest; import org.apache.ignite.util.CdcCommandTest; import org.apache.ignite.util.CdcResendCommandTest; +import org.apache.ignite.util.GridCommandHandlerClassPathTest; import org.apache.ignite.util.GridCommandHandlerConsistencyBinaryTest; import org.apache.ignite.util.GridCommandHandlerConsistencySensitiveTest; import org.apache.ignite.util.GridCommandHandlerConsistencyTest; @@ -74,7 +75,8 @@ SecurityCommandHandlerPermissionsTest.class, - IdleVerifyDumpTest.class + IdleVerifyDumpTest.class, + GridCommandHandlerClassPathTest.class }) public class IgniteControlUtilityTestSuite2 { } diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClassPathTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClassPathTest.java new file mode 100644 index 0000000000000..51c6d40d9594a --- /dev/null +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClassPathTest.java @@ -0,0 +1,422 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.util; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.classpath.ClassPathProcessor; +import org.apache.ignite.internal.classpath.ClassPathTestUtils; +import org.apache.ignite.internal.classpath.IgniteClassPath; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.ListeningTestLogger; +import org.apache.ignite.testframework.LogListener; +import org.junit.Test; + +import static org.apache.ignite.internal.classpath.ClassPathProcessor.metastorageKey; +import static org.apache.ignite.internal.classpath.ClassPathTestUtils.file; +import static org.apache.ignite.internal.classpath.ClassPathTestUtils.fileNames; +import static org.apache.ignite.internal.classpath.IgniteClassPathState.READY; +import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_INVALID_ARGUMENTS; +import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; +import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR; +import static org.apache.ignite.testframework.GridTestUtils.assertContains; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** + * Test for --classpath command. + */ +public class GridCommandHandlerClassPathTest extends GridCommandHandlerAbstractTest { + /** */ + public static final int GRID_CNT = 3; + + /** */ + public static final int FAIL_NODE_IDX = 2; + + /** */ + private Set cpFiles; + + /** */ + private String filesArg; + + /** */ + private ListeningTestLogger lsnrLog; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName).setGridLogger(lsnrLog); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + cpFiles = ClassPathTestUtils.files(); + + filesArg = String.join(",", ClassPathTestUtils.fileArg(cpFiles)); + + cleanPersistenceDir(); + + lsnrLog = new ListeningTestLogger(log); + + startGrids(GRID_CNT); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + // TODO: add CRC or other check of file integrity. + // TODO add in production code checks of files integriy. Perform file integrity check on startup. + + // Support pretty print for command. + // TODO: add MAX file size check, free disk amount check. + + /** Concurrent {@code --create} commands with the same name must result in exactly one success. */ + @Test + public void testConcurrentCreateSameName() throws Exception { + injectTestSystemOut(); + + int cmdCnt = 4; + + LogListener cpReadyLsnr = readyLogListener(GRID_CNT - 1); + + lsnrLog.registerListener(cpReadyLsnr); + + List> fileSets = new ArrayList<>(); + + for (int i = 0; i < cmdCnt; i++) + fileSets.add(Set.of(file(1000L * (i + 1) + i))); + + TestCommandHandler[] hnds = new TestCommandHandler[cmdCnt]; + + for (int i = 0; i < cmdCnt; i++) + hnds[i] = newCommandHandler(createTestLogger()); + + AtomicInteger okCnt = new AtomicInteger(); + AtomicInteger failCnt = new AtomicInteger(); + AtomicInteger idx = new AtomicInteger(); + + Set okFiles = new HashSet<>(); + + CountDownLatch latch = new CountDownLatch(cmdCnt); + + GridTestUtils.runMultiThreadedAsync(() -> { + int i = idx.getAndIncrement(); + + latch.countDown(); + + assertTrue(latch.await(getTestTimeout(), TimeUnit.MILLISECONDS)); + + int res = execute( + hnds[i], + "--class-path", "create", + "--name", cpName(), + "--files", String.join(",", ClassPathTestUtils.fileArg(fileSets.get(i))) + ); + + if (res == EXIT_CODE_OK) { + okCnt.incrementAndGet(); + + okFiles.addAll(fileSets.get(i)); + } + else + failCnt.incrementAndGet(); + + return null; + }, cmdCnt, "concurrent-cp-create").get(getTestTimeout()); + + assertEquals(1, okCnt.get()); + assertEquals(cmdCnt - 1, failCnt.get()); + assertTrue(waitForCondition(cpReadyLsnr::check, 30_000)); + + IgniteClassPath icp = classPath(); + + assertNotNull(icp); + assertEquals(READY, icp.state()); + assertEquals(GRID_CNT, icp.deployedOnNodes().size()); + + assertEquals(fileNames(okFiles), new HashSet<>(Arrays.asList(icp.files()))); + + checkFilesExists(cpName(), -1, okFiles); + } + + /** Tests --create command. */ + @Test + public void testCreateRemove() throws Exception { + injectTestSystemOut(); + + final TestCommandHandler hnd = newCommandHandler(createTestLogger()); + + // Empty root doesn't affect ClassPath creation. + if (commandHandler.equals(JMX_CMD_HND)) + assertTrue(grid(0).context().pdsFolderResolver().fileTree().classPathRoot(cpName()).mkdirs()); + + LogListener cpReadyLsnr = readyLogListener(GRID_CNT - 1); + + lsnrLog.registerListener(cpReadyLsnr); + + assertEquals(EXIT_CODE_OK, execute(hnd, "--class-path", "create", "--name", cpName(), "--files", filesArg)); + + assertTrue(waitForCondition(cpReadyLsnr::check, 30_000)); + + IgniteClassPath icp = classPath(); + + assertEquals(READY, icp.state()); + assertEquals(GRID_CNT, icp.deployedOnNodes().size()); + + ClassPathTestUtils.checkDeployedOn(grid(0), cpName()); + ClassPathTestUtils.checkDeployedOn(grid(1), cpName()); + ClassPathTestUtils.checkDeployedOn(grid(2), cpName()); + + checkFilesExists(cpName(), -1, cpFiles); + + if (commandHandler.equals(JMX_CMD_HND)) + assertEquals(EXIT_CODE_OK, execute(hnd, "--class-path", "remove", "--name", cpName())); + else + assertEquals(EXIT_CODE_OK, execute(hnd, "--class-path", "remove", "--force", "--name", cpName())); + + assertTrue(waitForCondition(() -> { + try { + return classPath() == null; + } + catch (IgniteCheckedException e) { + return false; + } + }, 30_000)); + + assertFalse(grid(0).context().pdsFolderResolver().fileTree().classPathRoot(cpName()).exists()); + assertFalse(grid(1).context().pdsFolderResolver().fileTree().classPathRoot(cpName()).exists()); + assertFalse(grid(2).context().pdsFolderResolver().fileTree().classPathRoot(cpName()).exists()); + } + + /** Tests --create command. */ + @Test + public void testCreateWhenRootExists() throws Exception { + injectTestSystemOut(); + + final TestCommandHandler hnd = newCommandHandler(createTestLogger()); + + File cpRoot = grid(FAIL_NODE_IDX).context().pdsFolderResolver().fileTree().classPathRoot(cpName()); + + assertTrue(cpRoot.mkdirs()); + + File f = new File(cpRoot, ".must_fail_cp_creation"); + + assertTrue(f.createNewFile()); + + LogListener downloadSucceedMsg = readyLogListener(1); + + lsnrLog.registerListener(downloadSucceedMsg); + + assertEquals(EXIT_CODE_OK, execute(hnd, "--class-path", "create", "--name", cpName(), "--files", filesArg)); + + assertTrue(waitForCondition(downloadSucceedMsg::check, 30_000)); + + IgniteClassPath icp = classPath(); + + assertEquals(READY, icp.state()); + assertEquals(2, icp.deployedOnNodes().size()); + + ClassPathTestUtils.checkDeployedOn(grid(0), cpName()); + ClassPathTestUtils.checkDeployedOn(grid(1), cpName()); + + checkFilesExists(cpName(), FAIL_NODE_IDX, cpFiles); + } + + /** */ + @Test + public void testFailWhenFileExists() throws Exception { + injectTestSystemOut(); + + final TestCommandHandler hnd = newCommandHandler(createTestLogger()); + + GridKernalContext ctx = grid(0).context(); + + ctx.distributedMetastorage().listen( + k -> k.equals(metastorageKey(cpName())), + (key, oldVal, newVal) -> + fileNames(cpFiles).forEach(f -> new File(ctx.pdsFolderResolver().fileTree().classPathRoot(cpName()), f).mkdirs()) + ); + + LogListener cpReadyLsnr = null; + + if (!cliCommandHandler()) { + cpReadyLsnr = LogListener + .matches("Failed to copy ClassPath file locally, the ClassPath will be removed") + .times(1) + .build(); + + lsnrLog.registerListener(cpReadyLsnr); + } + + assertEquals( + EXIT_CODE_UNEXPECTED_ERROR, + execute(hnd, "--class-path", "create", "--name", cpName(), "--files", filesArg) + ); + + String out = testOut.toString(); + + if (cliCommandHandler()) + assertTrue(out.contains("Starting to upload files:")); + else + assertTrue(waitForCondition(cpReadyLsnr::check, 30_000)); + + assertTrue(out.contains("File exists")); + assertNull("Metastorage record must be removed", classPath()); + assertFalse( + "Classpath directory must be removed", + grid(0).context().pdsFolderResolver().fileTree().classPathRoot(cpName()).exists() + ); + } + + /** */ + @Test + public void testFailWhenMetastoreExists() throws Exception { + injectTestSystemOut(); + + final TestCommandHandler hnd = newCommandHandler(createTestLogger()); + + assertTrue(grid(0).context().distributedMetastorage().compareAndSet( + metastorageKey(cpName()), + null, + 1 + )); + + assertEquals( + EXIT_CODE_UNEXPECTED_ERROR, + execute(hnd, "--class-path", "create", "--name", cpName(), "--files", filesArg) + ); + + String out = testOut.toString(); + + assertTrue(out.contains("Fail to register ClassPath. Same ClassPath exists, already?")); + assertNotNull(grid(0).context().distributedMetastorage().read(ClassPathProcessor.metastorageKey(cpName()))); + assertFalse( + "Classpath directory must be removed", + grid(0).context().pdsFolderResolver().fileTree().classPathRoot(cpName()).exists() + ); + assertTrue( + "Files must not be copied", + F.isEmpty(grid(0).context().pdsFolderResolver().fileTree().classPathRoot(cpName()).listFiles()) + ); + } + + /** Tests --create command arguments format. */ + @Test + public void testEmptyFilesArgument() { + injectTestSystemOut(); + + assertContains( + log, + executeCommand(EXIT_CODE_INVALID_ARGUMENTS, "--class-path", "create", "--name", "mysuperapp", "--files"), + "Please specify a value for argument: --files" + ); + + assertContains( + log, + executeCommand(EXIT_CODE_INVALID_ARGUMENTS, "--class-path", "create", "--name", "mysuperapp"), + "Mandatory argument(s) missing: [--files]" + ); + + assertContains( + log, + executeCommand(EXIT_CODE_INVALID_ARGUMENTS, "--class-path", "create", "--name", "--files", "some_files"), + "Please specify a value for argument: --name" + ); + + assertContains( + log, + executeCommand(EXIT_CODE_INVALID_ARGUMENTS, "--class-path", "create", "--files", "some_files"), + "Mandatory argument(s) missing: [--name]" + ); + + assertContains( + log, + executeCommand(EXIT_CODE_INVALID_ARGUMENTS, "--class-path", "create", "--name", "mysuperapp", "--files", ""), + cliCommandHandler() ? "File name must not be empty" : "Argument --files required" + ); + + assertContains( + log, + executeCommand(EXIT_CODE_INVALID_ARGUMENTS, "--class-path", "create", "--files", "f.txt,f.txt"), + "Mandatory argument(s) missing: [--name]" + ); + } + + /** */ + @Test + public void testRemoveUnknown() { + injectTestSystemOut(); + + assertContains( + log, + executeCommand(EXIT_CODE_UNEXPECTED_ERROR, "--class-path", "remove", "--name", "unknown"), + "ClassPath not found: unknown" + ); + + assertContains( + log, + executeCommand(EXIT_CODE_UNEXPECTED_ERROR, "--class-path", "remove", "--force", "--name", "unknown"), + "ClassPath not found: unknown" + ); + } + + /** */ + private void checkFilesExists(String cpName, int skip, Set files) throws IOException, IgniteCheckedException { + for (int i = 0; i < GRID_CNT; i++) { + if (skip == i) + continue; + + ClassPathTestUtils.checkFilesExists(grid(i), cpName, files); + } + } + + /** */ + private IgniteClassPath classPath() throws IgniteCheckedException { + return grid(0).context().distributedMetastorage().read(ClassPathProcessor.metastorageKey(cpName())); + } + + /** */ + private static LogListener readyLogListener(int succeed) { + return LogListener + .matches("IgniteClassPath task done [task=download") + .times(succeed) + .build(); + } + + /** */ + private String cpName() { + return "mysuperapp_" + commandHandler; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java index 10ceff0dfbc8d..5cd65c6af8260 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/CoreMessagesProvider.java @@ -23,6 +23,8 @@ import org.apache.ignite.internal.cache.query.index.IndexQueryResultMeta; import org.apache.ignite.internal.cache.query.index.sorted.IndexKeyDefinition; import org.apache.ignite.internal.cache.query.index.sorted.IndexKeyTypeSettings; +import org.apache.ignite.internal.classpath.DownloadClassPathFailureMessage; +import org.apache.ignite.internal.classpath.DownloadClassPathMessage; import org.apache.ignite.internal.management.cache.PartitionKey; import org.apache.ignite.internal.managers.checkpoint.GridCheckpointRequest; import org.apache.ignite.internal.managers.communication.CompressedMessage; @@ -699,6 +701,9 @@ public CoreMessagesProvider(Marshaller dfltMarsh, Marshaller schemaAwareMarsh, C withNoSchema(IgniteComponentFeatureSet.class); withNoSchema(RollingUpgradeClusterData.class); + withNoSchema(DownloadClassPathMessage.class); + withNoSchema(DownloadClassPathFailureMessage.class); + assert msgIdx <= MAX_MESSAGE_ID; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java index cab4d2a814686..87fbd827aeb1d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java @@ -26,6 +26,7 @@ import org.apache.ignite.internal.binary.BinaryMarshaller; import org.apache.ignite.internal.cache.query.index.IndexProcessor; import org.apache.ignite.internal.cache.transform.CacheObjectTransformerProcessor; +import org.apache.ignite.internal.classpath.ClassPathProcessor; import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager; import org.apache.ignite.internal.managers.collision.GridCollisionManager; import org.apache.ignite.internal.managers.communication.GridIoManager; @@ -664,6 +665,11 @@ public interface GridKernalContext extends Iterable { */ public RollingUpgradeProcessor rollingUpgrade(); + /** + * @return Class path processor. + */ + public ClassPathProcessor classPath(); + /** * Executor that is in charge of processing user async continuations. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java index fb3dcdb3412a7..5ee4cd1ed1831 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java @@ -41,6 +41,7 @@ import org.apache.ignite.internal.binary.GridBinaryMarshaller; import org.apache.ignite.internal.cache.query.index.IndexProcessor; import org.apache.ignite.internal.cache.transform.CacheObjectTransformerProcessor; +import org.apache.ignite.internal.classpath.ClassPathProcessor; import org.apache.ignite.internal.maintenance.MaintenanceProcessor; import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager; import org.apache.ignite.internal.managers.collision.GridCollisionManager; @@ -369,6 +370,10 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable @GridToStringExclude private RollingUpgradeProcessor rollUpProc; + /** Classpath processor. */ + @GridToStringExclude + private ClassPathProcessor classPathProc; + /** */ private Thread.UncaughtExceptionHandler hnd; @@ -602,6 +607,8 @@ else if (comp instanceof PerformanceStatisticsProcessor) perfStatProc = (PerformanceStatisticsProcessor)comp; else if (comp instanceof RollingUpgradeProcessor) rollUpProc = (RollingUpgradeProcessor)comp; + else if (comp instanceof ClassPathProcessor) + classPathProc = (ClassPathProcessor)comp; else if (comp instanceof IndexProcessor) indexProc = (IndexProcessor)comp; else if (!(comp instanceof DiscoveryNodeValidationProcessor @@ -1117,6 +1124,11 @@ public void recoveryMode(boolean recoveryMode) { return rollUpProc; } + /** {@inheritDoc} */ + @Override public ClassPathProcessor classPath() { + return classPathProc; + } + /** {@inheritDoc} */ @Override public Executor getAsyncContinuationExecutor() { return config().getAsyncContinuationExecutor() == null diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index 063774e7b13b5..6ddcd7634dc3b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -89,6 +89,7 @@ import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.cache.query.index.IndexProcessor; import org.apache.ignite.internal.cache.transform.CacheObjectTransformerProcessor; +import org.apache.ignite.internal.classpath.ClassPathProcessor; import org.apache.ignite.internal.cluster.ClusterGroupAdapter; import org.apache.ignite.internal.cluster.IgniteClusterEx; import org.apache.ignite.internal.maintenance.MaintenanceProcessor; @@ -1031,6 +1032,7 @@ public void start( // Start the encryption manager after assigning the discovery manager to context, so it will be // able to register custom event listener. startManager(new GridEncryptionManager(ctx)); + startProcessor(new ClassPathProcessor(ctx)); startProcessor(new PdsConsistentIdProcessor(ctx)); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/classpath/ChangeNodesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/classpath/ChangeNodesTask.java new file mode 100644 index 0000000000000..337e626561b02 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/classpath/ChangeNodesTask.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.classpath; + +import java.util.UUID; +import org.apache.ignite.internal.GridKernalContext; + +/** + * Removes or adds {@link #node} to {@link IgniteClassPath#deployedOnNodes()} set. + */ +public class ChangeNodesTask extends ClassPathProcessor.ClassPathTask { + /** Node. */ + private final UUID node; + + /** Add flag. */ + private final boolean add; + + /** */ + private ChangeNodesTask(GridKernalContext ctx, UUID icpId, UUID node, boolean add) { + super(ctx, icpId); + this.node = node; + this.add = add; + } + + /** */ + public static ChangeNodesTask removeNode(GridKernalContext ctx, UUID icpId, UUID node) { + return new ChangeNodesTask(ctx, icpId, node, false); + } + + /** */ + public static ChangeNodesTask addNode(GridKernalContext ctx, UUID icpId, UUID node) { + return new ChangeNodesTask(ctx, icpId, node, true); + } + + /** {@inheritDoc} */ + @Override void start0() { + ctx.classPath() + .modifyInMetastorageAsync(icpId, null, icp -> add ? icp.addDeployedOnNode(node) : icp.removeDeployedOnNode(node), this::stopped) + .listen(this::finishTaskWithFutureResult); + } + + /** {@inheritDoc} */ + @Override String name() { + return add ? "add node" : "remove node"; + } + + /** {@inheritDoc} */ + @Override void ok() { + if (log.isDebugEnabled()) + log.debug("ClassPath nodes changed [icpId=" + icpId + ", node=" + node + ", add=" + add + ']'); + } + + /** {@inheritDoc} */ + @Override void fail(Throwable t) { + log.warning("Fail to change ClassPath nodes [icpId=" + icpId + ", node=" + node + ", add=" + add + ']', t); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/classpath/ChangeStateTask.java b/modules/core/src/main/java/org/apache/ignite/internal/classpath/ChangeStateTask.java new file mode 100644 index 0000000000000..13137cde31b37 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/classpath/ChangeStateTask.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.classpath; + +import java.util.UUID; +import org.apache.ignite.internal.GridKernalContext; + +/** + * Changes {@link IgniteClassPath} state to {@link #state}. + */ +class ChangeStateTask extends ClassPathProcessor.ClassPathTask { + /** New {@link IgniteClassPath} state. */ + private final IgniteClassPathState state; + + /** */ + public ChangeStateTask(GridKernalContext ctx, UUID icpId, IgniteClassPathState state) { + super(ctx, icpId); + + this.state = state; + } + + /** {@inheritDoc} */ + @Override void start0() { + ctx.classPath() + .modifyInMetastorageAsync(icpId, null, icp -> icp.newState(state), this::stopped) + .listen(this::finishTaskWithFutureResult); + } + + /** {@inheritDoc} */ + @Override String name() { + return "change state to " + state; + } + + /** {@inheritDoc} */ + @Override void ok() { + if (log.isDebugEnabled()) + log.debug("ClassPath state changed [icpId=" + icpId + ", newState=" + state + ']'); + } + + /** {@inheritDoc} */ + @Override void fail(Throwable t) { + log.warning("Fail to change ClassPath state [icpId=" + icpId + ", newState=" + state + ']', t); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/classpath/ClassPathChangeListener.java b/modules/core/src/main/java/org/apache/ignite/internal/classpath/ClassPathChangeListener.java new file mode 100644 index 0000000000000..cb7e503211e9e --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/classpath/ClassPathChangeListener.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.classpath; + +import java.io.Serializable; +import java.util.concurrent.RejectedExecutionException; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.classpath.ClassPathProcessor.ClassPathTask; +import org.apache.ignite.internal.processors.metastorage.DistributedMetaStorage; +import org.apache.ignite.internal.processors.metastorage.DistributedMetaStorageListener; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import static org.apache.ignite.internal.classpath.ClassPathProcessor.className; +import static org.apache.ignite.internal.classpath.ClassPathProcessor.isClassPath; +import static org.apache.ignite.internal.classpath.IgniteClassPathState.LOST; +import static org.apache.ignite.internal.classpath.IgniteClassPathState.NEW; + +/** + * Listener of {@link DistributedMetaStorage} updates related to {@link IgniteClassPath} instances. + * Starts {@link ClassPathTask} to handle {@link IgniteClassPath} changes on local node. + * + * @see ClassPathProcessor + */ +class ClassPathChangeListener implements DistributedMetaStorageListener { + /** */ + private final IgniteLogger log; + + /** */ + private final GridKernalContext ctx; + + /** + * @param ctx Grid kernal context. + */ + public ClassPathChangeListener(GridKernalContext ctx) { + this.ctx = ctx; + this.log = ctx.log(ClassPathChangeListener.class); + } + + /** {@inheritDoc} */ + @Override public void onUpdate(@NotNull String key, @Nullable Serializable oldVal, @Nullable Serializable newVal) { + if (!isClassPath(oldVal) || !isClassPath(newVal)) { + log.warning("Wrong data in IgniteClassPath metastorage data " + + "[key=" + key + ", oldVal=" + className(oldVal) + ", newVal=" + className(newVal) + ']'); + + return; + } + + IgniteClassPath oldIcp = (IgniteClassPath)oldVal; + IgniteClassPath newIcp = (IgniteClassPath)newVal; + + if (newIcp == null) { + onRemoveFromMetastorage(oldIcp); + + return; + } + + if (newIcp.deployedOnNodes().isEmpty()) { + onNoDeployNodes(newIcp); + + return; + } + + if (newIcp.equalsWithoutNodes(oldIcp)) { + if (log.isDebugEnabled()) + log.debug("Event ignored. Old and new state equals [old=" + oldIcp + ", new = " + newIcp + ']'); + + return; + } + + onChange(newIcp, oldIcp); + } + + /** + * Handles regular change in {@link IgniteClassPath}. + */ + private void onChange(IgniteClassPath newIcp, IgniteClassPath oldIcp) { + switch (newIcp.state()) { + case NEW: + log.info("IgniteClassPath created. Waiting for READY state to start download file to local node."); + + break; + case READY: + if (oldIcp == null || oldIcp.state() == NEW || oldIcp.state() == LOST) { + if (newIcp.deployedOnNodes().contains(ctx.localNodeId())) { + if (log.isDebugEnabled()) + log.debug("Event ignored. IgniteClassPath deployed on node, already: " + newIcp); + + return; + } + + log.info("IgniteClassPath READY. Starting download to local node."); + + ctx.classPath().addClassPathTask(newIcp, new DownloadTask(ctx, newIcp.id())); + } + else + log.warning("Wrong state change. Ignore [prev=" + oldIcp.state() + ", new=" + newIcp.state() + ']'); + + break; + + case LOST: + // No-op. + + break; + + case REMOVING: + // Don't check anything. Download task may be in progress. + // Cleanup will just raise log warning if local data not exists. + log.info("IgniteClassPath remove started. Local cleanup: " + newIcp); + + ctx.classPath().addClassPathTask(newIcp, CleanupTask.removeFiles(ctx, newIcp)); + ctx.classPath().addClassPathTask(newIcp, ChangeNodesTask.removeNode(ctx, newIcp.id(), ctx.localNodeId())); + + break; + + default: + throw new IllegalArgumentException("Unknown IgniteClassPath state: " + newIcp.state()); + } + } + + /** + * Handle {@link IgniteClassPath} event when it removed from metastorage. + */ + private void onRemoveFromMetastorage(IgniteClassPath icp) { + // Maybe remove with force flag or error while uploading. + // Cancelling all in flight and remove right away. + IgniteInternalFuture cancelFut = ctx.classPath().cancelTasksAndDisallowNew(icp.id(), icp.name(), false); + + try { + ctx.pools().getSystemExecutorService().submit(() -> { + try { + if (log.isDebugEnabled()) + log.debug("Waiting all tasks stoped: " + icp); + + cancelFut.get(); + } + catch (IgniteCheckedException ignore) { + // Expected, because cancelled. + } + + if (log.isDebugEnabled()) + log.debug("All tasks stoped. Removing local files: " + icp); + + if (!ctx.pdsFolderResolver().fileTree().classPathRoot(icp.name()).exists()) { + log.info("IgniteClassPath removed: " + icp); + + return; + } + + // Concurrent class path tasks not run. Cleaning up without queue. + // Mostly harmless (no local data at the moment until force remove). + CleanupTask t = CleanupTask.removeFiles(ctx, icp); + + t.result().listen(() -> log.info("IgniteClassPath removed: " + icp)); + + ctx.classPath().start(t); + }); + } + catch (RejectedExecutionException e) { + log.warning("Can't submit remove files task", e); + } + } + + /** Handle {@link IgniteClassPath} event when no nodes that stores ClassPath files in cluster. */ + private void onNoDeployNodes(IgniteClassPath icp) { + assert icp.deployedOnNodes().isEmpty(); + + switch (icp.state()) { + case READY: + case NEW: + log.warning("All nodes that haves IgniteClassPath files left the grid [icp=" + icp.name() + ']'); + + ctx.classPath().addClassPathTask(icp, new ChangeStateTask(ctx, icp.id(), LOST)); + + break; + + case REMOVING: + log.info("ClassPath removed from all nodes. Removing from metastore"); + + ctx.classPath().cancelTasksAndDisallowNew(icp.id(), icp.name(), true); + + // Remove from metastorage after all nodes cleanup. + CleanupTask t = CleanupTask.removeFromMetastore(ctx, icp); + + t.result().listen(fut -> { + if (fut.error() == null) + log.info("IgniteClassPath removed from metastore: " + icp.name()); + else + log.warning("IgniteClassPath not removed from metastore: " + icp.name(), fut.error()); + }); + + ctx.classPath().startAsync(t); + + break; + case LOST: + // No-op. + + break; + + default: + throw new IllegalStateException("Unknown state: " + icp.state()); + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/classpath/ClassPathFilesTransmissionHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/classpath/ClassPathFilesTransmissionHandler.java new file mode 100644 index 0000000000000..c35e535fe75bc --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/classpath/ClassPathFilesTransmissionHandler.java @@ -0,0 +1,474 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.classpath; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.Queue; +import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; +import java.util.function.Predicate; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.FailureType; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.GridTopic; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException; +import org.apache.ignite.internal.managers.communication.GridIoManager.TransmissionSender; +import org.apache.ignite.internal.managers.communication.GridMessageListener; +import org.apache.ignite.internal.managers.communication.TransmissionCancelledException; +import org.apache.ignite.internal.managers.communication.TransmissionHandler; +import org.apache.ignite.internal.managers.communication.TransmissionMeta; +import org.apache.ignite.internal.managers.communication.TransmissionPolicy; +import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.internal.U; + +import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; +import static org.apache.ignite.events.EventType.EVT_NODE_LEFT; +import static org.apache.ignite.internal.classpath.ClassPathProcessor.ensureNotStopped; +import static org.apache.ignite.internal.classpath.ClassPathProcessor.fromMetastorage; +import static org.apache.ignite.internal.classpath.IgniteClassPathState.READY; +import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL; +import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.SNP_NODE_STOPPING_ERR_MSG; + +/** + * This manager is responsible for requesting and handling files from a remote node. + */ +class ClassPathFilesTransmissionHandler implements TransmissionHandler, GridMessageListener { + /** ClassPath topic to receive files from remote node. */ + static final Object FILES_TOPIC = GridTopic.TOPIC_CLASSLOAD.topic("rmt_files"); + + /** Transmission parameter for {@link IgniteClassPath#id()}. */ + private static final String ICP_ID_PARAM = "icpId"; + + /** Transmission parameter for {@link IgniteClassPath} file name. */ + private static final String NAME_PARAM = "name"; + + /** System discovery message listener. */ + private DiscoveryEventListener discoLsnr; + + /** */ + private volatile DownloadClassPathTask active; + + /** + * Queue of asynchronous tasks to execute. + * Head of queue is taks that currently executing. + */ + private final Queue queue = new ConcurrentLinkedDeque<>(); + + /** Kernal context. */ + private final GridKernalContext ctx; + + /** Logger. */ + private final IgniteLogger log; + + /** {@code true} if the node is stopping. */ + private volatile boolean stopping; + + /** */ + public ClassPathFilesTransmissionHandler(GridKernalContext ctx) { + this.ctx = ctx; + this.log = ctx.log(ClassPathFilesTransmissionHandler.class); + } + + /** + * Downloads {@link IgniteClassPath} files locally from the remote node specified by {@code rmtNodeId}. + * @param rmtNodeId Remote node id. + * @param icp ClassPath. + * @param stopped Stop flag supplier. + * @return Future for download operation. + */ + IgniteInternalFuture downloadLocally(UUID rmtNodeId, IgniteClassPath icp, BooleanSupplier stopped) { + assert !rmtNodeId.equals(ctx.localNodeId()); + + if (log.isInfoEnabled()) + log.info("Start download ClassPath files [icp=" + icp.name() + ", rmtNode=" + rmtNodeId + ']'); + + DownloadClassPathTask task = new DownloadClassPathTask(rmtNodeId, icp, stopped); + + try { + submit(task); + } + catch (Throwable t) { + task.res.onDone(t); + } + + return task.res; + } + + /** Starts handler. */ + synchronized void start() { + ctx.event().addDiscoveryEventListener(discoLsnr = (evt, discoCache) -> { + UUID leftNodeId = evt.eventNode().id(); + + if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED) + onNodeLeft(leftNodeId); + }, EVT_NODE_LEFT, EVT_NODE_FAILED); + + ctx.io().addMessageListener(FILES_TOPIC, this); + ctx.io().addTransmissionHandler(FILES_TOPIC, this); + } + + /** Stopping handler. */ + void stop() { + synchronized (this) { + if (discoLsnr != null) + ctx.event().removeDiscoveryEventListener(discoLsnr); + + ctx.io().removeMessageListener(FILES_TOPIC); + ctx.io().removeTransmissionHandler(FILES_TOPIC); + + stopping = true; + } + + cancelAll(new IgniteException(SNP_NODE_STOPPING_ERR_MSG), r -> true); + } + + /** + * @param nodeId A node left the cluster. + */ + void onNodeLeft(UUID nodeId) { + cancelAll( + new ClusterTopologyCheckedException("The node from which ClassPath files has been requested left the grid"), + r -> r.rmtNodeId.equals(nodeId) + ); + } + + /** + * @param err Task result. + * @param filter Filter to select tasks for cancel. + */ + private synchronized void cancelAll(Throwable err, Predicate filter) { + queue.forEach(r -> cancel(r, err, filter)); + + cancel(active, err, filter); + } + + /** + * @param r Task to cancel. + * @param err Result error. + * @param filter Task filter. + * @return {@code True} if task was canceled. + */ + private synchronized boolean cancel(DownloadClassPathTask r, Throwable err, Predicate filter) { + if (r == null || !filter.test(r)) + return false; + + return r.res.onDone(err); + } + + /** {@inheritDoc} */ + @Override public Consumer chunkHandler(UUID nodeId, TransmissionMeta initMeta) { + throw new UnsupportedOperationException("Loading file by chunks is not supported: " + nodeId); + } + + /** {@inheritDoc} */ + @Override public void onMessage(UUID nodeId, Object msg0, byte plc) { + try { + if (msg0 instanceof DownloadClassPathMessage msg) { + IgniteClassPath icp; + + try { + icp = fromMetastorage(msg.icpId, READY, ctx); + + NodeFileTree ft = ctx.pdsFolderResolver().fileTree(); + + File root = ft.classPathRoot(icp.name()); + + // TODO: make async. + if (!root.exists()) + throw new IgniteException("Classpath root not exists: " + root); + + try (TransmissionSender sndr = ctx.io().openTransmissionSender(nodeId, FILES_TOPIC)) { + for (String name : icp.files()) { + File f = new File(root, name); + + if (!f.exists()) + throw new IgniteException("Classpath file not exists: " + f); + + if (stopping) + throw new IgniteException("Node stopping"); + + sndr.send(f, Map.of(ICP_ID_PARAM, icp.id(), NAME_PARAM, name), TransmissionPolicy.FILE); + + log.info("ClassPath file sent to the node " + + "[icp=" + icp.name() + ", file=" + name + ", rmtNode=" + nodeId + ']'); + } + } + } + catch (Exception t) { + U.error(log, "Error processing ClassPath file request [request=" + msg + ", nodeId=" + nodeId + ']', t); + + try { + ctx.io().sendToCustomTopic( + nodeId, + FILES_TOPIC, + new DownloadClassPathFailureMessage(msg.icpId, t.getMessage()), + SYSTEM_POOL + ); + } + catch (Exception e) { + log.warning("Error notifying node of send error", e); + } + } + } + else if (msg0 instanceof DownloadClassPathFailureMessage msg) { + String errMsg = "File download cancelled. ClassPath operation stopped on the remote node. Error: " + msg.err; + + if (log.isDebugEnabled()) + log.debug(errMsg); + + if (!cancel(active, new IgniteException(errMsg), t -> t.icp.id().equals(msg.icpId))) { + if (log.isDebugEnabled()) { + log.debug("A stale ClassPath failure message has been received. Will be ignored " + + "[fromNodeId=" + nodeId + ", icpId=" + msg.icpId + "]: " + msg.err); + } + } + } + } + catch (Throwable e) { + U.error(log, "Processing ClassPath request from remote node fails with an error", e); + + ctx.failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e)); + } + } + + /** {@inheritDoc} */ + @Override public void onEnd(UUID nodeId) { + DownloadClassPathTask task = active; + + if (!ensureTask(nodeId, task)) + return; + + if (task.stopped.getAsBoolean()) { + task.res.onDone(new IgniteException("Stopped")); + + return; + } + + int filesLeft = task.filesLeft.get(); + + if (filesLeft != 0) { + task.res.onDone(new IllegalStateException("onEnd invoked, but more files left: " + filesLeft + + ", completing download process with an error")); + + return; + } + + task.res.onDone((Void)null); + } + + /** {@inheritDoc} */ + @Override public void onException(UUID nodeId, Throwable ex) { + DownloadClassPathTask task = active; + + if (!ensureTask(nodeId, task)) + return; + + if (task.stopped.getAsBoolean()) { + task.res.onDone(new IgniteException("Stopped")); + + return; + } + + task.res.onDone(ex); + } + + /** {@inheritDoc} */ + @Override public String filePath(UUID nodeId, TransmissionMeta fileMeta) { + UUID icpId = (UUID)fileMeta.params().get(ICP_ID_PARAM); + String name = (String)fileMeta.params().get(NAME_PARAM); + + IgniteClassPath icp = fromMetastorage(icpId, READY, ctx); + + DownloadClassPathTask task = active; + + if (!ensureTask(nodeId, task)) { + throw new TransmissionCancelledException("Stale ClassPath transmission will be ignored " + + "[icpId=" + icp.id() + ", file=" + name + ']'); + } + + if (task.stopped.getAsBoolean()) + throw new TransmissionCancelledException("Task stopped [icpId=" + icp.id() + ", file=" + name + ']'); + + NodeFileTree ft = ctx.pdsFolderResolver().fileTree(); + + File root = ft.classPathRoot(icp.name()); + + NodeFileTree.mkdir(root, "classpath root"); + + return new File(root, name).getAbsolutePath(); + } + + /** {@inheritDoc} */ + @Override public Consumer fileHandler(UUID nodeId, TransmissionMeta initMeta) { + UUID icpId = (UUID)initMeta.params().get(ICP_ID_PARAM); + String name = (String)initMeta.params().get(NAME_PARAM); + + IgniteClassPath icp = fromMetastorage(icpId, READY, ctx); + + return file -> { + DownloadClassPathTask task = active; + + if (!ensureTask(nodeId, task)) { + throw new TransmissionCancelledException("Stale ClassPath transmission will be ignored " + + "[icpId=" + icp.id() + ", file=" + name + ']'); + } + + if (task.stopped.getAsBoolean()) + throw new TransmissionCancelledException("Task stopped [icpId=" + icp.id() + ", file=" + name + ']'); + + int filesLeft = task.filesLeft.decrementAndGet(); + + if (log.isInfoEnabled()) { + log.info("ClassPath file from remote node has been received " + + "[icp=" + task.icp.name() + ", file=" + name + ", filesLeft=" + filesLeft + ']'); + } + }; + } + + /** + * Starts {@code task} or adds it to queue. + * + * @param next Task to execute. + */ + private void submit(DownloadClassPathTask next) { + ClusterNode rmtNode; + + synchronized (this) { + if (stopping) { + next.res.onDone(new IgniteException(SNP_NODE_STOPPING_ERR_MSG)); + + return; + } + + if (active != null && !active.res.isDone()) { + if (!queue.offer(next)) { + next.res.onDone(new IgniteException("Can't put task in queue: " + next.icp)); + } + + return; + } + + rmtNode = ctx.discovery().node(next.rmtNodeId); + + if (rmtNode == null) { + next.res.onDone(new IgniteException("Can't download classpath files. " + + "Remote node left the grid [rmtNodeId=" + rmtNode + ']')); + + return; + } + + active = next; + + next.res.listen(this::onActiveDone); + } + + try { + // submit can be invoked from discovery thread. + // sendOrderedMessage can be blocking so invoke it in separate thread to release discovery. + ctx.pools().getSystemExecutorService().submit(() -> { + try { + ensureNotStopped(next.stopped); + + ctx.cache().context().gridIO().sendOrderedMessage( + rmtNode, + FILES_TOPIC, + new DownloadClassPathMessage(next.icp), + SYSTEM_POOL, + Long.MAX_VALUE, + true + ); + } + catch (Throwable e) { + next.res.onDone(new IgniteException("Can't download classpath files. " + + "Remote node left the grid [rmtNodeId=" + next.rmtNodeId + ']')); + } + }); + } + catch (RejectedExecutionException e) { + next.res.onDone(e); + } + } + + /** Starts next task if exists. */ + private void onActiveDone(IgniteInternalFuture doneFut) { + DownloadClassPathTask next; + + synchronized (this) { + if (active == null || doneFut != active.res) + return; + + active = null; + + next = queue.poll(); + + while (next != null && next.res.isDone()) + next = queue.poll(); + } + + if (next != null) + submit(next); + } + + /** */ + private static boolean ensureTask(UUID nodeId, DownloadClassPathTask task) { + return task != null + && !task.res.isDone() + && task.rmtNodeId.equals(nodeId); + } + + /** + * Task responsible for downloading {@link IgniteClassPath} files from remote node. + */ + private static class DownloadClassPathTask { + /** Node to download files from. */ + final UUID rmtNodeId; + + /** ClassPath to download files for. */ + final IgniteClassPath icp; + + /** Result of download. */ + final GridFutureAdapter res; + + /** Stop flag supplier. */ + final BooleanSupplier stopped; + + /** Files counter. */ + final AtomicInteger filesLeft; + + /** */ + public DownloadClassPathTask(UUID rmtNodeId, IgniteClassPath icp, BooleanSupplier stopped) { + this.rmtNodeId = rmtNodeId; + this.icp = icp; + this.res = new GridFutureAdapter<>(); + this.filesLeft = new AtomicInteger(icp.files().length); + this.stopped = stopped; + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/classpath/ClassPathProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/classpath/ClassPathProcessor.java new file mode 100644 index 0000000000000..48aed04f745dc --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/classpath/ClassPathProcessor.java @@ -0,0 +1,988 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.classpath; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Objects; +import java.util.Properties; +import java.util.Queue; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.RejectedExecutionException; +import java.util.function.BiConsumer; +import java.util.function.BooleanSupplier; +import java.util.function.Function; +import java.util.function.LongSupplier; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.failure.FailureContext; +import org.apache.ignite.failure.FailureType; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener; +import org.apache.ignite.internal.processors.GridProcessorAdapter; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; +import org.apache.ignite.internal.processors.metastorage.DistributedMetastorageLifecycleListener; +import org.apache.ignite.internal.util.future.GridCompoundFuture; +import org.apache.ignite.internal.util.future.GridFinishedFuture; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.A; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.jetbrains.annotations.Nullable; + +import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; +import static org.apache.ignite.events.EventType.EVT_NODE_LEFT; +import static org.apache.ignite.internal.classpath.ChangeNodesTask.addNode; +import static org.apache.ignite.internal.classpath.ChangeNodesTask.removeNode; +import static org.apache.ignite.internal.classpath.IgniteClassPathState.NEW; +import static org.apache.ignite.internal.classpath.IgniteClassPathState.READY; +import static org.apache.ignite.internal.classpath.IgniteClassPathState.REMOVING; +import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.TMP_SUFFIX; + +/** + * TODO: + * 1. How to check data integrity on start? + * Do we want to do this for txt file or for jar only? + * 2. Check and remove obsolete icp from dist on start. + * Do we want to have some flag to skip remove in this case? (if we preparing for ICP registration). + * 3. Should we include CP into snapshots and dumps? + * 4. Should we control file size to ensure disk free space enough to upload CP files? + * TODO: update local file on changes. + */ +public class ClassPathProcessor extends GridProcessorAdapter implements DistributedMetastorageLifecycleListener { + /** */ + private static final Queue> NO_NEW_TASKS = new ConcurrentLinkedQueue<>(); + + /** Prefix for metastorage keys. */ + static final String METASTORE_PREFIX = "icp."; + + /** Handles download requests for {@link IgniteClassPath} files. */ + final ClassPathFilesTransmissionHandler icpFilesHnd; + + /** System discovery message listener. */ + private DiscoveryEventListener discoLsnr; + + /** + * {@link IgniteClassPath} tasks. + * Any actions with the specific {@link IgniteClassPath} instance must be done one by one to ensure local atomicity of changes. + */ + private final ConcurrentMap>> icpTasks = new ConcurrentHashMap<>(); + + /** + * @param ctx Kernal context. + */ + public ClassPathProcessor(GridKernalContext ctx) { + super(ctx); + + icpFilesHnd = new ClassPathFilesTransmissionHandler(ctx); + } + + /** {@inheritDoc} */ + @Override public void start() throws IgniteCheckedException { + icpFilesHnd.start(); + + synchronized (this) { + ctx.event().addDiscoveryEventListener(discoLsnr = (evt, discoCache) -> { + UUID leftNodeId = evt.eventNode().id(); + + try { + ctx.pools().getSystemExecutorService().submit(() -> { + try { + iterateMetastorage( + (key, icp) -> addClassPathTask(icp, removeNode(ctx, icp.id(), leftNodeId)) + ); + } + catch (IgniteCheckedException e) { + log.warning("Distributed metastore iteration error", e); + } + }); + } + catch (RejectedExecutionException e) { + log.warning("System pool rejected task", e); + } + }, EVT_NODE_LEFT, EVT_NODE_FAILED); + } + } + + /** {@inheritDoc} */ + @Override public void onKernalStart(boolean active) throws IgniteCheckedException { + super.onKernalStart(active); + + // Removing stale local data: + // 1. Partially downloaded: descriptor not exists or can't be read. + // 2. Differs from cluster state: id not equals. + // 3. Remove in progress: state REMOVING. + File[] staleRoots = staleLocalRoots(); + + if (!F.isEmpty(staleRoots)) { + for (File cpRoot : staleRoots) + if (!U.delete(cpRoot)) + throw new IgniteException("Cant delete stale ClassPath root: " + cpRoot); + } + + ctx.distributedMetastorage().listen(key -> key.startsWith(METASTORE_PREFIX), new ClassPathChangeListener(ctx)); + + iterateMetastorage((key, icp) -> { + NodeFileTree ft = ctx.pdsFolderResolver().fileTree(); + + IgniteClassPath loc = readClassPathDescriptor(ft.classPathDescriptor(icp.name())); + + // Partially downloaded or locally unknown ClassPath. + if (loc == null) { + addClassPathTask(icp, CleanupTask.removeFiles(ctx, icp)); + + if (icp.state() == READY) + addClassPathTask(icp, new DownloadTask(ctx, icp.id())); + + return; + } + + addClassPathTask(icp, addNode(ctx, icp.id(), ctx.localNodeId())); + }); + } + + /** {@inheritDoc} */ + @Override public void onKernalStop(boolean cancel) { + icpFilesHnd.stop(); + + synchronized (this) { + if (discoLsnr != null) + ctx.event().removeDiscoveryEventListener(discoLsnr); + } + + if (cancel) { + Set ids = icpTasks.keySet(); + + for (UUID id : ids) + cancelTasksAndDisallowNew(id, "unknown", false); + } + } + + /** + * Register new classpath in metastorage it same name not exists. + * Fails if exists. + * + * @param name Class path name. + * @param files Files included. + * @param lengths Files lengths. + * @return Class path id. + */ + public UUID startCreation(String name, String[] files, long[] lengths) throws IgniteCheckedException { + A.ensure(files.length == lengths.length, "wrong arrays lengths"); + A.ensure(U.alphanumericUnderscore(name), "Classpath name must satisfy the following name pattern: a-zA-Z0-9_"); + + for (String file : files) + ensureFilename(file); + + IgniteClassPath icp = new IgniteClassPath(UUID.randomUUID(), Set.of(ctx.localNodeId()), name, files, lengths, NEW); + + createRootAndCheckIsEmpty(icp); + + Boolean metastorageWritten = casToMetastorageAsync(null, icp).get(); + + if (metastorageWritten == null || !metastorageWritten) { + removeClassPathLocally(icp, true); + + throw new IgniteException("Fail to register ClassPath. Same ClassPath exists, already?"); + } + + File cpRoot = ctx.pdsFolderResolver().fileTree().classPathRoot(icp.name()); + + log.info("New IgniteClasspath created [root = " + cpRoot + ", icp=" + icp + ']'); + + return icp.id(); + } + + /** + * Writes {@code batch} to the Ignite Class Path file. + * + * @param icpId ClassPath id. + * @param name File name. + * @param offset Offset to write data to. + * @param batch Batch. + */ + public synchronized void writeFilePartFromClient( + UUID icpId, + String name, + long offset, + byte[] batch + ) { + IgniteClassPath icp = fromMetastorage(icpId, NEW, ctx); + + try { + ensureKnownFilename(name, icp, () -> offset + batch.length); + + File root = ctx.pdsFolderResolver().fileTree().classPathRoot(icp.name()); + + File f = new File(root, name); + + if (offset == 0) { + A.ensure(root.equals(f.getParentFile()), "filename"); + + log.info("Creating new classpath file: " + f); + + if (!f.createNewFile()) + throw new IgniteException("File exists: " + f); + } + + try (RandomAccessFile raf = new RandomAccessFile(f, "rw")) { + if (raf.length() < offset) { + throw new IgniteException("Wrong offset [icp=" + icp.name() + ", lib=" + name + ", " + + "fileLength=" + raf.length() + ", offset=" + offset + ']'); + } + + raf.seek(offset); + raf.write(batch); + } + } + catch (Throwable e) { + log.error("Failed to upload ClassPath file, the ClassPath will be removed " + + "[name=" + icp.name() + ", id=" + icpId + ", file=" + name + ']', e); + + // Cleaning up synchronously. + cleanAll(icp); + + throw new IgniteException(e); + } + } + + /** + * Copies local file to class path directory. + * + * @param icpId ClassPath id. + * @param file File to copy. + */ + public void copyClassPathFileLocally(UUID icpId, Path file) { + IgniteClassPath icp = fromMetastorage(icpId, NEW, ctx); + + try { + String name = file.getFileName().toString(); + + ensureKnownFilename(name, icp, () -> { + try { + return Files.size(file); + } + catch (IOException e) { + throw new IgniteException(e); + } + }); + + File root = ctx.pdsFolderResolver().fileTree().classPathRoot(icp.name()); + + Path f = new File(root, name).toPath(); + + if (Files.exists(f)) { + if (Files.isSameFile(file, f)) { + log.info("Skip copying new classpath file, already there: " + f); + + return; + } + + throw new IgniteException("File exists: " + f); + } + + A.ensure(root.equals(f.toFile().getParentFile()), "filename"); + + log.info("Copying new classpath file: " + f); + + Files.copy(file, f); + } + catch (Throwable e) { + log.error("Failed to copy ClassPath file locally, the ClassPath will be removed " + + "[name=" + icp.name() + ", id=" + icpId + ']', e); + + // Cleaning up synchronously. + cleanAll(icp); + + throw new IgniteException(e); + } + } + + /** */ + private void cleanAll(IgniteClassPath icp) { + try { + removeFromMetastorage(icp, () -> false); + } + catch (IgniteCheckedException e) { + log.warning("Remove from mestastorage fail", e); + } + finally { + removeClassPathLocally(icp, true); + } + } + + /** */ + public void cleanQueue(UUID icpId) { + icpTasks.remove(icpId); + } + + /** */ + public void makeReady(UUID icpId) throws IgniteCheckedException { + IgniteClassPath icp = fromMetastorage(icpId, NEW, ctx); + + ChangeStateTask task = new ChangeStateTask(ctx, icpId, READY); + + addClassPathTask(icp, task); + + task.result().get(); + } + + /** */ + public IgniteInternalFuture remove(String name, boolean force) { + try { + Serializable icp0 = ctx.distributedMetastorage().read(metastorageKey(name)); + + if (icp0 == null || !isClassPath(icp0)) + throw new IgniteException("ClassPath not found: " + name); + + IgniteClassPath icp = (IgniteClassPath)icp0; + + if (force) { + log.warning("Forcefully removing ClassPath: " + icp); + + try { + removeFromMetastorage(icp, () -> false); + + // Cleanup will be performed in {@link ClassPathChangeListener}. + return new GridFinishedFuture<>(); + } + catch (Exception e) { + return new GridFinishedFuture<>(e); + } + finally { + // Cleanup queue after key removed from metastore. + cancelTasksAndDisallowNew(icp.id(), icp.name(), false); + } + } + + if (icp.state() == REMOVING) + return new GridFinishedFuture<>(); + + // Cleanup and metastorage removal will be performed in {@link ClassPathChangeListener}. + ChangeStateTask changeState = new ChangeStateTask(ctx, icp.id(), REMOVING); + + addClassPathTask(icp, changeState); + + return changeState.result(); + } + catch (Exception e) { + log.warning("Error while removing ClassPath", e); + + return new GridFinishedFuture<>(e); + } + } + + /** */ + IgniteInternalFuture cancelTasksAndDisallowNew(UUID icpId, String name, boolean keepQueueLock) { + Queue> tasks = keepQueueLock + ? icpTasks.put(icpId, NO_NEW_TASKS) + : icpTasks.remove(icpId); + + if (!F.isEmpty(tasks)) { + for (ClassPathTask task : tasks) { + log.info("Cancelling task [icp=" + name + ", task=" + task.name() + ']'); + + task.stopped = true; + } + + GridCompoundFuture fut = new GridCompoundFuture<>(); + + for (ClassPathTask task : tasks) + fut.add((IgniteInternalFuture)task.result()); + + return fut.markInitialized(); + } + + return new GridFinishedFuture<>(); + } + + /** */ + void addClassPathTask(IgniteClassPath icp, ClassPathTask task) { + UUID icpId = icp.id(); + + task.result().listen(doneFut -> { + if (task.result().error() == null) + log.info("IgniteClassPath task done [task=" + task.name() + ", icp=" + icp + ", res=" + task.result().result() + ']'); + else + log.warning("IgniteClassPath task failure [task=" + task.name() + ", icp=" + icp + ']', task.result().error()); + + icpTasks.compute(icpId, (ignored, tasks) -> { + if (tasks == NO_NEW_TASKS) { + if (log.isDebugEnabled()) + log.debug("Tasks dropped. No new task allowed for ClassPath. Remove in progress? [icp=" + icp + ']'); + + return NO_NEW_TASKS; + } + + if (tasks == null) + tasks = new ConcurrentLinkedQueue<>(); + + if (tasks.peek() != null && tasks.peek().result() == doneFut) { + tasks.poll(); + + if (!F.isEmpty(tasks)) + startAsync(tasks.peek()); + } + else + tasks.removeIf(t -> t == task); + + return F.isEmpty(tasks) ? null : tasks; + }); + }); + + icpTasks.compute(icpId, (ignored, tasks) -> { + if (tasks == NO_NEW_TASKS) { + if (log.isDebugEnabled()) + log.debug("Tasks dropped. No new task allowed for ClassPath. Remove in progress? [icp=" + icp + ']'); + + return NO_NEW_TASKS; + } + + if (tasks == null) + tasks = new ConcurrentLinkedQueue<>(); + + while (!F.isEmpty(tasks) && tasks.peek().result().isDone()) + tasks.poll(); + + boolean isFirst = F.isEmpty(tasks); + + tasks.add(task); + + if (isFirst) + startAsync(task); + + return tasks; + }); + } + + /** */ + void startAsync(ClassPathTask t) { + try { + ctx.pools().getSystemExecutorService().submit(() -> start(t)); + } + catch (RejectedExecutionException e) { + t.result().onDone(e); + } + } + + /** */ + void start(ClassPathTask t) { + if (ctx.isStopping()) { + t.result().onDone(new IgniteException("Node stopping")); + + return; + } + + try { + t.start(); + } + catch (Throwable e) { + t.result().onDone(e); + } + } + + /** */ + void removeFromMetastorage(IgniteClassPath icp, BooleanSupplier stopped) throws IgniteCheckedException { + String key = metastorageKey(icp.name()); + + int iter = 0; + + while (true) { + ensureNotStopped(stopped); + + Serializable curData = ctx.distributedMetastorage().read(key); + + if (curData == null || !isClassPath(curData) || !Objects.equals(((IgniteClassPath)curData).id(), icp.id())) + break; + + if (ctx.distributedMetastorage().compareAndRemove(key, curData)) + break; + + iter++; + + if (iter == 500) + throw new IgniteException("Too many iterations"); + + if (iter % 100 == 0) + log.warning("Remove operation makes too many iterations. Bug? [icp=" + icp + ", curData=" + curData + ']'); + + } + } + + /** */ + GridFutureAdapter casToMetastorageAsync(@Nullable IgniteClassPath prev, IgniteClassPath icp) { + try { + String key = metastorageKey(icp.name()); + + if (log.isDebugEnabled()) + log.debug("Writing new ClassPath state [new=" + icp + ", prev=" + prev + ']'); + + GridFutureAdapter res = ctx.distributedMetastorage().compareAndSetAsync(key, prev, icp); + + res.listen(casFut -> { + if (casFut.error() == null) + return; + + try { + Object val = ctx.distributedMetastorage().read(key); + + log.warning("Fail to write new ClassPath state [exp=" + prev + ", actual=" + val + ']'); + } + catch (IgniteCheckedException e) { + log.warning("Can't read metastore key", e); + } + }); + + return res; + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + + /** + * Modifies {@link IgniteClassPath} in metastorage with thre provided {@code change}. + * Retries modification if {@link #casToMetastorageAsync(IgniteClassPath, IgniteClassPath)} failed. + * If {@code change} returns same {@link IgniteClassPath} no updates will be done. + * If {@code change} or read from metastorge throws then operation will finish with the corresponding exception. + * @return Future with the new {@link IgniteClassPath} instance. + */ + public GridFutureAdapter modifyInMetastorageAsync( + UUID icpId, + @Nullable IgniteClassPathState state, + Function change, + BooleanSupplier stopped + ) { + GridFutureAdapter res = new GridFutureAdapter<>(); + + modifyWithRetriesAsync(icpId, state, change, res, stopped, 100); + + return res; + } + + /** */ + private void modifyWithRetriesAsync( + UUID icpId, + @Nullable IgniteClassPathState state, + Function change, + GridFutureAdapter res, + BooleanSupplier stopped, + int hardLimit + ) { + if (hardLimit == 0) { + log.error("Reached hard limit on CAS attempts. Fail operation [icpId=" + icpId + ", state=" + state + ']'); + + res.onDone(new IgniteException("Hard limit of CAS reached")); + + return; + } + + try { + ensureNotStopped(stopped); + + IgniteClassPath prev = fromMetastorage(icpId, state, ctx); + IgniteClassPath icp = change.apply(prev); + + if (Objects.equals(prev, icp)) { + res.onDone(icp); + + return; + } + + if (log.isDebugEnabled()) + log.debug("Trying to CAS [prev=" + prev + ", new=" + icp + ", hardLmit=" + hardLimit + ']'); + + casToMetastorageAsync(prev, icp).listen(casRes -> { + if (casRes.error() != null) { + res.onDone(casRes.error()); + + return; + } + + boolean metastorageWritten = casRes.result() != null && casRes.result(); + + if (metastorageWritten) { + res.onDone(icp); + + return; + } + + // Concurrent node modifies first? It's OK, trying to repeat. + modifyWithRetriesAsync(icpId, state, change, res, stopped, hardLimit - 1); + + }); + } + catch (Exception e) { + res.onDone(e); + } + } + + /** + * @param icpId ClassPath id. + * @param ctx Kernal context. + * @return Class path. + */ + static IgniteClassPath fromMetastorage(UUID icpId, @Nullable IgniteClassPathState expState, GridKernalContext ctx) { + try { + IgniteClassPath[] icp = new IgniteClassPath[1]; + + ctx.distributedMetastorage().iterate(METASTORE_PREFIX, (key, icp0) -> { + if (icpId.equals(((IgniteClassPath)icp0).id())) + icp[0] = (IgniteClassPath)icp0; + }); + + if (icp[0] == null) + throw new IgniteException("ClassPath not found: " + icpId); + + if (expState != null && icp[0].state() != expState) + throw new IgniteException("ClassPath in wrong state [expected=" + expState + ", status=" + icp[0].state() + ']'); + + return icp[0]; + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + + /** */ + public static String metastorageKey(String name) { + return METASTORE_PREFIX + name; + } + + /** */ + private static void ensureFilename(String file) { + Path path = Path.of(file); + + A.ensure(path.getNameCount() == 1 && !path.isAbsolute(), "simple filename expected"); + } + + /** */ + private static void ensureKnownFilename(String name, IgniteClassPath icp, LongSupplier lastByteOffset) { + ensureFilename(name); + + int idx = F.indexOf(icp.files(), name); + + if (idx == -1) + throw new IllegalArgumentException("Unknown lib [icp=" + icp.name() + ", unknown_lib=" + name + ']'); + + long length = icp.lengths()[idx]; + + if (lastByteOffset.getAsLong() > length) + throw new IllegalArgumentException("Unexpected file offset [icp=" + icp.name() + ", file=" + name + ']'); + } + + /** + * Iterates all existing {@link IgniteClassPath} in metastorage. + * @param action Action to invoke for each {@link IgniteClassPath} instance. + * @throws IgniteCheckedException In case of error. + */ + private void iterateMetastorage(BiConsumer action) throws IgniteCheckedException { + ctx.distributedMetastorage().iterate(METASTORE_PREFIX, (key, val) -> { + if (!isClassPath(val)) { + log.warning("Wrong data in IgniteClassPath metastorage data [key=" + key + ", val=" + (val) + ']'); + + return; + } + + action.accept(key, (IgniteClassPath)val); + }); + } + + /** */ + public static boolean isClassPath(Serializable icp) { + return icp == null || icp instanceof IgniteClassPath; + } + + /** */ + public static String className(@Nullable Serializable oldVal) { + return oldVal == null ? null : oldVal.getClass().getName(); + } + + /** */ + public static void writeClassPathDescriptor(NodeFileTree ft, IgniteClassPath icp) throws IOException { + File desc = ft.classPathDescriptor(icp.name()); + + File tmp = new File(desc.getParentFile(), desc.getName() + TMP_SUFFIX); + + try (FileOutputStream fos = new FileOutputStream(tmp)) { + icp.toProperties().store(fos, null); + + fos.getFD().sync(); + } + + Files.move(tmp.toPath(), desc.toPath(), StandardCopyOption.ATOMIC_MOVE); + } + + /** */ + public @Nullable IgniteClassPath readClassPathDescriptor(File desc) { + if (!desc.exists()) + return null; + + try (FileInputStream fis = new FileInputStream(desc)) { + Properties p = new Properties(); + + p.load(fis); + + return IgniteClassPath.fromProperties(p, ctx.localNodeId()); + } + catch (Exception e) { + log.warning("Can't read ClassPath descriptor", e); + + return null; + } + } + + /** */ + synchronized void createRootAndCheckIsEmpty(IgniteClassPath icp) { + File cpRoot = ctx.pdsFolderResolver().fileTree().classPathRoot(icp.name()); + + if (!cpRoot.exists()) + NodeFileTree.mkdir(cpRoot, "Ignite Class Path root"); + else if (!F.isEmpty(cpRoot.listFiles())) + throw new IgniteException("ClassPath root exists and not empty: " + cpRoot); + + try { + guardFile(cpRoot, icp.id()).createNewFile(); + } + catch (IOException e) { + throw new IgniteException(e); + } + } + + /** */ + synchronized void removeClassPathLocally(IgniteClassPath icp, boolean force) { + File cpRoot = ctx.pdsFolderResolver().fileTree().classPathRoot(icp.name()); + + if (!guardFile(cpRoot, icp.id()).exists() && !force) { + log.debug("No guard file found. Skip local data removal. " + + "ClassPath not presented locally or concurrently recreated? [icp=" + icp + ']'); + + return; + } + + if (!cpRoot.exists() || U.delete(cpRoot)) + return; + + String err = "Can't delete local files. Remove manually [dir=" + cpRoot + ']'; + + log.warning(err); + + ctx.failure().process(new FailureContext(FailureType.CRITICAL_ERROR, new IgniteException(err))); + } + + /** */ + public static File guardFile(File cpRoot, UUID icpId) { + return new File(cpRoot, "." + icpId); + } + + /** + * Checks local ClassPath roots with the cluster state. + * + * @return Array of local ClassPath root directories to remove on start. + */ + private File[] staleLocalRoots() { + NodeFileTree ft = ctx.pdsFolderResolver().fileTree(); + + return ft.classPathRoot().listFiles(cpRoot -> { + if (!cpRoot.isDirectory()) + return false; + + try { + IgniteClassPath loc = readClassPathDescriptor(ft.classPathDescriptor(cpRoot.getName())); + + if (loc == null) + return true; + + Serializable fromMetastore = ctx.distributedMetastorage().read(metastorageKey(loc.name())); + + if (fromMetastore == null) { + log.info("Unknown local data. Removing [icp=" + loc + ']'); + + return true; + } + + if (!isClassPath(fromMetastore)) { + log.warning("Wrong data in IgniteClassPath metastorage data " + + "[key=" + metastorageKey(loc.name()) + ", val=" + fromMetastore + ']'); + + return true; + } + + IgniteClassPath rmt = (IgniteClassPath)fromMetastore; + + if (!Objects.equals(loc.id(), rmt.id()) || rmt.state() == REMOVING) { + log.info("Stale local data. Removing [loc=" + loc + ", rmt=" + rmt + ']'); + + return true; + } + } + catch (Exception e) { + log.warning("Error filter local root: " + cpRoot, e); + + return true; + } + + return false; + }); + } + + /** */ + static void ensureNotStopped(BooleanSupplier stopped) { + if (stopped.getAsBoolean()) + throw new IgniteException("Stopped"); + } + + /** */ + abstract static class ClassPathTask { + /** */ + private final GridFutureAdapter res = new GridFutureAdapter(); + + /** */ + protected final GridKernalContext ctx; + + /** */ + protected final UUID icpId; + + /** */ + protected final IgniteLogger log; + + /** */ + volatile boolean stopped; + + /** */ + protected ClassPathTask(GridKernalContext ctx, UUID icpId) { + this.ctx = ctx; + this.icpId = icpId; + this.log = ctx.log(getClass()); + + result().listen(resFut -> { + try { + if (resFut.error() == null) + ok(); + else + fail(resFut.error()); + } + finally { + ClassPathTask t = this; + + ctx.classPath().icpTasks.compute(icpId, (ignored, tasks) -> { + if (tasks == null) + return null; + + tasks.remove(t); + + return F.isEmpty(tasks) ? null : tasks; + }); + } + }); + } + + /** */ + void finishTaskWithFutureResult(IgniteInternalFuture action) { + if (action.error() == null) { + if (updateDescriptor(action.result())) + result().onDone(); + } + else + result().onDone(action.error()); + } + + /** + * Starts task execution. + * Must be nonblocking and fast. + * In case any exception throwed, {@link #result()} will be finished and task treated as done. + */ + final void start() { + if (stopped) { + result().onDone(new IgniteException("Stopped before start")); + + return; + } + + try { + start0(); + } + catch (Throwable e) { + res.onDone(e); + } + } + + /** + * Starts task execution. + * Must be nonblocking and fast. + * In case any exception throwed, {@link #result()} will be finished and task treated as done. + */ + abstract void start0(); + + /** + * @return Task name. + */ + abstract String name(); + + /** + * @return Task results. + */ + GridFutureAdapter result() { + return res; + } + + /** */ + boolean stopped() { + return stopped; + } + + /** */ + protected boolean updateDescriptor(IgniteClassPath icp) { + if (!ctx.pdsFolderResolver().fileTree().classPathRoot(icp.name()).exists()) { + log.warning("ClassPath root not exists: " + icp.name()); + + return true; + } + + try { + writeClassPathDescriptor(ctx.pdsFolderResolver().fileTree(), icp); + + return true; + } + catch (Exception e) { + result().onDone(e); + + return false; + } + } + + /** */ + abstract void ok(); + + /** */ + abstract void fail(Throwable t); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/classpath/CleanupTask.java b/modules/core/src/main/java/org/apache/ignite/internal/classpath/CleanupTask.java new file mode 100644 index 0000000000000..b843a61b36745 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/classpath/CleanupTask.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.classpath; + +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.GridKernalContext; + +/** + * Removes {@link IgniteClassPath} local files. + */ +class CleanupTask extends ClassPathProcessor.ClassPathTask { + /** */ + private final IgniteClassPath icp; + + /** */ + private final boolean rmvFromMetastore; + + /** */ + private CleanupTask(GridKernalContext ctx, IgniteClassPath icp, boolean rmvFromMetastore) { + super(ctx, icp.id()); + this.icp = icp; + this.rmvFromMetastore = rmvFromMetastore; + } + + /** @return Task to clean up local files. */ + public static CleanupTask removeFiles(GridKernalContext ctx, IgniteClassPath icp) { + return new CleanupTask(ctx, icp, false); + } + + /** @return Task to remove metastore record. */ + public static CleanupTask removeFromMetastore(GridKernalContext ctx, IgniteClassPath icp) { + return new CleanupTask(ctx, icp, true); + } + + /** {@inheritDoc} */ + @Override void start0() { + if (rmvFromMetastore) { + try { + ctx.classPath().removeFromMetastorage(icp, this::stopped); + } + catch (IgniteCheckedException e) { + result().onDone(e); + + return; + } + } + else + ctx.classPath().removeClassPathLocally(icp, false); + + result().onDone(); + } + + /** {@inheritDoc} */ + @Override String name() { + return "cleanup " + (rmvFromMetastore ? "metastore" : "local files"); + } + + /** {@inheritDoc} */ + @Override void ok() { + if (log.isDebugEnabled()) + log.debug("ClassPath cleanup done [icp=" + icp + ", rmvFromMetastore=" + rmvFromMetastore + ']'); + } + + /** {@inheritDoc} */ + @Override void fail(Throwable t) { + if (log.isDebugEnabled()) + log.debug("Fail to cleanup ClassPath [icp=" + icp + ", rmvFromMetastore=" + rmvFromMetastore + "]: " + t.getMessage()); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/classpath/DownloadClassPathFailureMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/classpath/DownloadClassPathFailureMessage.java new file mode 100644 index 0000000000000..5fd7811808fcd --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/classpath/DownloadClassPathFailureMessage.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.classpath; + +import java.util.UUID; +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.managers.communication.TransmissionHandler; +import org.apache.ignite.plugin.extensions.communication.Message; + +/** + * Message sent by node in case it can't send classpath to receiver. + * + * @see TransmissionHandler + * @see ClassPathFilesTransmissionHandler + * @see DownloadClassPathFailureMessage + */ +public class DownloadClassPathFailureMessage implements Message { + /** + * Classpath ID. + * + * @see IgniteClassPath#id() + */ + @Order(0) + UUID icpId; + + /** Error message. */ + @Order(1) + String err; + + /** */ + public DownloadClassPathFailureMessage() { + // No-op. + } + + /** */ + public DownloadClassPathFailureMessage(UUID icpId, String err) { + this.icpId = icpId; + this.err = err; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/classpath/DownloadClassPathMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/classpath/DownloadClassPathMessage.java new file mode 100644 index 0000000000000..9a3f17ae99f92 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/classpath/DownloadClassPathMessage.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.classpath; + +import java.util.UUID; +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.managers.communication.TransmissionHandler; +import org.apache.ignite.plugin.extensions.communication.Message; + +/** + * Message to initiate download of classpath files from remote node. + * + * @see TransmissionHandler + * @see ClassPathFilesTransmissionHandler + * @see DownloadClassPathFailureMessage + */ +public class DownloadClassPathMessage implements Message { + /** + * Classpath ID. + * + * @see IgniteClassPath#id() + */ + @Order(0) + UUID icpId; + + /** */ + public DownloadClassPathMessage() { + // No-op. + } + + /** */ + public DownloadClassPathMessage(IgniteClassPath icp) { + this.icpId = icp.id(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/classpath/DownloadTask.java b/modules/core/src/main/java/org/apache/ignite/internal/classpath/DownloadTask.java new file mode 100644 index 0000000000000..59b0558c8b743 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/classpath/DownloadTask.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.classpath; + +import java.util.UUID; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.util.typedef.F; + +import static org.apache.ignite.internal.classpath.ClassPathProcessor.fromMetastorage; +import static org.apache.ignite.internal.classpath.IgniteClassPathState.READY; + +/** + * Download {@link IgniteClassPath} files to local node from random node that has files, already. + * + * @see ClassPathFilesTransmissionHandler + */ +class DownloadTask extends ClassPathProcessor.ClassPathTask { + /** + * @param ctx Kernal context. + * @param icpId Ignite class path. + */ + protected DownloadTask(GridKernalContext ctx, UUID icpId) { + super(ctx, icpId); + } + + /** {@inheritDoc} */ + @Override public void start0() { + IgniteClassPath icp = fromMetastorage(icpId, READY, ctx); + + if (icp.deployedOnNodes().contains(ctx.localNodeId())) { + if (log.isDebugEnabled()) + log.debug("Skip download ClassPath files. Node has files, already [icp=" + icp.name() + ']'); + + result().onDone(); + + return; + } + + if (icp.deployedOnNodes().isEmpty()) { + result().onDone(new IgniteException("Deployed on nodes empty. Can't download files: " + icpId)); + + return; + } + + ctx.classPath().createRootAndCheckIsEmpty(icp); + + UUID rmtNode = F.rand(icp.deployedOnNodes()); + + if (stopped) { + result().onDone(new IgniteException("Download task stopped")); + + return; + } + + IgniteInternalFuture downloadRes = ctx.classPath().icpFilesHnd.downloadLocally(rmtNode, icp, this::stopped); + + downloadRes.listen(f -> { + if (f.error() != null) { + result().onDone(f.error()); + + return; + } + + if (log.isDebugEnabled()) + log.debug("ClassPath files from remote node has been fully received [icp=" + icp.name() + ']'); + + ctx.classPath() + .modifyInMetastorageAsync(icpId, READY, state -> state.addDeployedOnNode(ctx.localNodeId()), this::stopped) + .listen(this::finishTaskWithFutureResult); + }); + } + + /** {@inheritDoc} */ + @Override public void ok() { + if (log.isDebugEnabled()) + log.debug("ClassPath files downloaded [icpId=" + icpId + ']'); + } + + /** {@inheritDoc} */ + @Override public void fail(Throwable t) { + try { + IgniteClassPath icp = fromMetastorage(icpId, READY, ctx); + + log.warning("Failed to download ClassPath files [icp=" + icp.name() + "]", t); + + // Cleanup local files. + ctx.classPath().addClassPathTask(icp, CleanupTask.removeFiles(ctx, icp)); + } + catch (Exception e) { + log.warning("onDowloadFailed", e); + } + } + + /** {@inheritDoc} */ + @Override public String name() { + return "download"; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/classpath/IgniteClassPath.java b/modules/core/src/main/java/org/apache/ignite/internal/classpath/IgniteClassPath.java new file mode 100644 index 0000000000000..c4064ec6b5f22 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/classpath/IgniteClassPath.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.classpath; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import java.util.UUID; +import org.apache.ignite.internal.util.typedef.internal.S; + +import static org.apache.ignite.internal.classpath.IgniteClassPathState.READY; + +/** + * Class path POJO. + */ +public class IgniteClassPath implements Serializable { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private UUID id; + + /** */ + private Set deployedOnNodes; + + /** */ + private String name; + + /** */ + private String[] files; + + /** */ + private long[] lengths; + + /** */ + private IgniteClassPathState state; + + /** + * @param id Unique id of classpath. + * @param name User provided name. + * @param files Files to include to classpath. + */ + public IgniteClassPath( + UUID id, + Set deployedOnNodes, + String name, + String[] files, + long[] lengths, + IgniteClassPathState state + ) { + assert files.length == lengths.length; + + this.id = id; + this.deployedOnNodes = deployedOnNodes; + this.name = name; + this.files = files; + this.lengths = lengths; + this.state = state; + } + + /** */ + public static IgniteClassPath fromProperties(Properties p, UUID locNodeId) { + int sz = 0; + + while (p.containsKey("files." + sz)) + sz++; + + String[] files = new String[sz]; + long[] lengths = new long[sz]; + + for (int i = 0; i < sz; i++) { + files[i] = p.getProperty("files." + i); + lengths[i] = Long.parseLong(p.getProperty("length." + i)); + } + + return new IgniteClassPath( + UUID.fromString(p.getProperty("id")), + Set.of(locNodeId), + p.getProperty("name"), + files, + lengths, + IgniteClassPathState.valueOf(p.getProperty("state")) + ); + } + + /** */ + public Properties toProperties() { + Properties p = new Properties(); + + p.setProperty("id", id.toString()); + p.setProperty("name", name); + p.setProperty("state", state.name()); + + for (int i = 0; i < files.length; i++) { + p.setProperty("files." + i, files[i]); + p.setProperty("length." + i, Long.toString(lengths[i])); + } + + return p; + } + + /** + * @return {@code True} if {@code this} and {@code that} instances has all fields equals except {@link #deployedOnNodes}. + */ + public boolean equalsWithoutNodes(IgniteClassPath that) { + return equals(that, false); + } + + /** */ + private boolean equals(IgniteClassPath that, boolean includeNodes) { + if (that == null) + return false; + + return Objects.equals(id, that.id) + && (!includeNodes || Objects.equals(deployedOnNodes, that.deployedOnNodes)) + && Objects.equals(name, that.name) + && Objects.deepEquals(files, that.files) + && Objects.deepEquals(lengths, that.lengths) + && state == that.state; + } + + /** + * @param state New state. + */ + IgniteClassPath newState(IgniteClassPathState state) { + if (this.state == state) + return this; + + return new IgniteClassPath(id, deployedOnNodes, name, files, lengths, state); + } + + /** + * Adds {@code node} to {@link #deployedOnNodes()} set and returns new instance. + * @param node Node to add. + * @return Instance with modified set. + */ + IgniteClassPath addDeployedOnNode(UUID node) { + if (deployedOnNodes.contains(node)) + return this; + + Set deployedOnNodes0 = new HashSet<>(deployedOnNodes); + + deployedOnNodes0.add(node); + + IgniteClassPathState newState = state == IgniteClassPathState.LOST ? READY : state; + + return new IgniteClassPath(id, deployedOnNodes0, name, files, lengths, newState); + } + + /** + * Removes {@code node} from {@link #deployedOnNodes()} set and returns new instance. + * @param node Node to add. + * @return Instance with modified set. + */ + public IgniteClassPath removeDeployedOnNode(UUID node) { + if (!deployedOnNodes.contains(node)) + return this; + + Set deployedOnNodes0 = new HashSet<>(deployedOnNodes); + + deployedOnNodes0.remove(node); + + IgniteClassPathState newState = (state == READY && deployedOnNodes0.isEmpty()) ? IgniteClassPathState.LOST : state; + + return new IgniteClassPath(id, deployedOnNodes0, name, files, lengths, newState); + } + + /** */ + public IgniteClassPathState state() { + return state; + } + + /** */ + public UUID id() { + return id; + } + + /** */ + public Set deployedOnNodes() { + return deployedOnNodes; + } + + /** */ + public String name() { + return name; + } + + /** */ + public String[] files() { + return files; + } + + /** */ + public long[] lengths() { + return lengths; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(IgniteClassPath.class, this); + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + IgniteClassPath that = (IgniteClassPath)o; + return equals(that, true); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return Objects.hash(id, deployedOnNodes, name, Arrays.hashCode(files), Arrays.hashCode(lengths), state); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/classpath/IgniteClassPathState.java b/modules/core/src/main/java/org/apache/ignite/internal/classpath/IgniteClassPathState.java new file mode 100644 index 0000000000000..5f3a3f0d26010 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/classpath/IgniteClassPathState.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.classpath; + +/** State of {@link IgniteClassPath}. */ +public enum IgniteClassPathState { + /** Creationg process in progress. */ + NEW, + + /** Ready for usage. */ + READY, + + /** There are no nodes with ClassPath files. */ + LOST, + + /** Marked for removal. Newly started code can't use corresponding classpath. */ + REMOVING +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/IgniteCommandRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/management/IgniteCommandRegistry.java index f7c9ea9c57fcb..7609cbaa971f5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/IgniteCommandRegistry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/IgniteCommandRegistry.java @@ -25,6 +25,7 @@ import org.apache.ignite.internal.management.cache.CacheCommand; import org.apache.ignite.internal.management.cdc.CdcCommand; import org.apache.ignite.internal.management.checkpoint.CheckpointCommand; +import org.apache.ignite.internal.management.classpath.ClassPathCommand; import org.apache.ignite.internal.management.consistency.ConsistencyCommand; import org.apache.ignite.internal.management.defragmentation.DefragmentationCommand; import org.apache.ignite.internal.management.diagnostic.DiagnosticCommand; @@ -77,7 +78,8 @@ public IgniteCommandRegistry() { new PerformanceStatisticsCommand(), new CdcCommand(), new ConsistencyCommand(), - new EventCommand() + new EventCommand(), + new ClassPathCommand() ); U.loadService(CommandsProvider.class).forEach(p -> p.commands().forEach(this::register)); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathCommand.java new file mode 100644 index 0000000000000..b253c687f2765 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathCommand.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.classpath; + +import org.apache.ignite.internal.classpath.IgniteClassPath; +import org.apache.ignite.internal.management.api.CommandRegistryImpl; + +/** Command to manage {@link IgniteClassPath}. */ +public class ClassPathCommand extends CommandRegistryImpl { + /** */ + public ClassPathCommand() { + super( + new ClassPathCreateCommand(), + new ClassPathRemoveCommand() + ); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathCreateCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathCreateCommand.java new file mode 100644 index 0000000000000..eba7fe83498f4 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathCreateCommand.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.classpath; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; +import org.apache.ignite.Ignite; +import org.apache.ignite.client.IgniteClient; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.classpath.IgniteClassPath; +import org.apache.ignite.internal.client.thin.TcpIgniteClient; +import org.apache.ignite.internal.management.api.CommandUtils; +import org.apache.ignite.internal.management.api.NativeCommand; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.A; +import org.jetbrains.annotations.Nullable; + +/** + * Ignite classpath creation command. + * + * @see IgniteClassPath + */ +public class ClassPathCreateCommand implements NativeCommand { + /** {@inheritDoc} */ + @Override public String description() { + return "Create ClassPath instance from set of local file"; + } + + /** {@inheritDoc} */ + @Override public Class argClass() { + return ClassPathCreateCommandArg.class; + } + + /** {@inheritDoc} */ + @Override public Void execute( + @Nullable IgniteClient client, + @Nullable Ignite ignite, + ClassPathCreateCommandArg arg, + Consumer printer + ) throws Exception { + if (client == null && !(ignite instanceof IgniteEx)) + throw new IllegalStateException("Client or IgniteEx required"); + + List files = prepareFiles(arg); + + ClusterNode uploadNode = uploadNode(client, ignite); + + printer.accept("Upload node: " + uploadNode.id()); + + UUID icpID = CommandUtils.execute(client, ignite, ClassPathStartCreationTask.class, arg, Collections.singletonList(uploadNode)); + + printer.accept("New ClassPath created [uploadNode=" + uploadNode.id() + ", name=" + arg.name + ", id=" + icpID.toString() + ']'); + + uploadFiles(client, ignite, printer, files, uploadNode, icpID); + + CommandUtils.execute(client, ignite, SetClassPathStateReadyTask.class, icpID, Collections.singletonList(uploadNode)); + + return null; + } + + /** */ + private static void uploadFiles( + @Nullable IgniteClient client, + @Nullable Ignite ignite, + Consumer printer, + List files, + ClusterNode uploadNode, + UUID icpId + ) throws IOException { + printer.accept("Starting to upload files:"); + + // TODO: add pretty print here. + for (Path file : files) { + printer.accept(String.valueOf(file.toAbsolutePath())); + + if (client != null) + ((TcpIgniteClient)client).uploadClasspathFile(uploadNode, icpId, file); + else { + ((IgniteEx)ignite).context().classPath().copyClassPathFileLocally(icpId, file); + } + + printer.accept("DONE"); + } + } + + /** */ + private static List prepareFiles(ClassPathCreateCommandArg arg) throws IOException { + List files = new ArrayList<>(arg.files.length); + Set names = new HashSet<>(); + + arg.lengths = new long[arg.files.length]; + + for (int i = 0; i < arg.files.length; i++) { + A.notEmpty(arg.files[i], "File name"); + + Path f = Path.of(arg.files[i]); + + if (!Files.exists(f) || Files.isDirectory(f)) + throw new IllegalArgumentException("File not exists or directory: " + f); + + if (!names.add(f.getFileName().toString())) + throw new IllegalArgumentException("Duplicate file name: " + f.getFileName()); + + files.add(f); + + arg.lengths[i] = Files.size(f); + + // Don't want to send full path to server nodes. + // Server nodes require files names, only. + arg.files[i] = f.getFileName().toString(); + } + + return files; + } + + /** */ + private static ClusterNode uploadNode(IgniteClient client, Ignite ignite) { + if (client != null) { + List nodes = ((TcpIgniteClient)client).connectedToNodes(); + + if (F.isEmpty(nodes)) + throw new IllegalStateException("Not connected to node"); + + ClusterNode node = client.cluster().node(F.first(nodes)); + + if (node == null) + throw new IllegalStateException("Selected node leave the cluster: "); + + return node; + } + + return ignite.cluster().localNode(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathCreateCommandArg.java b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathCreateCommandArg.java new file mode 100644 index 0000000000000..586654424a80d --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathCreateCommandArg.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.classpath; + +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.management.api.Argument; + +/** */ +public class ClassPathCreateCommandArg extends IgniteDataTransferObject { + /** */ + private static final long serialVersionUID = 0; + + /** */ + @Order(0) + @Argument(description = "Name of the classpath") + String name; + + /** */ + @Order(1) + @Argument(description = "Files to add to classpath") + String[] files; + + /** */ + @Order(2) + long[] lengths; + + /** */ + public String name() { + return name; + } + + /** */ + public void name(String name) { + this.name = name; + } + + /** */ + public String[] files() { + return files; + } + + /** */ + public void files(String[] files) { + this.files = files; + } + + /** */ + public long[] lengths() { + return lengths; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathRemoveCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathRemoveCommand.java new file mode 100644 index 0000000000000..e50f1496c4e08 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathRemoveCommand.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.management.classpath; + +import org.apache.ignite.internal.management.api.ComputeCommand; +import org.apache.ignite.internal.visor.VisorMultiNodeTask; + +/** */ +public class ClassPathRemoveCommand implements ComputeCommand { + /** {@inheritDoc} */ + @Override public Class> taskClass() { + return ClassPathRemoveTask.class; + } + + /** {@inheritDoc} */ + @Override public String description() { + return "Removes IgniteClassPath from cluster"; + } + + /** {@inheritDoc} */ + @Override public Class argClass() { + return ClassPathRemoveCommandArg.class; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathRemoveCommandArg.java b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathRemoveCommandArg.java new file mode 100644 index 0000000000000..792c31a72f2dc --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathRemoveCommandArg.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.management.classpath; + +import org.apache.ignite.internal.Order; +import org.apache.ignite.internal.dto.IgniteDataTransferObject; +import org.apache.ignite.internal.management.api.Argument; + +/** */ +public class ClassPathRemoveCommandArg extends IgniteDataTransferObject { + /** */ + private static final long serialVersionUID = 0; + + /** */ + @Order(0) + @Argument(description = "Name of the classpath") + String name; + + /** */ + @Order(1) + @Argument(description = "Forcefully remove ClassPath. Don't wait for running tasks to finish. Use with cautious!", optional = true) + boolean force; + + /** */ + public String name() { + return name; + } + + /** */ + public void name(String name) { + this.name = name; + } + + /** */ + public boolean force() { + return force; + } + + /** */ + public void force(boolean force) { + this.force = force; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathRemoveTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathRemoveTask.java new file mode 100644 index 0000000000000..13a512a6a8afc --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathRemoveTask.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.classpath; + +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.visor.VisorJob; +import org.apache.ignite.internal.visor.VisorOneNodeTask; +import org.jetbrains.annotations.Nullable; + +/** */ +public class ClassPathRemoveTask extends VisorOneNodeTask { + /** */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override protected VisorJob job(ClassPathRemoveCommandArg arg) { + return new ClassPathRemoveJob(arg, debug); + } + + /** */ + private static class ClassPathRemoveJob extends VisorJob { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + protected ClassPathRemoveJob(@Nullable ClassPathRemoveCommandArg arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected Void run(@Nullable ClassPathRemoveCommandArg arg) throws IgniteException { + try { + return ignite.context().classPath().remove(arg.name(), arg.force()).get(); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathStartCreationTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathStartCreationTask.java new file mode 100644 index 0000000000000..606253e5a85e0 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/ClassPathStartCreationTask.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.classpath; + +import java.util.UUID; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.visor.VisorJob; +import org.apache.ignite.internal.visor.VisorOneNodeTask; +import org.jetbrains.annotations.Nullable; + +/** */ +public class ClassPathStartCreationTask extends VisorOneNodeTask { + /** */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override protected VisorJob job(ClassPathCreateCommandArg arg) { + return new ClassPathStartCreationJob(arg, debug); + } + + /** */ + private static class ClassPathStartCreationJob extends VisorJob { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + protected ClassPathStartCreationJob(@Nullable ClassPathCreateCommandArg arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected UUID run(@Nullable ClassPathCreateCommandArg arg) throws IgniteException { + try { + return ignite.context().classPath().startCreation(arg.name, arg.files, arg.lengths); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/SetClassPathStateReadyTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/SetClassPathStateReadyTask.java new file mode 100644 index 0000000000000..651efecca33aa --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/classpath/SetClassPathStateReadyTask.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.management.classpath; + +import java.util.UUID; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.classpath.IgniteClassPath; +import org.apache.ignite.internal.classpath.IgniteClassPathState; +import org.apache.ignite.internal.visor.VisorJob; +import org.apache.ignite.internal.visor.VisorOneNodeTask; +import org.jetbrains.annotations.Nullable; + +/** Task to move newly created {@link IgniteClassPath} to {@link IgniteClassPathState#READY} state. */ +public class SetClassPathStateReadyTask extends VisorOneNodeTask { + /** */ + private static final long serialVersionUID = 0L; + + /** {@inheritDoc} */ + @Override protected VisorJob job(UUID arg) { + return new SetClassPathStateReadyJob(arg, debug); + } + + /** */ + private static class SetClassPathStateReadyJob extends VisorJob { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + protected SetClassPathStateReadyJob(@Nullable UUID arg, boolean debug) { + super(arg, debug); + } + + /** {@inheritDoc} */ + @Override protected Void run(@Nullable UUID arg) throws IgniteException { + try { + ignite.context().classPath().makeReady(arg); + + return null; + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java index 159eeb77db7c9..ce5be13792d80 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/NodeFileTree.java @@ -37,6 +37,7 @@ import org.apache.ignite.internal.cdc.CdcMain; import org.apache.ignite.internal.cdc.CdcManager; import org.apache.ignite.internal.cdc.CdcMode; +import org.apache.ignite.internal.classpath.IgniteClassPath; import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.A; @@ -99,6 +100,16 @@ * │ ├── lock * │ ├── marshaller ← marshaller (shared between all local nodes) * │ │ └── 1645778359.classname0 + * │ ├── classpath ← classpath (shared between all local nodes) + * │ │ └── node00-e57e62a9-2ccf-4e1b-a11e-c24c21b9ed4c ← node classpath root dir (node 0). + * │ │ └── myapp_v1 ← classpath dir (ClassPath name "myapp_v1"). + * │ │ ├── mapp-api-1.0.0.jar ← classpath lib. + * │ │ └── mapp-impl-1.0.0.jar ← classpath lib. + * │ │ └── icp.properties ← classpath descriptor. + * │ │ └── myapp_v2 ← classpath dir (ClassPath name "myapp_v2"). + * │ │ ├── mapp-api-2.0.0.jar ← classpath lib. + * │ │ └── mapp-impl-2.0.0.jar ← classpath lib. + * │ │ └── icp.properties ← classpath descriptor. * │ ├── node00-e57e62a9-2ccf-4e1b-a11e-c24c21b9ed4c ← nodeStorage (node 0). * │ │ ├── cache-default ← cacheStorage (cache name "default"). * │ │ │ ├── cache_data.dat ← cache("default") configuration file. @@ -286,6 +297,9 @@ public class NodeFileTree extends SharedFileTree { /** Maintenance file name. */ private static final String MAINTENANCE_FILE_NAME = "maintenance_tasks.mntc"; + /** {@link IgniteClassPath} descriptor name. */ + public static final String ICP_DESCRIPTOR_NAME = "icp.properties"; + /** Folder name for consistent id. */ private final String folderName; @@ -295,6 +309,9 @@ public class NodeFileTree extends SharedFileTree { /** Path to the storage directory. */ private final File nodeStorage; + /** Path to the root classpath directory. */ + private final File icp; + /** * Key is the path from {@link DataStorageConfiguration#getExtraStoragePaths()}, may be relative. Value is storage. * @see DataStorageConfiguration#getExtraStoragePaths() @@ -334,6 +351,7 @@ public NodeFileTree(File root, String folderName) { this.folderName = folderName; binaryMeta = new File(binaryMetaRoot, folderName); + icp = new File(icpRoot, folderName); nodeStorage = rootRelative(DB_DIR); checkpoint = new File(nodeStorage, CHECKPOINT_DIR); wal = rootRelative(DFLT_WAL_PATH); @@ -382,6 +400,7 @@ protected NodeFileTree(IgniteConfiguration cfg, File root, String folderName, bo this.folderName = folderName; binaryMeta = new File(binaryMetaRoot, folderName); + icp = new File(icpRoot, folderName); DataStorageConfiguration dsCfg = cfg.getDataStorageConfiguration(); @@ -431,6 +450,11 @@ public File binaryMeta() { return binaryMeta; } + /** @return Root directory for Ignite class path files. */ + public File classPathRoot() { + return icp; + } + /** @return Path to the directory containing active WAL segments. */ public @Nullable File wal() { return wal; @@ -1082,6 +1106,22 @@ public File maintenanceFile() { return new File(nodeStorage, MAINTENANCE_FILE_NAME); } + /** + * @param name {@link IgniteClassPath} name. + * @return {@link IgniteClassPath} directory. + */ + public File classPathRoot(String name) { + return new File(icp, name); + } + + /** + * @param name {@link IgniteClassPath} name. + * @return {@link IgniteClassPath} descriptor file. + */ + public File classPathDescriptor(String name) { + return new File(classPathRoot(name), ICP_DESCRIPTOR_NAME); + } + /** * @param includeMeta If {@code true} then include metadata directory into results. * @param filter Cache group names to filter. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java index 33df1ed636040..199b9a00e38d4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/SharedFileTree.java @@ -36,6 +36,7 @@ * ├── db ← db (shared between all local nodes). * │ ├── binary_meta ← binaryMetaRoot (shared between all local nodes). * │ ├── marshaller ← marshaller (shared between all local nodes). + * │ ├── classpath ← classpath (shared between all local nodes). * └── snapshots ← snpsRoot (shared between all local nodes). * * @@ -48,6 +49,9 @@ public class SharedFileTree { /** Name of marshaller mappings folder. */ public static final String MARSHALLER_DIR = "marshaller"; + /** Name of classpath folder. */ + public static final String CLASSPATH_DIR = "classpath"; + /** Database default folder. */ protected static final String DB_DIR = "db"; @@ -60,6 +64,9 @@ public class SharedFileTree { /** Path to the directory containing marshaller files. */ private final File marshaller; + /** Path to the directory containing classpath files. */ + protected final File icpRoot; + /** Path to the snapshot root directory. */ private final File snpsRoot; @@ -77,6 +84,7 @@ protected SharedFileTree(File root, String snpsRoot) { marshaller = Paths.get(rootStr, DB_DIR, MARSHALLER_DIR).toFile(); binaryMetaRoot = Paths.get(rootStr, DB_DIR, BINARY_METADATA_DIR).toFile(); + icpRoot = Paths.get(rootStr, DB_DIR, CLASSPATH_DIR).toFile(); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java index dd0d90b416719..ef8ef39c6371b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java @@ -44,6 +44,7 @@ import org.apache.ignite.internal.binary.GridBinaryMarshaller; import org.apache.ignite.internal.cache.query.index.IndexProcessor; import org.apache.ignite.internal.cache.transform.CacheObjectTransformerProcessor; +import org.apache.ignite.internal.classpath.ClassPathProcessor; import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager; import org.apache.ignite.internal.managers.collision.GridCollisionManager; import org.apache.ignite.internal.managers.communication.GridIoManager; @@ -762,6 +763,11 @@ private void setField(IgniteEx kernal, String name, Object val) throws NoSuchFie return null; } + /** {@inheritDoc} */ + @Override public ClassPathProcessor classPath() { + return null; + } + /** {@inheritDoc} */ @Override public Executor getAsyncContinuationExecutor() { return null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerInternalRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerInternalRequest.java new file mode 100644 index 0000000000000..733e737383ce5 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerInternalRequest.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.odbc; + +import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; + +/** + * Marker interface for requests that can be executed by control.sh, only + * @see ClientConnectionContext#managementClient() + */ +public interface ClientListenerInternalRequest extends ClientListenerRequest { + /** {@inheritDoc} */ + @Override default boolean internal() { + return true; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerRequest.java index 2ee25e4aab614..e9b02b93eb86a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerRequest.java @@ -35,4 +35,9 @@ public interface ClientListenerRequest { default boolean beforeStartupRequest() { return false; } + + /** @return {@code True} if request can be executed only by control.sh client. */ + default boolean internal() { + return false; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientMessageParser.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientMessageParser.java index d0f54ef2136ba..a1bfe607c5e4f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientMessageParser.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientMessageParser.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.platform.client; +import org.apache.ignite.client.ClientException; import org.apache.ignite.internal.binary.BinaryReaderEx; import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.binary.BinaryWriterEx; @@ -76,6 +77,7 @@ import org.apache.ignite.internal.processors.platform.client.cache.ClientCacheScanQueryRequest; import org.apache.ignite.internal.processors.platform.client.cache.ClientCacheSqlFieldsQueryRequest; import org.apache.ignite.internal.processors.platform.client.cache.ClientCacheSqlQueryRequest; +import org.apache.ignite.internal.processors.platform.client.classpath.ClientClassPathFileUploadRequest; import org.apache.ignite.internal.processors.platform.client.cluster.ClientClusterChangeStateRequest; import org.apache.ignite.internal.processors.platform.client.cluster.ClientClusterGetDataCenterNodesRequest; import org.apache.ignite.internal.processors.platform.client.cluster.ClientClusterGetStateRequest; @@ -115,6 +117,8 @@ import org.apache.ignite.internal.processors.platform.client.streamer.ClientDataStreamerStartRequest; import org.apache.ignite.internal.processors.platform.client.tx.ClientTxEndRequest; import org.apache.ignite.internal.processors.platform.client.tx.ClientTxStartRequest; +import org.apache.ignite.internal.thread.context.Scope; +import org.apache.ignite.plugin.security.SecurityPermission; /** * Thin client message parser. @@ -410,10 +414,16 @@ public class ClientMessageParser implements ClientListenerMessageParser { /** Get service topology. */ private static final short OP_SERVICE_GET_TOPOLOGY = 7003; + /** File upload. */ + public static final short FILE_UPLOAD = 9030; + /** Operations that are performed before a node is joined to the topology. */ /** Stop warmup. */ private static final short OP_STOP_WARMUP = 10000; + /** */ + public static final String INTERNAL_REQ_ERR_MSG = "Only management client are allowed to execute this"; + /** Marshaller. */ private final GridBinaryMarshaller marsh; @@ -451,6 +461,9 @@ public class ClientMessageParser implements ClientListenerMessageParser { if (ctx.kernalContext().recoveryMode() && !req.beforeStartupRequest()) return new ClientRawRequest(req.requestId(), ClientStatus.FAILED, "Node in recovery mode."); + if (req.internal()) + checkInternalRequestAllowed(); + return req; } @@ -735,6 +748,9 @@ public ClientListenerRequest decode(BinaryReaderEx reader) { case OP_SERVICE_GET_TOPOLOGY: return new ClientServiceTopologyRequest(reader); + case FILE_UPLOAD: + return new ClientClassPathFileUploadRequest(reader); + case OP_STOP_WARMUP: return new ClientCacheStopWarmupRequest(reader); } @@ -771,4 +787,22 @@ public ClientListenerRequest decode(BinaryReaderEx reader) { @Override public long decodeRequestId(ClientMessage msg) { return 0; } + + /** + * Check permissions to invoke internal requests. + */ + private void checkInternalRequestAllowed() { + if (!ctx.managementClient()) + throw new ClientException(INTERNAL_REQ_ERR_MSG); + + // When security is enabled, only an administrator can connect and execute commands. + if (ctx.securityContext() != null) { + try (Scope ignored = ctx.kernalContext().security().withContext(ctx.securityContext())) { + ctx.kernalContext().security().authorize(SecurityPermission.ADMIN_OPS); + } + catch (SecurityException e) { + throw new ClientException("ADMIN_OPS permission required"); + } + } + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/classpath/ClientClassPathFileUploadRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/classpath/ClientClassPathFileUploadRequest.java new file mode 100644 index 0000000000000..d7bc9d0dd2535 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/classpath/ClientClassPathFileUploadRequest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.platform.client.classpath; + +import java.util.UUID; +import org.apache.ignite.binary.BinaryRawReader; +import org.apache.ignite.internal.classpath.IgniteClassPath; +import org.apache.ignite.internal.processors.odbc.ClientListenerInternalRequest; +import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; +import org.apache.ignite.internal.processors.platform.client.ClientRequest; +import org.apache.ignite.internal.processors.platform.client.ClientResponse; + +/** + * Must be used only from control.sh command. + * Writes part of {@link IgniteClassPath} file. + * + * @see ClientListenerInternalRequest + * @see IgniteClassPath + */ +public class ClientClassPathFileUploadRequest extends ClientRequest implements ClientListenerInternalRequest { + /** Upload node ID. */ + private final UUID uploadNodeId; + + /** ClassPath ID. */ + private final UUID icpId; + + /** File name. */ + private final String name; + + /** Offset to write data to. */ + private final long offset; + + /** Batch. */ + private final byte[] batch; + + /** + * Creates the file upload request. + * + * @param reader Reader. + */ + public ClientClassPathFileUploadRequest(BinaryRawReader reader) { + super(reader); + + uploadNodeId = reader.readUuid(); + icpId = reader.readUuid(); + name = reader.readString(); + offset = reader.readLong(); + batch = reader.readByteArray(); + } + + /** {@inheritDoc} */ + @Override public ClientResponse process(ClientConnectionContext ctx) { + UUID locNodeId = ctx.kernalContext().localNodeId(); + + if (!uploadNodeId.equals(locNodeId)) + throw new IllegalStateException("Wrong node [uploadNode=" + uploadNodeId + ", localNode=" + locNodeId + ']'); + + ctx.kernalContext().classPath().writeFilePartFromClient(icpId, name, offset, batch); + + return new ClientResponse(requestId()); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/classpath/ClassPathCreationFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/classpath/ClassPathCreationFailoverTest.java new file mode 100644 index 0000000000000..43b88d788d5ca --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/classpath/ClassPathCreationFailoverTest.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.classpath; + +import java.io.File; +import java.nio.file.Path; +import java.util.Set; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.Ignition; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.management.classpath.ClassPathCreateCommand; +import org.apache.ignite.internal.management.classpath.ClassPathCreateCommandArg; +import org.apache.ignite.internal.managers.communication.GridIoMessage; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.lang.IgniteInClosure; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.ListeningTestLogger; +import org.apache.ignite.testframework.LogListener; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static org.apache.ignite.internal.classpath.ClassPathProcessor.metastorageKey; +import static org.apache.ignite.internal.classpath.IgniteClassPathState.LOST; +import static org.apache.ignite.internal.classpath.IgniteClassPathState.READY; +import static org.apache.ignite.testframework.GridTestUtils.getFieldValue; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** */ +public class ClassPathCreationFailoverTest extends GridCommonAbstractTest { + /** */ + public static final int TIMEOUT = 10_000; + + /** */ + private static IgniteEx failedNode; + + /** */ + private static boolean sendMessage; + + /** */ + private static Runnable callback; + + /** */ + private ListeningTestLogger lsnrLog; + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + stopAllGrids(); + + cleanPersistenceDir(); + + lsnrLog = new ListeningTestLogger(log); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + failedNode = null; + + sendMessage = false; + + callback = null; + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setGridLogger(lsnrLog) + .setCommunicationSpi(new TcpCommunicationSpi() { + @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure ackC) { + if (msg instanceof GridIoMessage + && ((GridIoMessage)msg).message() instanceof DownloadClassPathMessage + && !failedNode.localNode().id().equals(ignite.cluster().localNode().id()) + ) { + if (callback != null) + callback.run(); + + if (!sendMessage) + return; + } + + super.sendMessage(node, msg, ackC); + } + }); + } + + // TODO: add stopped check during async operations. + // TODO: don't create on client node. ??? + + /** */ + @Test + public void testUploadNodeFailDuringDeployment() throws Exception { + failedNode = startGrid(0); + + callback = () -> GridTestUtils.runAsync(() -> failedNode.close()); + + IgniteEx grid1 = startGrid(1); + + awaitPartitionMapExchange(); + + Set cpFiles = ClassPathTestUtils.files(); + + LogListener createdMsg = newClassPathListener(); + LogListener uploadNodeFailedMsg = logListener("Failed to download ClassPath files [icp=testcp]"); + + lsnrLog.registerAllListeners(createdMsg, uploadNodeFailedMsg); + + ClassPathCreateCommandArg arg = new ClassPathCreateCommandArg(); + + arg.name("testcp"); + arg.files(ClassPathTestUtils.fileArg(cpFiles)); + + // ClassPath must be created on `failedNode`. Because, it first in cluster. + new ClassPathCreateCommand().execute(null, grid(0), arg, l -> failedNode.log().info(l)); + + assertTrue(waitForCondition(createdMsg::check, TIMEOUT)); + assertTrue(waitForCondition(() -> failedNode.context().isStopping(), TIMEOUT)); + + stopGrid(0); + + assertEquals(1, Ignition.allGrids().size()); + + assertTrue(waitForCondition(() -> { + Object t = getFieldValue(ClassPathTestUtils.transmissionHandler(grid1), "active"); + + if (t == null) + return true; + + GridFutureAdapter res = getFieldValue(t, "res"); + + return res.isDone(); + }, TIMEOUT)); + + assertTrue(waitForCondition(uploadNodeFailedMsg::check, TIMEOUT)); + assertTrue(waitForCondition(() -> { + try { + return grid1.context().distributedMetastorage().read(metastorageKey("testcp")).state() == LOST; + } + catch (IgniteCheckedException e) { + return false; + } + }, + TIMEOUT + )); + assertTrue(waitForCondition( + () -> !grid1.context().pdsFolderResolver().fileTree().classPathRoot("testcp").exists(), + TIMEOUT + )); + + IgniteClassPath icp = grid1.context().distributedMetastorage().read(metastorageKey("testcp")); + + assertNotNull(icp); + assertEquals(LOST, icp.state()); + assertTrue(icp.deployedOnNodes().isEmpty()); + } + + /** */ + @Test + public void testDownloadNodeFailDuringDeployment() throws Exception { + startGrids(3); + + callback = () -> GridTestUtils.runAsync(() -> failedNode.close()); + + failedNode = grid(1); + sendMessage = true; + + awaitPartitionMapExchange(); + + Set cpFiles = ClassPathTestUtils.files(); + + LogListener createdMsg = newClassPathListener(); + LogListener oneNodeFailMsg = logListener("IgniteClassPath task failure [task=download"); + LogListener donloadSucceedMsg = logListener("IgniteClassPath task done [task=download"); + + lsnrLog.registerAllListeners(createdMsg, oneNodeFailMsg, donloadSucceedMsg); + + ClassPathCreateCommandArg arg = new ClassPathCreateCommandArg(); + + arg.name("testcp"); + arg.files(ClassPathTestUtils.fileArg(cpFiles)); + + // ClassPath must be created on `failedNode`. Because, it first in cluster. + new ClassPathCreateCommand().execute(null, grid(0), arg, l -> failedNode.log().info(l)); + + assertTrue(waitForCondition(createdMsg::check, TIMEOUT)); + assertTrue(waitForCondition(() -> failedNode.context().isStopping(), TIMEOUT)); + + stopGrid(1); + + assertEquals(2, Ignition.allGrids().size()); + + assertTrue(waitForCondition(oneNodeFailMsg::check, TIMEOUT)); + assertTrue(waitForCondition(donloadSucceedMsg::check, TIMEOUT)); + + ClassPathTestUtils.checkFilesExists(grid(0), "testcp", cpFiles); + ClassPathTestUtils.checkFilesExists(grid(2), "testcp", cpFiles); + + IgniteClassPath icp = grid(0).context().distributedMetastorage().read(ClassPathProcessor.metastorageKey("testcp")); + + assertNotNull(icp); + assertEquals(READY, icp.state()); + assertEquals(2, icp.deployedOnNodes().size()); + + ClassPathTestUtils.checkDeployedOn(grid(0), "testcp"); + ClassPathTestUtils.checkDeployedOn(grid(2), "testcp"); + } + + /** */ + @Test + public void testConcurrentFileCreationDuringDeployment() throws Exception { + startGrids(2); + + sendMessage = true; + failedNode = grid(0); + + awaitPartitionMapExchange(); + + Set cpFiles = ClassPathTestUtils.files(); + + callback = () -> { + File cpRoot = grid(1).context().pdsFolderResolver().fileTree().classPathRoot("testcp"); + + ClassPathTestUtils.fileNames(cpFiles).forEach(f -> new File(cpRoot, f).mkdirs()); + }; + + LogListener createdMsg = newClassPathListener(); + LogListener oneNodeFailMsg = logListener("Failed to download ClassPath files [icp=testcp]"); + + lsnrLog.registerAllListeners(createdMsg, oneNodeFailMsg); + + ClassPathCreateCommandArg arg = new ClassPathCreateCommandArg(); + + arg.name("testcp"); + arg.files(ClassPathTestUtils.fileArg(cpFiles)); + + // ClassPath must be created on `failedNode`. Because, it first in cluster. + new ClassPathCreateCommand().execute(null, grid(0), arg, l -> grid(0).log().info(l)); + + assertTrue(waitForCondition(createdMsg::check, TIMEOUT)); + + assertEquals(2, Ignition.allGrids().size()); + + assertTrue(waitForCondition(oneNodeFailMsg::check, TIMEOUT)); + + ClassPathTestUtils.checkFilesExists(grid(0), "testcp", cpFiles); + + IgniteClassPath icp = grid(0).context().distributedMetastorage().read(metastorageKey("testcp")); + + assertNotNull(icp); + assertEquals(READY, icp.state()); + assertEquals(1, icp.deployedOnNodes().size()); + + ClassPathTestUtils.checkDeployedOn(grid(0), "testcp"); + + assertFalse(grid(1).context().pdsFolderResolver().fileTree().classPathRoot("testcp").exists()); + } + + /** */ + private LogListener newClassPathListener() { + return logListener("New ClassPath created [uploadNode=" + grid(0).localNode().id()); + } + + /** */ + private static LogListener logListener(String msg) { + return LogListener.matches(msg).times(1).build(); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/classpath/ClassPathMaliciuosRequestsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/classpath/ClassPathMaliciuosRequestsTest.java new file mode 100644 index 0000000000000..8e54f8c9f7c47 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/classpath/ClassPathMaliciuosRequestsTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.classpath; + +import java.nio.file.Path; +import java.util.UUID; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static org.apache.ignite.testframework.GridTestUtils.assertThrows; + +/** + * TODO: add filter for class path descriptor. + */ +public class ClassPathMaliciuosRequestsTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + startGrid(0); + } + + /** */ + @Test + public void testMaliciousFilename() { + String[] hackNames = new String[]{ + "/", + "../../optional/ignite-cdc/myjar.jar", + "./file.txt", + "../file.txt", + "/file.txt", + "~/file.txt" + }; + + for (String hackName : hackNames) { + assertThrows( + null, + () -> cpProc().startCreation("mycp", new String[]{hackName}, new long[]{42}), + IllegalArgumentException.class, + "simple filename expected" + ); + } + } + + /** */ + @Test + public void testMaliciousClassPathName() { + String[] hackNames = new String[]{ + "/", + "../../optional/ignite-cdc", + "./files", + "../files", + "/files", + "~/files" + }; + + for (String hackName : hackNames) { + assertThrows( + null, + () -> cpProc().startCreation(hackName, new String[]{"file.txt"}, new long[]{42}), + IllegalArgumentException.class, + "Classpath name must satisfy the following name pattern: a-zA-Z0-9_" + ); + } + } + + /** */ + @Test + public void testUnknownFilename() throws IgniteCheckedException { + UUID icpId0 = cpProc().startCreation("mycp", new String[]{"file.txt"}, new long[]{42}); + + assertThrows( + null, + () -> { + cpProc().copyClassPathFileLocally(icpId0, Path.of("other.txt")); + return null; + }, + IgniteException.class, + "Unknown lib" + ); + + UUID icpId1 = cpProc().startCreation("mycp", new String[]{"file.txt"}, new long[]{42}); + + assertThrows( + null, + () -> { + cpProc().writeFilePartFromClient(icpId1, "other.txt", 0, new byte[1]); + return null; + }, + IgniteException.class, + "Unknown lib" + ); + } + + /** */ + @Test + public void testWrongOffset() throws IgniteCheckedException { + UUID icpId = cpProc().startCreation("mycp", new String[]{"file.txt"}, new long[]{42}); + + assertThrows( + null, + () -> { + cpProc().writeFilePartFromClient(icpId, "file.txt", U.GB, new byte[] {0}); + return null; + }, + IgniteException.class, + "Unexpected file offset [icp=mycp, file=file.txt]" + ); + } + + /** */ + private ClassPathProcessor cpProc() { + return grid(0).context().classPath(); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/classpath/ClassPathSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/classpath/ClassPathSelfTest.java new file mode 100644 index 0000000000000..741acdde367e3 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/classpath/ClassPathSelfTest.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.classpath; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Set; +import java.util.UUID; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.management.classpath.ClassPathCreateCommand; +import org.apache.ignite.internal.management.classpath.ClassPathCreateCommandArg; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.ListeningTestLogger; +import org.apache.ignite.testframework.LogListener; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static org.apache.ignite.internal.classpath.ClassPathCreationFailoverTest.TIMEOUT; +import static org.apache.ignite.internal.classpath.ClassPathProcessor.metastorageKey; +import static org.apache.ignite.internal.classpath.IgniteClassPathState.LOST; +import static org.apache.ignite.internal.classpath.IgniteClassPathState.READY; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** */ +public class ClassPathSelfTest extends GridCommonAbstractTest { + // TODO: New node moves LOST to READY. + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** */ + @Test + public void testNewNodeDownloadExistingClassPathes() throws Exception { + IgniteEx srv = startGrid(0); + + Set cpFiles = ClassPathTestUtils.files(); + + ClassPathCreateCommandArg arg = new ClassPathCreateCommandArg(); + + arg.name("testcp"); + arg.files(ClassPathTestUtils.fileArg(cpFiles)); + + new ClassPathCreateCommand().execute(null, grid(0), arg, l -> log.info(l)); + + checkState(srv, "testcp"); + + ClassPathTestUtils.checkDeployedOn(srv, "testcp"); + ClassPathTestUtils.checkFilesExists(srv, "testcp", cpFiles); + + IgniteEx srv1 = startGrid(1); + + ClassPathTestUtils.checkDeployedOn(srv1, "testcp"); + ClassPathTestUtils.checkFilesExists(grid(1), "testcp", cpFiles); + } + + /** */ + @Test + public void testNewNodeRemoveStaleOnStart() throws Exception { + IgniteEx srv = startGrid(0); + + Set cpFiles = ClassPathTestUtils.files(); + + ClassPathCreateCommandArg arg = new ClassPathCreateCommandArg(); + + arg.name("testcp"); + arg.files(ClassPathTestUtils.fileArg(cpFiles)); + + new ClassPathCreateCommand().execute(null, grid(0), arg, l -> log.info(l)); + + arg.name("testcp_v2"); + arg.files(ClassPathTestUtils.fileArg(cpFiles)); + + new ClassPathCreateCommand().execute(null, grid(0), arg, l -> log.info(l)); + + checkState(srv, "testcp"); + checkState(srv, "testcp_v2"); + + ClassPathTestUtils.checkFilesExists(grid(0), "testcp", cpFiles); + ClassPathTestUtils.checkFilesExists(grid(0), "testcp_v2", cpFiles); + + NodeFileTree ft = new NodeFileTree(new File(U.defaultWorkDirectory()), "second_node"); + + IgniteClassPath stale = new IgniteClassPath( + UUID.randomUUID(), + Collections.emptySet(), + "testcp_v2", + new String[] {".stale"}, + new long[] {42}, + READY + ); + + IgniteClassPath unknown = new IgniteClassPath( + UUID.randomUUID(), + Collections.emptySet(), + "unknown", + new String[] {".unknown"}, + new long[] {42}, + READY + ); + + assertTrue(ft.classPathRoot(stale.name()).mkdirs()); + assertTrue(ft.classPathRoot(unknown.name()).mkdirs()); + + ClassPathProcessor.writeClassPathDescriptor(ft, stale); + ClassPathProcessor.writeClassPathDescriptor(ft, unknown); + + ListeningTestLogger lsnrLog = new ListeningTestLogger(log); + + LogListener rmvStaleLsnr = LogListener.matches("Stale local data. Removing " + + "[loc=IgniteClassPath [id=" + stale.id() + ", name=" + stale.name() + ", state=" + stale.state() + "]") + .times(1).build(); + + LogListener rmvUnknownLsnr = LogListener.matches("Unknown local data. Removing " + + "[icp=IgniteClassPath [id=" + unknown.id() + ", name=" + unknown.name() + ", state=" + unknown.state() + "]]") + .times(1).build(); + + lsnrLog.registerAllListeners(rmvStaleLsnr, rmvUnknownLsnr); + + IgniteEx srv1 = startGrid(getConfiguration(getTestIgniteInstanceName(1)) + .setConsistentId("second_node") + .setGridLogger(lsnrLog)); + + assertTrue(waitForCondition(rmvStaleLsnr::check, TIMEOUT)); + assertTrue(waitForCondition(rmvUnknownLsnr::check, TIMEOUT)); + + ClassPathTestUtils.checkDeployedOn(srv, "testcp"); + ClassPathTestUtils.checkDeployedOn(srv1, "testcp_v2"); + + assertNull(srv.context().distributedMetastorage().read(metastorageKey("unknown"))); + } + + /** */ + @Test + public void testNewNodeMoveFromLostToReady() throws Exception { + IgniteEx srv = startGrid(0); + + Set cpFiles = ClassPathTestUtils.files(); + + int idx = 0; + String[] names = new String[cpFiles.size()]; + long[] lengths = new long[cpFiles.size()]; + + for (Path cpFile : cpFiles) { + names[idx] = cpFile.getFileName().toString(); + lengths[idx] = Files.size(cpFile); + idx++; + } + + IgniteClassPath lost = new IgniteClassPath( + UUID.randomUUID(), + Collections.emptySet(), + "testcp", + names, + lengths, + LOST + ); + + assertTrue(srv.context().distributedMetastorage().compareAndSet(metastorageKey(lost.name()), null, lost)); + + NodeFileTree ft = new NodeFileTree(new File(U.defaultWorkDirectory()), "second_node"); + + File cpRoot = ft.classPathRoot(lost.name()); + + assertTrue(cpRoot.mkdirs()); + assertTrue(ClassPathProcessor.guardFile(cpRoot, lost.id()).createNewFile()); + ClassPathProcessor.writeClassPathDescriptor(ft, lost); + + for (Path cpFile : cpFiles) + Files.copy(cpFile, new File(cpRoot, cpFile.getFileName().toString()).toPath()); + + ListeningTestLogger lsnrLog = new ListeningTestLogger(log); + + IgniteEx srv1 = startGrid(getConfiguration(getTestIgniteInstanceName(1)) + .setConsistentId("second_node") + .setGridLogger(lsnrLog)); + + ClassPathTestUtils.checkDeployedOn(srv, "testcp"); + ClassPathTestUtils.checkDeployedOn(srv1, "testcp"); + + ClassPathTestUtils.checkFilesExists(srv, "testcp", cpFiles); + ClassPathTestUtils.checkFilesExists(srv1, "testcp", cpFiles); + + assertEquals(READY, srv.context().distributedMetastorage().read(metastorageKey(lost.name())).state()); + } + + /** */ + private static void checkState(IgniteEx srv, String testcp_v2) throws IgniteCheckedException { + IgniteClassPath icp = srv.context().distributedMetastorage().read(metastorageKey(testcp_v2)); + + assertNotNull(icp); + assertEquals(READY, icp.state()); + assertTrue(icp.deployedOnNodes().contains(srv.localNode().id())); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/classpath/ClassPathTestUtils.java b/modules/core/src/test/java/org/apache/ignite/internal/classpath/ClassPathTestUtils.java new file mode 100644 index 0000000000000..6cd4b629a5ce0 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/classpath/ClassPathTestUtils.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.classpath; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree; +import org.apache.ignite.internal.util.typedef.internal.U; + +import static org.apache.ignite.internal.classpath.ClassPathProcessor.metastorageKey; +import static org.apache.ignite.testframework.GridTestUtils.getFieldValue; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** */ +public class ClassPathTestUtils { + /** */ + public static Set files() throws IOException { + return Stream.of( + file(0), file(1025), file(U.MB), file(2 * U.MB), file(2 * U.MB + 42)).collect(Collectors.toSet() + ); + } + + /** */ + public static Set fileNames(Set dirs) { + return dirs.stream().map(Path::getFileName).map(Path::toString).collect(Collectors.toSet()); + } + + /** */ + public static String[] fileArg(Set cpFiles) { + String[] files = new String[cpFiles.size()]; + + int i = 0; + + for (Path p : cpFiles) + files[i++] = p.toAbsolutePath().toString(); + + return files; + } + + /** */ + public static Path file(long size) throws IOException { + File f = File.createTempFile(size + "_bytes", ".temp"); + + f.deleteOnExit(); + + try (OutputStream os = Files.newOutputStream(f.toPath())) { + long written = 0; + + byte[] batch = new byte[4 * (int)U.KB]; + + while (written < size) { + ThreadLocalRandom.current().nextBytes(batch); + + int toWrite = (int)Math.min(size - written, batch.length); + + os.write(batch, 0, toWrite); + + written += toWrite; + } + } + + assert f.length() == size; + + return f.toPath(); + } + + /** */ + public static void checkFilesExists(IgniteEx node, String cpName, Set cpFiles) throws IOException, IgniteCheckedException { + NodeFileTree ft = node.context().pdsFolderResolver().fileTree(); + IgniteClassPath icp = node.context().distributedMetastorage().read(metastorageKey(cpName)); + + Set nodeCpFiles = Files.list(ft.classPathRoot(cpName).toPath()).collect(Collectors.toSet()); + + Set nodeCpFilesNames = fileNames(nodeCpFiles); + + nodeCpFilesNames.remove(NodeFileTree.ICP_DESCRIPTOR_NAME); + nodeCpFilesNames.remove(ClassPathProcessor.guardFile(ft.classPathRoot(cpName), icp.id()).getName()); + + assertEquals( + "Files must be deployed on each node", + fileNames(cpFiles), + nodeCpFilesNames + ); + + for (Path cpFile : cpFiles) { + Path nodeFile = nodeCpFiles.stream() + .filter(p -> Objects.equals(p.getFileName().toString(), cpFile.getFileName().toString())) + .findFirst().get(); + + assertEquals(Files.size(cpFile), Files.size(nodeFile)); + } + } + + /** */ + public static void checkDeployedOn(IgniteEx srv, String cpName) throws IgniteInterruptedCheckedException { + assertTrue(waitForCondition(() -> { + try { + return srv.context().distributedMetastorage().read(metastorageKey(cpName)) + .deployedOnNodes().contains(srv.localNode().id()); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + }, 10_000)); + } + + /** */ + static ClassPathFilesTransmissionHandler transmissionHandler(IgniteEx grid1) { + return getFieldValue(grid1.context().classPath(), "icpFilesHnd"); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/client/ThinClientPermissionCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/client/ThinClientPermissionCheckTest.java index 401edd85015dd..f0cf1ee674b01 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/client/ThinClientPermissionCheckTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/client/ThinClientPermissionCheckTest.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.security.client; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -24,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -36,6 +38,7 @@ import org.apache.ignite.client.ClientAuthenticationException; import org.apache.ignite.client.ClientAuthorizationException; import org.apache.ignite.client.ClientCache; +import org.apache.ignite.client.ClientConnectionException; import org.apache.ignite.client.ClientException; import org.apache.ignite.client.Config; import org.apache.ignite.client.IgniteClient; @@ -49,6 +52,7 @@ import org.apache.ignite.configuration.ThinClientConfiguration; import org.apache.ignite.events.CacheEvent; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.client.thin.TcpIgniteClient; import org.apache.ignite.internal.processors.cache.eviction.paged.TestObject; import org.apache.ignite.internal.processors.platform.cache.expiry.PlatformExpiryPolicyFactory; import org.apache.ignite.internal.processors.security.AbstractSecurityTest; @@ -440,6 +444,49 @@ public void testConnectAsManagementClient() throws Exception { checkDflt.run(); } + /** */ + @Test + public void testClassPathUploadFilePart() throws Exception { + File tmpFile = File.createTempFile("file", "temp"); + + tmpFile.deleteOnExit(); + + // Client has no permission to invoke internal methods. + // Trying to invoke without specifying "management client" flag must fail. + for (String name: new String[] {CLIENT, ADMIN}) { + assertThrows(log, () -> { + try (IgniteClient cli = startClient(name)) { + ((TcpIgniteClient)cli).uploadClasspathFile(F.first(cli.cluster().nodes()), UUID.randomUUID(), tmpFile.toPath()); + return null; + } + }, ClientConnectionException.class, "Channel is closed"); + } + + userAttrs = F.asMap(MANAGEMENT_CLIENT_ATTR, "true"); + + try { + // Trying to invoke as CLIENT with "management client" flag must fail, because of security. + // CLIENT has no ADMIN_OPS permission. + assertThrows(log, () -> { + try (IgniteClient cli = startClient(CLIENT)) { + ((TcpIgniteClient)cli).uploadClasspathFile(F.first(cli.cluster().nodes()), UUID.randomUUID(), tmpFile.toPath()); + return null; + } + }, ClientConnectionException.class, "Channel is closed"); + + // Check that request actually invoked and failed due to unknown ClassPath. + assertThrows(log, () -> { + try (IgniteClient cli = startClient(ADMIN)) { + ((TcpIgniteClient)cli).uploadClasspathFile(F.first(cli.cluster().nodes()), UUID.randomUUID(), tmpFile.toPath()); + return null; + } + }, ClientException.class, "ClassPath not found"); + } + finally { + userAttrs = null; + } + } + /** * Gets all operations. * diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDeploymentSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDeploymentSelfTestSuite.java index 5218686113ced..f6f2e95280e59 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDeploymentSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDeploymentSelfTestSuite.java @@ -17,6 +17,9 @@ package org.apache.ignite.testsuites; +import org.apache.ignite.internal.classpath.ClassPathCreationFailoverTest; +import org.apache.ignite.internal.classpath.ClassPathMaliciuosRequestsTest; +import org.apache.ignite.internal.classpath.ClassPathSelfTest; import org.apache.ignite.spi.deployment.local.GridLocalDeploymentSpiSelfTest; import org.apache.ignite.spi.deployment.local.GridLocalDeploymentSpiStartStopSelfTest; import org.junit.runner.RunWith; @@ -28,7 +31,10 @@ @RunWith(Suite.class) @Suite.SuiteClasses({ GridLocalDeploymentSpiSelfTest.class, - GridLocalDeploymentSpiStartStopSelfTest.class + GridLocalDeploymentSpiStartStopSelfTest.class, + ClassPathSelfTest.class, + ClassPathMaliciuosRequestsTest.class, + ClassPathCreationFailoverTest.class }) public class IgniteSpiDeploymentSelfTestSuite { } diff --git a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientOperation.java b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientOperation.java index 112dc47c7125a..747dfd5867dff 100644 --- a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientOperation.java +++ b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientOperation.java @@ -294,6 +294,9 @@ public enum ClientOperation { /** IgniteSet.iterator page. */ OP_SET_ITERATOR_GET_PAGE(9023), + /** File upload. */ + FILE_UPLOAD(9030), + /** Stop warmup. */ OP_STOP_WARMUP(10000); diff --git a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/PayloadInputChannel.java b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/PayloadInputChannel.java index e8f6938890d25..38d0e5d13416d 100644 --- a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/PayloadInputChannel.java +++ b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/PayloadInputChannel.java @@ -35,8 +35,8 @@ class PayloadInputChannel { * Constructor. */ PayloadInputChannel(ClientChannel ch, ByteBuffer payload) { - in = BinaryStreams.inputStream(payload); this.ch = ch; + in = BinaryStreams.inputStream(payload); } /** diff --git a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ReliableChannelImpl.java b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ReliableChannelImpl.java index b29bb69629fe3..2e5fab26ed574 100644 --- a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ReliableChannelImpl.java +++ b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ReliableChannelImpl.java @@ -942,7 +942,6 @@ private T applyOnNodeChannelWithFallback(UUID tryNodeId, Function(); @@ -1034,7 +1033,7 @@ class ClientChannelHolder { private volatile ClientChannel ch; /** ID of the last server node that {@link #ch} is or was connected to. */ - private volatile UUID serverNodeId; + volatile UUID serverNodeId; /** Address that holder is bind to (chCfg.addr) is not in use now. So close the holder. */ private volatile boolean close; @@ -1075,7 +1074,7 @@ private boolean applyReconnectionThrottling() { /** * Get or create channel. */ - private ClientChannel getOrCreateChannel() + public ClientChannel getOrCreateChannel() throws ClientConnectionException, ClientAuthenticationException, ClientProtocolError { return getOrCreateChannel(false); } @@ -1175,6 +1174,19 @@ void setConfiguration(ClientChannelConfiguration chCfg) { } } + /** + * @param id Node id. + * @return Client channel for node. + */ + public ClientChannel nodeClientChannel(UUID id) { + ClientChannelHolder cliCh = nodeChannels.get(id); + + if (cliCh == null) + throw new ClientConnectionException("Node can't be found [id=" + id + ']'); + + return cliCh.getOrCreateChannel(); + } + /** * Get holders reference. For test purposes. */ diff --git a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java index 3aca5f9fde6b9..000ecd1f24229 100644 --- a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java +++ b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java @@ -18,16 +18,22 @@ package org.apache.ignite.internal.client.thin; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; import java.util.EventListener; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.stream.Collectors; import org.apache.ignite.IgniteBinary; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteLogger; @@ -55,6 +61,7 @@ import org.apache.ignite.client.events.ClientLifecycleEventListener; import org.apache.ignite.client.events.ClientStartEvent; import org.apache.ignite.client.events.ClientStopEvent; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.BinaryConfiguration; import org.apache.ignite.configuration.ClientConfiguration; import org.apache.ignite.configuration.ClientTransactionConfiguration; @@ -625,6 +632,61 @@ private void retrieveBinaryConfiguration(ClientConfiguration cfg) { marsh.setBinaryConfiguration(resCfg); } + /** + * @param node Node to upload file to. + * @param icpID Classpath ID. + * @param file File to upload. + */ + public void uploadClasspathFile(ClusterNode node, UUID icpID, Path file) throws IOException { + ClientChannel cliCh = ch.nodeClientChannel(node.id()); + + String name = file.getFileName().toString(); + byte[] batch = new byte[(int)(CommonUtils.MB)]; + + try (InputStream fis = Files.newInputStream(file)) { + long[] offset = new long[]{0}; + int[] bytesCnt = new int[1]; + + // We want to create empty file on the server side. + // So, even 0 bytes read, one request need to be sent. + bytesCnt[0] = fis.read(batch); + + do { + cliCh.service( + ClientOperation.FILE_UPLOAD, + ch -> { + try (BinaryWriterEx w = BinaryUtils.writer(marsh.context(), ch.out(), null)) { + w.writeUuid(node.id()); + w.writeUuid(icpID); + w.writeString(name); + w.writeLong(offset[0]); + w.writeByteArray(batch, 0, Math.max(0, bytesCnt[0])); + } + }, + in -> null + ); + + if (bytesCnt[0] > 0) + offset[0] += bytesCnt[0]; + + bytesCnt[0] = fis.read(batch); + } + while (bytesCnt[0] > 0); + + // Check all data read and sent. + if (offset[0] != Files.size(file)) + throw new IOException("Can't read all data from file"); + } + } + + /** @return Node IDs client connected to. */ + public List connectedToNodes() { + return ch.getChannelHolders().stream() + .map(hldr -> hldr.serverNodeId) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + /** * Thin client implementation of {@link BinaryMetadataHandler}. */