diff --git a/docs/content.zh/docs/sql/materialized-table/deployment.md b/docs/content.zh/docs/sql/materialized-table/deployment.md index 17f833e5f7a084..ad001795cc1704 100644 --- a/docs/content.zh/docs/sql/materialized-table/deployment.md +++ b/docs/content.zh/docs/sql/materialized-table/deployment.md @@ -98,7 +98,7 @@ Catalog 必须支持创建物化表，目前只有 [Paimon Catalog](https://paim ### 在 Standalone 集群运行刷新作业 ```sql -Flink SQL> SET 'execution.mode' = 'remote'; +Flink SQL> SET 'execution.target' = 'remote'; [INFO] Execute statement succeeded. FLINK SQL> CREATE MATERIALIZED TABLE my_materialized_table @@ -113,7 +113,7 @@ FLINK SQL> CREATE MATERIALIZED TABLE my_materialized_table **Kubernetes session 模式：** ```sql -Flink SQL> SET 'execution.mode' = 'kubernetes-session'; +Flink SQL> SET 'execution.target' = 'kubernetes-session'; [INFO] Execute statement succeeded. Flink SQL> SET 'kubernetes.cluster-id' = 'flink-cluster-mt-session-1'; @@ -124,12 +124,12 @@ FLINK SQL> CREATE MATERIALIZED TABLE my_materialized_table [INFO] Execute statement succeeded. ``` -设置 `execution.mode` 为 `kubernetes-session` 并设置参数 `kubernetes.cluster-id` 指向一个已经存在的 Kubernetes session 集群. +设置 `execution.target` 为 `kubernetes-session` 并设置参数 `kubernetes.cluster-id` 指向一个已经存在的 Kubernetes session 集群. **YARN session 模式：** ```sql -Flink SQL> SET 'execution.mode' = 'yarn-session'; +Flink SQL> SET 'execution.target' = 'yarn-session'; [INFO] Execute statement succeeded. Flink SQL> SET 'yarn.application.id' = 'application-xxxx'; @@ -139,14 +139,14 @@ FLINK SQL> CREATE MATERIALIZED TABLE my_materialized_table > ... [INFO] Execute statement succeeded. ``` -设置 `execution.mode` 为 `yarn-session` 并设置参数 `yarn.application.id` 指向一个已经存在的 YARN session 集群。 +设置 `execution.target` 为 `yarn-session` 并设置参数 `yarn.application.id` 指向一个已经存在的 YARN session 集群。 ### 在 application 模式下运行刷新作业 **Kubernetes application 模式：** ```sql -Flink SQL> SET 'execution.mode' = 'kubernetes-application'; +Flink SQL> SET 'execution.target' = 'kubernetes-application'; [INFO] Execute statement succeeded. Flink SQL> SET 'kubernetes.cluster-id' = 'flink-cluster-mt-application-1'; @@ -156,23 +156,23 @@ FLINK SQL> CREATE MATERIALIZED TABLE my_materialized_table > ... [INFO] Execute statement succeeded. ``` -设置 `execution.mode` 为 `kubernetes-application` ，`kubernetes.cluster-id` 是一个可选配置，如果未配置，在提交作业时会自动生成。 +设置 `execution.target` 为 `kubernetes-application` ，`kubernetes.cluster-id` 是一个可选配置，如果未配置，在提交作业时会自动生成。 **YARN application 模式：** ```sql -Flink SQL> SET 'execution.mode' = 'yarn-application'; +Flink SQL> SET 'execution.target' = 'yarn-application'; [INFO] Execute statement succeeded. FLINK SQL> CREATE MATERIALIZED TABLE my_materialized_table > ... [INFO] Execute statement succeeded. ``` -设置 `execution.mode` 为 `yarn-application` ，`yarn.application.id` 无需配置。 +设置 `execution.target` 为 `yarn-application` ，`yarn.application.id` 无需配置。 ## 运维操作 -集群信息（如 `execution.mode` 或 `kubernetes.cluster-id`）已持久化在 Catalog 中，暂停或恢复物化表刷新作业时无需重复设置。 +集群信息（如 `execution.target` 或 `kubernetes.cluster-id`）已持久化在 Catalog 中，暂停或恢复物化表刷新作业时无需重复设置。 ### 暂停刷新作业 ```sql diff --git a/docs/content/docs/sql/materialized-table/deployment.md b/docs/content/docs/sql/materialized-table/deployment.md index 04a0d326d559cd..4b8d0107b4f092 100644 --- a/docs/content/docs/sql/materialized-table/deployment.md +++ b/docs/content/docs/sql/materialized-table/deployment.md @@ -49,7 +49,7 @@ Materialized Table refresh jobs currently support execution in these cluster env ## Flink SQL Gateway Deployment -Materialized Tables must be created through SQL Gateway, which requires specific configurations for metadata persistence and job scheduling. +Materialized Tables in full refresh mode must be created and operated through SQL Gateway, which schedules their refresh workflows. SQL Gateway requires specific configurations for metadata persistence and job scheduling. Materialized Tables in continuous refresh mode can also be created from any `TableEnvironment` (see [Using Table API](#using-table-api)). ### Configure Catalog Store @@ -99,7 +99,7 @@ Example using SQL Client: ### Refresh Jobs Running on Standalone Cluster ```sql -Flink SQL> SET 'execution.mode' = 'remote'; +Flink SQL> SET 'execution.target' = 'remote'; [INFO] Execute statement succeeded. FLINK SQL> CREATE MATERIALIZED TABLE my_materialized_table @@ -114,7 +114,7 @@ For session modes, pre-create session cluster as documented in [yarn-session]({ **Kubernetes session mode:** ```sql -Flink SQL> SET 'execution.mode' = 'kubernetes-session'; +Flink SQL> SET 'execution.target' = 'kubernetes-session'; [INFO] Execute statement succeeded. Flink SQL> SET 'kubernetes.cluster-id' = 'flink-cluster-mt-session-1'; @@ -124,12 +124,12 @@ Flink SQL> CREATE MATERIALIZED TABLE my_materialized_table > ... ; [INFO] Execute statement succeeded. ``` -Set `execution.mode` to `kubernetes-session` and specify a valid `kubernetes.cluster-id` corresponding to an existing Kubernetes session cluster. +Set `execution.target` to `kubernetes-session` and specify a valid `kubernetes.cluster-id` corresponding to an existing Kubernetes session cluster. **YARN session mode:** ```sql -Flink SQL> SET 'execution.mode' = 'yarn-session'; +Flink SQL> SET 'execution.target' = 'yarn-session'; [INFO] Execute statement succeeded. Flink SQL> SET 'yarn.application.id' = 'application-xxxx'; @@ -139,14 +139,14 @@ Flink SQL> CREATE MATERIALIZED TABLE my_materialized_table > ... ; [INFO] Execute statement succeeded. ``` -Set `execution.mode` to `yarn-session` and specify a valid `yarn.application.id` corresponding to an existing YARN session cluster. +Set `execution.target` to `yarn-session` and specify a valid `yarn.application.id` corresponding to an existing YARN session cluster. ### Refresh Jobs Running in Application Mode **Kubernetes application mode:** ```sql -Flink SQL> SET 'execution.mode' = 'kubernetes-application'; +Flink SQL> SET 'execution.target' = 'kubernetes-application'; [INFO] Execute statement succeeded. Flink SQL> SET 'kubernetes.cluster-id' = 'flink-cluster-mt-application-1'; @@ -156,23 +156,23 @@ Flink SQL> CREATE MATERIALIZED TABLE my_materialized_table > ... ; [INFO] Execute statement succeeded. ``` -Set `execution.mode` to `kubernetes-application`. The `kubernetes.cluster-id` is optional; if not set, it will be automatically generated. +Set `execution.target` to `kubernetes-application`. The `kubernetes.cluster-id` is optional; if not set, it will be automatically generated. **YARN application mode:** ```sql -Flink SQL> SET 'execution.mode' = 'yarn-application'; +Flink SQL> SET 'execution.target' = 'yarn-application'; [INFO] Execute statement succeeded. Flink SQL> CREATE MATERIALIZED TABLE my_materialized_table > ... ; [INFO] Execute statement succeeded. ``` -Only set `execution.mode` to yarn-application. The `yarn.application.id` doesn’t need to be set; it will be automatically generated during submission. +Only set `execution.target` to yarn-application. The `yarn.application.id` doesn’t need to be set; it will be automatically generated during submission. ## Maintenance Operations -Cluster information (e.g., `execution.mode` or `kubernetes.cluster-id`) is already persisted in the catalog and does not need to be set when suspend or resume the refresh jobs of Materialized Table. +Cluster information (e.g., `execution.target` or `kubernetes.cluster-id`) is already persisted in the catalog and does not need to be set when suspending or dropping a Materialized Table. Resuming a Materialized Table, or altering an activated one, submits a new refresh job to the session's `execution.target` and cluster id, so set them as when creating the Materialized Table. ### Suspend Refresh Job @@ -199,3 +199,29 @@ Flink SQL> ALTER MATERIALIZED TABLE my_materialized_table > ... ; [INFO] Execute statement succeeded. ``` + +# Using Table API + +Materialized tables in continuous refresh mode can also be managed from any `TableEnvironment`. Full refresh mode still requires SQL Gateway, which schedules the refresh workflows. + +Configure the session cluster on the program's `TableConfig`, or in the configuration the program runs with: + +| Session cluster | Options | +|---|---| +| Standalone | `execution.target: remote`, `rest.address`, `rest.port` | +| YARN session | `execution.target: yarn-session`, `yarn.application.id` | +| Kubernetes session | `execution.target: kubernetes-session`, `kubernetes.cluster-id` | + +```java +TableEnvironment tEnv = TableEnvironment.create(EnvironmentSettings.inStreamingMode()); +tEnv.getConfig().set("execution.target", "remote"); +tEnv.getConfig().set("rest.address", "flink-jobmanager"); +tEnv.getConfig().set("execution.checkpointing.savepoint-dir", "s3://my-bucket/savepoints"); + +tEnv.executeSql("CREATE MATERIALIZED TABLE my_materialized_table ..."); +``` + +* A statement that submits a refresh job fails before anything changes if the target can't host one: an unset, `local` or `embedded` target, an application target, or a session target without its cluster id. Application mode requires SQL Gateway. +* Suspending a materialized table, and altering an activated one, stop the refresh job with a savepoint and require `execution.checkpointing.savepoint-dir`. +* Any program that uses the same catalog, and SQL Gateway, can operate the materialized table. The refresh job is reached through the execution target and cluster id recorded in the catalog. For `remote`, configure the same `rest.address` as the program that submitted it. +* Resuming a materialized table, and altering an activated one, restart the refresh job on the program's own target, which must match the recorded one. A refresh job that SQL Gateway deployed in its own application cluster can be suspended, refreshed and dropped from a program, but must be resumed or altered through SQL Gateway. diff --git a/docs/content/docs/sql/materialized-table/statements.md b/docs/content/docs/sql/materialized-table/statements.md index 2dbb49a8902ed0..712de4142fa677 100644 --- a/docs/content/docs/sql/materialized-table/statements.md +++ b/docs/content/docs/sql/materialized-table/statements.md @@ -696,7 +696,7 @@ ALTER MATERIALIZED TABLE my_materialized_table SET ('format' = 'json', 'sink.par ALTER MATERIALIZED TABLE my_materialized_table SET ('format' = 'json', 'format' = 'csv'); ``` -Note When run through the Flink SQL Gateway, the behavior depends on the refresh mode and current refresh status: +Note When run through the Flink SQL Gateway or a `TableEnvironment`, the behavior depends on the refresh mode and current refresh status: - `FULL` mode: the change is applied to the catalog. The refresh workflow is not touched. - `CONTINUOUS` mode, `ACTIVATED` status: the running refresh job is stopped with savepoint, the change is applied to the catalog, and a new refresh job is started using the updated options. The new job does **not** restore from the savepoint taken during suspend, so streaming state is reset. - `CONTINUOUS` mode, `SUSPENDED` status: the change is applied to the catalog and the savepoint stored in the refresh handler is cleared, so the next `RESUME` will also start a fresh job. @@ -730,7 +730,7 @@ ALTER MATERIALIZED TABLE my_materialized_table RESET ('format', 'sink.parallelis ALTER MATERIALIZED TABLE my_materialized_table RESET ('format', 'format'); ``` -Note When run through the Flink SQL Gateway, the behavior depends on the refresh mode and current refresh status: +Note When run through the Flink SQL Gateway or a `TableEnvironment`, the behavior depends on the refresh mode and current refresh status: - `FULL` mode: the change is applied to the catalog. The refresh workflow is not touched. - `CONTINUOUS` mode, `ACTIVATED` status: the running refresh job is stopped with savepoint, the change is applied to the catalog, and a new refresh job is started using the updated options. The new job does **not** restore from the savepoint taken during suspend, so streaming state is reset. - `CONTINUOUS` mode, `SUSPENDED` status: the change is applied to the catalog and the savepoint stored in the refresh handler is cleared, so the next `RESUME` will also start a fresh job. diff --git a/flink-clients/src/main/java/org/apache/flink/client/deployment/executors/AbstractSessionClusterExecutor.java b/flink-clients/src/main/java/org/apache/flink/client/deployment/executors/AbstractSessionClusterExecutor.java index e1a667d159271c..e04e01b6eb3eb4 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/deployment/executors/AbstractSessionClusterExecutor.java +++ b/flink-clients/src/main/java/org/apache/flink/client/deployment/executors/AbstractSessionClusterExecutor.java @@ -19,6 +19,7 @@ package org.apache.flink.client.deployment.executors; import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.JobID; import org.apache.flink.api.dag.Pipeline; import org.apache.flink.client.ClientUtils; import org.apache.flink.client.deployment.ClusterClientFactory; @@ -27,8 +28,10 @@ import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.ClusterClientProvider; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.DeploymentOptions; import org.apache.flink.core.execution.CacheSupportedPipelineExecutor; import org.apache.flink.core.execution.JobClient; +import org.apache.flink.core.execution.JobRetrievalSupportedPipelineExecutor; import org.apache.flink.core.execution.JobStatusChangedListener; import org.apache.flink.core.execution.JobStatusChangedListenerUtils; import org.apache.flink.core.execution.PipelineExecutor; @@ -36,6 +39,7 @@ import org.apache.flink.streaming.api.graph.StreamGraph; import org.apache.flink.util.AbstractID; import org.apache.flink.util.concurrent.ExecutorThreadFactory; +import org.apache.flink.util.concurrent.FutureUtils; import org.apache.flink.util.function.FunctionUtils; import org.slf4j.Logger; @@ -63,7 +67,7 @@ @Internal public class AbstractSessionClusterExecutor< ClusterID, ClientFactory extends ClusterClientFactory> - implements CacheSupportedPipelineExecutor { + implements CacheSupportedPipelineExecutor, JobRetrievalSupportedPipelineExecutor { private static final Logger LOG = LoggerFactory.getLogger(AbstractSessionClusterExecutor.class); private final ExecutorService executorService = Executors.newFixedThreadPool( @@ -135,6 +139,28 @@ public CompletableFuture execute( } } + @Override + public CompletableFuture retrieveJobClient( + JobID jobId, Configuration configuration, ClassLoader userCodeClassloader) { + try { + final ClusterID clusterId = clusterClientFactory.getClusterId(configuration); + checkState( + clusterId != null, + "No cluster id is configured for execution target '%s'.", + configuration.get(DeploymentOptions.TARGET)); + + try (ClusterDescriptor clusterDescriptor = + clusterClientFactory.createClusterDescriptor( + new Configuration(configuration))) { + return CompletableFuture.completedFuture( + new ClusterClientJobClientAdapter<>( + clusterDescriptor.retrieve(clusterId), jobId, userCodeClassloader)); + } + } catch (Exception e) { + return FutureUtils.completedExceptionally(e); + } + } + @Override public CompletableFuture> listCompletedClusterDatasetIds( Configuration configuration, ClassLoader userCodeClassloader) throws Exception { diff --git a/flink-core/src/main/java/org/apache/flink/core/execution/JobRetrievalSupportedPipelineExecutor.java b/flink-core/src/main/java/org/apache/flink/core/execution/JobRetrievalSupportedPipelineExecutor.java new file mode 100644 index 00000000000000..6c36119a0f88e2 --- /dev/null +++ b/flink-core/src/main/java/org/apache/flink/core/execution/JobRetrievalSupportedPipelineExecutor.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.core.execution; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.JobID; +import org.apache.flink.configuration.Configuration; + +import java.util.concurrent.CompletableFuture; + +/** A pipeline executor that can reach jobs it did not submit. */ +@Internal +public interface JobRetrievalSupportedPipelineExecutor extends PipelineExecutor { + + /** + * A JobClient for an existing job on the cluster the configuration names. Fails if the cluster + * can't be retrieved. + */ + CompletableFuture retrieveJobClient( + JobID jobId, Configuration configuration, ClassLoader userCodeClassloader); +} diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/SqlGatewayServiceImpl.java b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/SqlGatewayServiceImpl.java index 371d8aa8993437..7752eb7f55789d 100644 --- a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/SqlGatewayServiceImpl.java +++ b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/SqlGatewayServiceImpl.java @@ -355,6 +355,17 @@ public ClusterID deployScript( Configuration mergedConfig = Configuration.fromMap(session.getSessionConfig()); mergedConfig.addAll(executionConfig); + try { + return deployApplicationCluster(mergedConfig, scriptUri, script); + } catch (Throwable t) { + LOG.error("Failed to deploy script to cluster.", t); + throw new SqlGatewayException("Failed to deploy script to cluster.", t); + } + } + + public static ClusterID deployApplicationCluster( + Configuration mergedConfig, @Nullable URI scriptUri, @Nullable String script) + throws Exception { List arguments = new ArrayList<>(); if (scriptUri != null) { arguments.add("--" + SqlDriver.OPTION_SQL_FILE.getLongOpt()); @@ -368,13 +379,8 @@ public ClusterID deployScript( ApplicationConfiguration applicationConfiguration = new ApplicationConfiguration( arguments.toArray(new String[0]), SqlDriver.class.getName()); - try { - return new ApplicationClusterDeployer(new DefaultClusterClientServiceLoader()) - .run(mergedConfig, applicationConfiguration); - } catch (Throwable t) { - LOG.error("Failed to deploy script to cluster.", t); - throw new SqlGatewayException("Failed to deploy script to cluster.", t); - } + return new ApplicationClusterDeployer(new DefaultClusterClientServiceLoader()) + .run(mergedConfig, applicationConfiguration); } @Override diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/SessionContext.java b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/SessionContext.java index b5463d44267fea..d4604b7fd4edbd 100644 --- a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/SessionContext.java +++ b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/context/SessionContext.java @@ -24,6 +24,7 @@ import org.apache.flink.configuration.ReadableConfig; import org.apache.flink.configuration.UnmodifiableConfiguration; import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.bridge.internal.AbstractStreamTableEnvironmentImpl; import org.apache.flink.table.api.config.TableConfigOptions; import org.apache.flink.table.api.internal.PlanCacheManager; import org.apache.flink.table.catalog.Catalog; @@ -31,6 +32,7 @@ import org.apache.flink.table.catalog.CatalogStoreHolder; import org.apache.flink.table.catalog.FunctionCatalog; import org.apache.flink.table.catalog.GenericInMemoryCatalog; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutorFactory; import org.apache.flink.table.factories.ApiFactoryUtil; import org.apache.flink.table.factories.CatalogStoreFactory; import org.apache.flink.table.factories.FactoryUtil; @@ -39,7 +41,7 @@ import org.apache.flink.table.gateway.api.session.SessionEnvironment; import org.apache.flink.table.gateway.api.session.SessionHandle; import org.apache.flink.table.gateway.api.utils.SqlGatewayException; -import org.apache.flink.table.gateway.service.materializedtable.MaterializedTableManager; +import org.apache.flink.table.gateway.service.materializedtable.MaterializedTableContext; import org.apache.flink.table.gateway.service.operation.OperationExecutor; import org.apache.flink.table.gateway.service.operation.OperationManager; import org.apache.flink.table.gateway.service.utils.SqlExecutionException; @@ -241,7 +243,7 @@ public void addStatementSetOperation(ModifyOperation operation) { public void open() { try { - sessionState.materializedTableManager.open(); + sessionState.materializedTableContext.open(); } catch (Exception e) { LOG.error( String.format( @@ -281,7 +283,7 @@ public void close() { e); } try { - sessionState.materializedTableManager.close(); + sessionState.materializedTableContext.close(); } catch (Exception e) { LOG.error( String.format( @@ -353,14 +355,18 @@ protected static SessionState initializeSessionState( final FunctionCatalog functionCatalog = new FunctionCatalog(configuration, resourceManager, catalogManager, moduleManager); - final MaterializedTableManager materializedTableManager = - new MaterializedTableManager(configuration, resourceManager.getUserClassLoader()); + final MaterializedTableContext materializedTableContext = + new MaterializedTableContext(configuration, resourceManager.getUserClassLoader()); + final MaterializedTableExecutorFactory materializedTableExecutorFactory = + AbstractStreamTableEnvironmentImpl.lookupMaterializedTableExecutorFactory( + resourceManager.getUserClassLoader()); return new SessionState( catalogManager, moduleManager, resourceManager, functionCatalog, - materializedTableManager); + materializedTableContext, + materializedTableExecutorFactory); } private static ModuleManager buildModuleManager( @@ -481,19 +487,22 @@ public static class SessionState { public final ResourceManager resourceManager; public final FunctionCatalog functionCatalog; public final ModuleManager moduleManager; - public final MaterializedTableManager materializedTableManager; + public final MaterializedTableContext materializedTableContext; + public final MaterializedTableExecutorFactory materializedTableExecutorFactory; public SessionState( CatalogManager catalogManager, ModuleManager moduleManager, ResourceManager resourceManager, FunctionCatalog functionCatalog, - MaterializedTableManager materializedTableManager) { + MaterializedTableContext materializedTableContext, + MaterializedTableExecutorFactory materializedTableExecutorFactory) { this.catalogManager = catalogManager; this.moduleManager = moduleManager; this.resourceManager = resourceManager; this.functionCatalog = functionCatalog; - this.materializedTableManager = materializedTableManager; + this.materializedTableContext = materializedTableContext; + this.materializedTableExecutorFactory = materializedTableExecutorFactory; } } } diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/GatewayMaterializedTableJobSubmitter.java b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/GatewayMaterializedTableJobSubmitter.java new file mode 100644 index 00000000000000..2cb705815f3fcd --- /dev/null +++ b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/GatewayMaterializedTableJobSubmitter.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.gateway.service.materializedtable; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.JobID; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.api.config.TableConfigOptions; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableJobSubmitter; +import org.apache.flink.table.delegation.materializedtable.RefreshJobResult; +import org.apache.flink.table.delegation.materializedtable.RefreshJobTarget; +import org.apache.flink.table.delegation.materializedtable.RefreshWorkflowContext; +import org.apache.flink.table.gateway.api.operation.OperationHandle; +import org.apache.flink.table.gateway.api.results.ResultSet; +import org.apache.flink.table.gateway.api.utils.SqlGatewayException; +import org.apache.flink.table.gateway.service.SqlGatewayServiceImpl; +import org.apache.flink.table.gateway.service.operation.OperationExecutor; +import org.apache.flink.table.gateway.service.result.ResultFetcher; +import org.apache.flink.table.refresh.RefreshHandler; +import org.apache.flink.table.workflow.WorkflowScheduler; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.apache.flink.configuration.DeploymentOptions.TARGET; +import static org.apache.flink.configuration.PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID; +import static org.apache.flink.table.factories.WorkflowSchedulerFactoryUtil.WORKFLOW_SCHEDULER_PREFIX; + +/** SQL Gateway implementation of {@link MaterializedTableJobSubmitter}. */ +@Internal +public class GatewayMaterializedTableJobSubmitter implements MaterializedTableJobSubmitter { + + private static final Logger LOG = + LoggerFactory.getLogger(GatewayMaterializedTableJobSubmitter.class); + + private final OperationExecutor operationExecutor; + private final OperationHandle handle; + private final MaterializedTableContext context; + + public GatewayMaterializedTableJobSubmitter( + OperationExecutor operationExecutor, + OperationHandle handle, + MaterializedTableContext context) { + this.operationExecutor = operationExecutor; + this.handle = handle; + this.context = context; + } + + @Override + public RefreshJobResult submitRefreshJob( + Configuration executionConfig, String insertStatement) { + final String executionTarget = + operationExecutor.getSessionContext().getSessionConf().get(TARGET); + if (executionTarget == null + || executionTarget.isEmpty() + || "local".equals(executionTarget)) { + final String errorMessage = + String.format( + "Unsupported execution target detected: %s." + + "Currently, only the following execution targets are supported: " + + "'remote', 'yarn-session', 'yarn-application', 'kubernetes-session', 'kubernetes-application'. ", + executionTarget); + LOG.error(errorMessage); + throw new ValidationException(errorMessage); + } + + if (executionTarget.endsWith("application")) { + return deployApplication(executionConfig, insertStatement); + } + + final String clusterId = + operationExecutor + .getSessionClusterId() + .orElseThrow( + () -> { + final String errorMessage = + String.format( + "No cluster ID found when executing materialized table refresh job. Execution target is : %s", + executionTarget); + LOG.error(errorMessage); + return new ValidationException(errorMessage); + }); + final ResultFetcher resultFetcher = + operationExecutor.executeStatement(handle, executionConfig, insertStatement); + final List results = fetchAllResults(resultFetcher); + final String jobId = results.get(0).getString(0).toString(); + + return new RefreshJobResult(new RefreshJobTarget(executionTarget, clusterId), jobId, null); + } + + @Override + public boolean supportsApplicationTargets() { + return true; + } + + @Override + public Optional getRefreshWorkflowContext() { + return Optional.ofNullable(context.workflowScheduler()) + .map(GatewayRefreshWorkflowContext::new); + } + + private RefreshJobResult deployApplication( + Configuration executionConfig, String insertStatement) { + final Configuration mergedConfig = + new Configuration(operationExecutor.getSessionContext().getSessionConf()); + mergedConfig.addAll(executionConfig); + final JobID jobId = new JobID(); + mergedConfig.set(PIPELINE_FIXED_JOB_ID, jobId.toString()); + + try { + final String clusterId = + SqlGatewayServiceImpl.deployApplicationCluster( + mergedConfig, null, insertStatement) + .toString(); + + return new RefreshJobResult( + new RefreshJobTarget(mergedConfig.get(TARGET), clusterId), + jobId.toString(), + null); + } catch (Throwable t) { + LOG.error("Failed to deploy script {} to application cluster.", insertStatement, t); + throw new SqlGatewayException("Failed to deploy script to cluster.", t); + } + } + + private static List fetchAllResults(ResultFetcher resultFetcher) { + Long token = 0L; + List results = new ArrayList<>(); + while (token != null) { + ResultSet result = resultFetcher.fetchResults(token, Integer.MAX_VALUE); + results.addAll(result.getData()); + token = result.getNextToken(); + } + return results; + } + + /** Gateway {@link RefreshWorkflowContext}. */ + private class GatewayRefreshWorkflowContext implements RefreshWorkflowContext { + + private final WorkflowScheduler scheduler; + + private GatewayRefreshWorkflowContext( + WorkflowScheduler scheduler) { + this.scheduler = scheduler; + } + + @Override + public WorkflowScheduler scheduler() { + return scheduler; + } + + @Override + public String restEndpointUrl() { + return context.restEndpointUrl(); + } + + @Override + public Map sessionInitializationConf() { + Map sessionConf = + operationExecutor.getSessionContext().getSessionConf().toMap(); + + // we only keep the session conf that is not in the default context or the conf value is + // different from the default context. + Map defaultContextConf = + operationExecutor + .getSessionContext() + .getDefaultContext() + .getFlinkConfig() + .toMap(); + sessionConf + .entrySet() + .removeIf( + entry -> { + String key = entry.getKey(); + String value = entry.getValue(); + return defaultContextConf.containsKey(key) + && defaultContextConf.get(key).equals(value); + }); + + // remove useless conf + sessionConf.remove(TableConfigOptions.RESOURCES_DOWNLOAD_DIR.key()); + sessionConf.keySet().removeIf(key -> key.startsWith(WORKFLOW_SCHEDULER_PREFIX)); + + return sessionConf; + } + } +} diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableContext.java b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableContext.java new file mode 100644 index 00000000000000..b0038c14a8175c --- /dev/null +++ b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableContext.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.gateway.service.materializedtable; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.factories.WorkflowSchedulerFactoryUtil; +import org.apache.flink.table.gateway.rest.SqlGatewayRestEndpointFactory; +import org.apache.flink.table.gateway.rest.util.SqlGatewayRestOptions; +import org.apache.flink.table.refresh.RefreshHandler; +import org.apache.flink.table.workflow.WorkflowScheduler; + +import javax.annotation.Nullable; + +import static org.apache.flink.table.gateway.api.endpoint.SqlGatewayEndpointFactoryUtils.getEndpointConfig; + +/** + * Session-scoped holder for the SQL Gateway capabilities a {@link + * GatewayMaterializedTableJobSubmitter} needs to run materialized table jobs. It owns the lifecycle + * of the workflow scheduler and is created once per session in {@code SessionContext}. + */ +@Internal +public class MaterializedTableContext { + + private final @Nullable WorkflowScheduler workflowScheduler; + + private final String restEndpointUrl; + + public MaterializedTableContext(Configuration configuration, ClassLoader userCodeClassLoader) { + this.restEndpointUrl = buildRestEndpointUrl(configuration); + this.workflowScheduler = buildWorkflowScheduler(configuration, userCodeClassLoader); + } + + public @Nullable WorkflowScheduler workflowScheduler() { + return workflowScheduler; + } + + public String restEndpointUrl() { + return restEndpointUrl; + } + + public void open() throws Exception { + if (workflowScheduler != null) { + workflowScheduler.open(); + } + } + + public void close() throws Exception { + if (workflowScheduler != null) { + workflowScheduler.close(); + } + } + + private static String buildRestEndpointUrl(Configuration configuration) { + Configuration restEndpointConfig = + Configuration.fromMap( + getEndpointConfig(configuration, SqlGatewayRestEndpointFactory.IDENTIFIER)); + String address = restEndpointConfig.get(SqlGatewayRestOptions.ADDRESS); + int port = restEndpointConfig.get(SqlGatewayRestOptions.PORT); + + return String.format("http://%s:%s", address, port); + } + + private static WorkflowScheduler buildWorkflowScheduler( + Configuration configuration, ClassLoader userCodeClassLoader) { + return WorkflowSchedulerFactoryUtil.createWorkflowScheduler( + configuration, userCodeClassLoader); + } +} diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/operation/OperationExecutor.java b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/operation/OperationExecutor.java index 4254b7ef6e6ad0..1eb2515d9ae7cc 100644 --- a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/operation/OperationExecutor.java +++ b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/operation/OperationExecutor.java @@ -58,6 +58,7 @@ import org.apache.flink.table.delegation.Executor; import org.apache.flink.table.delegation.ExecutorFactory; import org.apache.flink.table.delegation.Planner; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableJobSubmitter; import org.apache.flink.table.factories.FactoryUtil; import org.apache.flink.table.factories.PlannerFactoryUtil; import org.apache.flink.table.functions.FunctionDefinition; @@ -67,6 +68,7 @@ import org.apache.flink.table.gateway.api.results.TableInfo; import org.apache.flink.table.gateway.environment.SqlGatewayStreamExecutionEnvironment; import org.apache.flink.table.gateway.service.context.SessionContext; +import org.apache.flink.table.gateway.service.materializedtable.GatewayMaterializedTableJobSubmitter; import org.apache.flink.table.gateway.service.result.ResultFetcher; import org.apache.flink.table.gateway.service.utils.SqlExecutionException; import org.apache.flink.table.module.ModuleManager; @@ -120,6 +122,7 @@ import org.apache.flink.table.operations.ddl.DropTempSystemFunctionOperation; import org.apache.flink.table.operations.ddl.DropViewOperation; import org.apache.flink.table.operations.materializedtable.AlterMaterializedTableOperation; +import org.apache.flink.table.operations.materializedtable.AlterMaterializedTableRefreshOperation; import org.apache.flink.table.operations.materializedtable.CreateMaterializedTableOperation; import org.apache.flink.table.operations.materializedtable.DropMaterializedTableOperation; import org.apache.flink.table.operations.materializedtable.MaterializedTableOperation; @@ -224,6 +227,8 @@ public ResultFetcher configureSession(OperationHandle handle, String statement) return callSetOperation(tableEnv, handle, (SetOperation) op); } else if (op instanceof ResetOperation) { return callResetOperation(handle, (ResetOperation) op); + } else if (op instanceof MaterializedTableOperation) { + return callExecutableOperation(handle, (ExecutableOperation) op); } else { return callOperation(tableEnv, handle, op); } @@ -237,7 +242,9 @@ public ResultFetcher executeStatement( OperationHandle handle, Configuration customConfig, String statement) { // Instantiate the TableEnvironment lazily ResourceManager resourceManager = sessionContext.getSessionState().resourceManager.copy(); - TableEnvironmentInternal tableEnv = getTableEnvironment(resourceManager, customConfig); + TableEnvironmentInternal tableEnv = + getTableEnvironment( + resourceManager, customConfig, materializedTableJobSubmitter(handle)); PlanCacheManager planCacheManager = sessionContext.getPlanCacheManager(); CachedPlan cachedPlan = null; Operation op = null; @@ -317,15 +324,6 @@ public Set listTables( } } - public boolean tableExists(ObjectIdentifier tableIdentifier) { - return sessionContext - .getSessionState() - .catalogManager - .getCatalog(tableIdentifier.getCatalogName()) - .map(catalog -> catalog.tableExists(tableIdentifier.toObjectPath())) - .orElse(false); - } - public ResolvedCatalogBaseTable getTable(ObjectIdentifier tableIdentifier) { return getTableEnvironment() .getCatalogManager() @@ -405,6 +403,13 @@ public TableEnvironmentInternal getTableEnvironment() { public TableEnvironmentInternal getTableEnvironment( ResourceManager resourceManager, Configuration customConfig) { + return getTableEnvironment(resourceManager, customConfig, null); + } + + private TableEnvironmentInternal getTableEnvironment( + ResourceManager resourceManager, + Configuration customConfig, + @Nullable MaterializedTableJobSubmitter materializedTableJobSubmitter) { // checks the value of RUNTIME_MODE Configuration operationConfig = sessionContext.getSessionConf().clone(); operationConfig.addAll(executionConfig); @@ -429,7 +434,13 @@ public TableEnvironmentInternal getTableEnvironment( sessionContext.getSessionState().catalogManager, sessionContext.getSessionState().moduleManager, resourceManager, - sessionContext.getSessionState().functionCatalog.copy(resourceManager)); + sessionContext.getSessionState().functionCatalog.copy(resourceManager), + materializedTableJobSubmitter); + } + + private MaterializedTableJobSubmitter materializedTableJobSubmitter(OperationHandle handle) { + return new GatewayMaterializedTableJobSubmitter( + this, handle, sessionContext.getSessionState().materializedTableContext); } public Optional getSessionClusterId() { @@ -469,7 +480,8 @@ private TableEnvironmentInternal createStreamTableEnvironment( CatalogManager catalogManager, ModuleManager moduleManager, ResourceManager resourceManager, - FunctionCatalog functionCatalog) { + FunctionCatalog functionCatalog, + @Nullable MaterializedTableJobSubmitter materializedTableJobSubmitter) { final Planner planner = PlannerFactoryUtil.createPlanner( @@ -489,7 +501,9 @@ private TableEnvironmentInternal createStreamTableEnvironment( env, planner, executor, - settings.isStreamingMode()); + settings.isStreamingMode(), + sessionContext.getSessionState().materializedTableExecutorFactory, + materializedTableJobSubmitter); } private ResultFetcher executeOperationInStatementSetState( @@ -555,11 +569,6 @@ private ResultFetcher executeOperation( || op instanceof ShowFunctionsOperation || op instanceof DescribeFunctionOperation) { return callExecutableOperation(handle, (ExecutableOperation) op); - } else if (op instanceof MaterializedTableOperation) { - return sessionContext - .getSessionState() - .materializedTableManager - .callMaterializedTableOperation(this, handle, (MaterializedTableOperation) op); } else { return callOperation(tableEnv, handle, op); } @@ -585,22 +594,22 @@ public ResultFetcher refreshMaterializedTable( @Nullable String scheduleTime, Map staticPartitions, Map dynamicOptions) { - TableEnvironmentInternal tEnv = getTableEnvironment(); + TableEnvironmentInternal tEnv = + getTableEnvironment( + sessionContext.getSessionState().resourceManager, + new Configuration(), + materializedTableJobSubmitter(handle)); UnresolvedIdentifier unresolvedIdentifier = tEnv.getParser().parseIdentifier(materializedTableIdentifier); ObjectIdentifier objectIdentifier = tEnv.getCatalogManager().qualifyIdentifier(unresolvedIdentifier); - return sessionContext - .getSessionState() - .materializedTableManager - .refreshMaterializedTable( - this, - handle, - objectIdentifier, - staticPartitions, - dynamicOptions, - isPeriodic, - scheduleTime); + AlterMaterializedTableRefreshOperation refreshOperation = + isPeriodic + ? AlterMaterializedTableRefreshOperation.periodic( + objectIdentifier, scheduleTime, dynamicOptions) + : AlterMaterializedTableRefreshOperation.oneTime( + objectIdentifier, staticPartitions, dynamicOptions); + return callOperation(tEnv, handle, refreshOperation); } private TableConfig tableConfig() { @@ -780,7 +789,7 @@ private Set listViews(String catalogName, String databaseName) { .collect(Collectors.toUnmodifiableSet()); } - public ResultFetcher callStopJobOperation( + private ResultFetcher callStopJobOperation( TableEnvironmentInternal tableEnv, OperationHandle handle, StopJobOperation stopJobOperation) @@ -879,7 +888,7 @@ public ResultFetcher callShowJobsOperation( resultRows); } - public ResultFetcher callDescribeJobOperation( + private ResultFetcher callDescribeJobOperation( TableEnvironmentInternal tableEnv, OperationHandle operationHandle, DescribeJobOperation describeJobOperation) diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/utils/Constants.java b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/utils/Constants.java index be72df7a94f6bb..c2ceb435fe261e 100644 --- a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/utils/Constants.java +++ b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/utils/Constants.java @@ -29,5 +29,4 @@ public class Constants { public static final String SET_VALUE = "value"; public static final String COMPLETION_CANDIDATES = "candidates"; public static final String SAVEPOINT_PATH = "savepoint path"; - public static final String CLUSTER_INFO = "cluster info"; } diff --git a/flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/service/MaterializedTableStatementITCase.java b/flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/service/MaterializedTableStatementITCase.java index f71337cfd11e4c..65e35f87b660ae 100644 --- a/flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/service/MaterializedTableStatementITCase.java +++ b/flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/service/MaterializedTableStatementITCase.java @@ -33,6 +33,7 @@ import org.apache.flink.runtime.rest.messages.checkpoints.CheckpointingStatisticsHeaders; import org.apache.flink.runtime.rest.util.RestMapperUtils; import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.catalog.CatalogMaterializedTable.LogicalRefreshMode; import org.apache.flink.table.catalog.CatalogMaterializedTable.RefreshMode; @@ -819,7 +820,7 @@ void testAlterMaterializedTableWithRepeatedSuspendAndResumeInContinuousMode( sessionHandle, repeatedAlterMaterializedTableSuspendHandle)) .rootCause() - .isInstanceOf(SqlExecutionException.class) + .isInstanceOf(TableException.class) .hasMessageContaining( String.format( "Materialized table %s continuous refresh job has been suspended", @@ -841,7 +842,7 @@ void testAlterMaterializedTableWithRepeatedSuspendAndResumeInContinuousMode( sessionHandle, repeatedAlterMaterializedTableResumeHandle)) .rootCause() - .isInstanceOf(SqlExecutionException.class) + .isInstanceOf(TableException.class) .hasMessageContaining( String.format( "Materialized table %s continuous refresh job has been resumed", @@ -950,7 +951,7 @@ void testAlterMaterializedTableWithRepeatedSuspendAndResumeInFullMode() throws E sessionHandle, repeatedAlterMaterializedTableSuspendHandle)) .rootCause() - .isInstanceOf(SqlExecutionException.class) + .isInstanceOf(TableException.class) .hasMessageContaining( String.format( "Materialized table %s refresh workflow has been suspended.", @@ -973,7 +974,7 @@ void testAlterMaterializedTableWithRepeatedSuspendAndResumeInFullMode() throws E sessionHandle, repeatedAlterMaterializedTableResumeHandle)) .rootCause() - .isInstanceOf(SqlExecutionException.class) + .isInstanceOf(TableException.class) .hasMessageContaining( String.format( "Materialized table %s refresh workflow has been resumed.", @@ -1870,7 +1871,7 @@ void testRefreshMaterializedTableWithInvalidParameterInContinuousMode() throws E awaitOperationTermination( service, sessionHandle, invalidRefreshTableHandle2)) .rootCause() - .isInstanceOf(SqlExecutionException.class) + .isInstanceOf(TableException.class) .hasMessage( String.format( "Failed to parse a valid partition value for the field 'ds' in materialized table %s using the scheduler time '20240103 00:00:00.000' based on the date format 'yyyy-MM-dd HH:mm:ss'.", diff --git a/flink-table/flink-table-api-bridge-base/src/main/java/org/apache/flink/table/api/bridge/internal/AbstractStreamTableEnvironmentImpl.java b/flink-table/flink-table-api-bridge-base/src/main/java/org/apache/flink/table/api/bridge/internal/AbstractStreamTableEnvironmentImpl.java index 2e0bd471e5492e..15d32433d318ba 100644 --- a/flink-table/flink-table-api-bridge-base/src/main/java/org/apache/flink/table/api/bridge/internal/AbstractStreamTableEnvironmentImpl.java +++ b/flink-table/flink-table-api-bridge-base/src/main/java/org/apache/flink/table/api/bridge/internal/AbstractStreamTableEnvironmentImpl.java @@ -46,6 +46,8 @@ import org.apache.flink.table.delegation.ExecutorFactory; import org.apache.flink.table.delegation.Planner; import org.apache.flink.table.delegation.StreamExecutorFactory; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutorFactory; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableJobSubmitter; import org.apache.flink.table.expressions.ApiExpressionUtils; import org.apache.flink.table.expressions.Expression; import org.apache.flink.table.factories.FactoryUtil; @@ -86,6 +88,32 @@ public AbstractStreamTableEnvironmentImpl( Planner planner, boolean isStreamingMode, StreamExecutionEnvironment executionEnvironment) { + this( + catalogManager, + moduleManager, + resourceManager, + tableConfig, + executor, + functionCatalog, + planner, + isStreamingMode, + executionEnvironment, + lookupMaterializedTableExecutorFactory(resourceManager.getUserClassLoader()), + null); + } + + public AbstractStreamTableEnvironmentImpl( + CatalogManager catalogManager, + ModuleManager moduleManager, + ResourceManager resourceManager, + TableConfig tableConfig, + Executor executor, + FunctionCatalog functionCatalog, + Planner planner, + boolean isStreamingMode, + StreamExecutionEnvironment executionEnvironment, + MaterializedTableExecutorFactory materializedTableExecutorFactory, + @Nullable MaterializedTableJobSubmitter materializedTableJobSubmitter) { super( catalogManager, moduleManager, @@ -94,10 +122,20 @@ public AbstractStreamTableEnvironmentImpl( executor, functionCatalog, planner, - isStreamingMode); + isStreamingMode, + materializedTableExecutorFactory, + materializedTableJobSubmitter); this.executionEnvironment = executionEnvironment; } + public static MaterializedTableExecutorFactory lookupMaterializedTableExecutorFactory( + ClassLoader classLoader) { + return FactoryUtil.discoverFactory( + classLoader, + MaterializedTableExecutorFactory.class, + MaterializedTableExecutorFactory.DEFAULT_IDENTIFIER); + } + public static Executor lookupExecutor( ClassLoader classLoader, StreamExecutionEnvironment executionEnvironment) { final ExecutorFactory executorFactory; diff --git a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImpl.java b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImpl.java index 810546ed6a6a5e..52cdd12df7bc2b 100644 --- a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImpl.java +++ b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImpl.java @@ -45,6 +45,8 @@ import org.apache.flink.table.connector.ChangelogMode; import org.apache.flink.table.delegation.Executor; import org.apache.flink.table.delegation.Planner; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutorFactory; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableJobSubmitter; import org.apache.flink.table.expressions.Expression; import org.apache.flink.table.factories.ApiFactoryUtil; import org.apache.flink.table.factories.CatalogStoreFactory; @@ -62,6 +64,8 @@ import org.apache.flink.util.MutableURLClassLoader; import org.apache.flink.util.Preconditions; +import javax.annotation.Nullable; + import java.net.URL; import java.util.Arrays; import java.util.Optional; @@ -85,7 +89,9 @@ public StreamTableEnvironmentImpl( StreamExecutionEnvironment executionEnvironment, Planner planner, Executor executor, - boolean isStreamingMode) { + boolean isStreamingMode, + MaterializedTableExecutorFactory materializedTableExecutorFactory, + @Nullable MaterializedTableJobSubmitter materializedTableJobSubmitter) { super( catalogManager, moduleManager, @@ -95,7 +101,9 @@ public StreamTableEnvironmentImpl( functionCatalog, planner, isStreamingMode, - executionEnvironment); + executionEnvironment, + materializedTableExecutorFactory, + materializedTableJobSubmitter); } public static StreamTableEnvironment create( @@ -153,6 +161,9 @@ public static StreamTableEnvironment create( catalogManager, functionCatalog); + final MaterializedTableExecutorFactory materializedTableExecutorFactory = + lookupMaterializedTableExecutorFactory(userClassLoader); + return new StreamTableEnvironmentImpl( catalogManager, moduleManager, @@ -162,7 +173,9 @@ public static StreamTableEnvironment create( executionEnvironment, planner, executor, - settings.isStreamingMode()); + settings.isStreamingMode(), + materializedTableExecutorFactory, + null); } @Override diff --git a/flink-table/flink-table-api-java-bridge/src/test/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImplTest.java b/flink-table/flink-table-api-java-bridge/src/test/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImplTest.java index 63302f7f7889f9..e268c6187803c1 100644 --- a/flink-table/flink-table-api-java-bridge/src/test/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImplTest.java +++ b/flink-table/flink-table-api-java-bridge/src/test/java/org/apache/flink/table/api/bridge/java/internal/StreamTableEnvironmentImplTest.java @@ -30,6 +30,7 @@ import org.apache.flink.table.resource.ResourceManager; import org.apache.flink.table.utils.CatalogManagerMocks; import org.apache.flink.table.utils.ExecutorMock; +import org.apache.flink.table.utils.MaterializedTableExecutorFactoryMock; import org.apache.flink.table.utils.PlannerMock; import org.apache.flink.types.Row; @@ -94,7 +95,9 @@ private StreamTableEnvironmentImpl getStreamTableEnvironment( env, new TestPlanner(elements.getTransformation()), new ExecutorMock(), - true); + true, + new MaterializedTableExecutorFactoryMock(), + null); } private static class TestPlanner extends PlannerMock { diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java index 496369e78f269d..c574a460a4fdb4 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java @@ -47,6 +47,7 @@ import org.apache.flink.table.api.TableResult; import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.api.config.TableConfigOptions; +import org.apache.flink.table.api.internal.materializedtable.DefaultMaterializedTableJobSubmitter; import org.apache.flink.table.catalog.Catalog; import org.apache.flink.table.catalog.CatalogBaseTable; import org.apache.flink.table.catalog.CatalogDescriptor; @@ -76,6 +77,9 @@ import org.apache.flink.table.delegation.InternalPlan; import org.apache.flink.table.delegation.Parser; import org.apache.flink.table.delegation.Planner; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutor; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutorFactory; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableJobSubmitter; import org.apache.flink.table.execution.StagingSinkJobStatusHook; import org.apache.flink.table.expressions.ApiExpressionUtils; import org.apache.flink.table.expressions.DefaultSqlFactory; @@ -115,6 +119,7 @@ import org.apache.flink.table.operations.ddl.AnalyzeTableOperation; import org.apache.flink.table.operations.ddl.CompilePlanOperation; import org.apache.flink.table.operations.ddl.CreateTableOperation; +import org.apache.flink.table.operations.materializedtable.MaterializedTableOperation; import org.apache.flink.table.operations.utils.ExecutableOperationUtils; import org.apache.flink.table.operations.utils.OperationTreeBuilder; import org.apache.flink.table.resource.ResourceManager; @@ -132,6 +137,8 @@ import org.apache.flink.util.MutableURLClassLoader; import org.apache.flink.util.Preconditions; +import javax.annotation.Nullable; + import java.io.IOException; import java.net.URL; import java.util.ArrayList; @@ -173,6 +180,10 @@ public class TableEnvironmentImpl implements TableEnvironmentInternal { private final boolean isStreamingMode; private final ExecutableOperation.Context operationCtx; + private final MaterializedTableExecutorFactory materializedTableExecutorFactory; + private final @Nullable MaterializedTableJobSubmitter materializedTableJobSubmitter; + private @Nullable MaterializedTableExecutor materializedTableExecutor; + private static final String UNSUPPORTED_QUERY_IN_EXECUTE_SQL_MSG = "Unsupported SQL query! executeSql() only accepts a single SQL statement of type " + "CREATE TABLE, DROP TABLE, ALTER TABLE, CREATE DATABASE, DROP DATABASE, ALTER DATABASE, " @@ -191,7 +202,9 @@ protected TableEnvironmentImpl( Executor executor, FunctionCatalog functionCatalog, Planner planner, - boolean isStreamingMode) { + boolean isStreamingMode, + MaterializedTableExecutorFactory materializedTableExecutorFactory, + @Nullable MaterializedTableJobSubmitter materializedTableJobSubmitter) { this.catalogManager = catalogManager; this.moduleManager = moduleManager; this.resourceManager = resourceManager; @@ -239,6 +252,8 @@ protected TableEnvironmentImpl( resourceManager, tableConfig, isStreamingMode); + this.materializedTableExecutorFactory = materializedTableExecutorFactory; + this.materializedTableJobSubmitter = materializedTableJobSubmitter; } public static TableEnvironmentImpl create(Configuration configuration) { @@ -318,6 +333,12 @@ public static TableEnvironmentImpl create(EnvironmentSettings settings) { catalogManager, functionCatalog); + final MaterializedTableExecutorFactory materializedTableExecutorFactory = + FactoryUtil.discoverFactory( + userClassLoader, + MaterializedTableExecutorFactory.class, + MaterializedTableExecutorFactory.DEFAULT_IDENTIFIER); + return new TableEnvironmentImpl( catalogManager, moduleManager, @@ -326,7 +347,9 @@ public static TableEnvironmentImpl create(EnvironmentSettings settings) { executor, functionCatalog, planner, - settings.isStreamingMode()); + settings.isStreamingMode(), + materializedTableExecutorFactory, + null); } @Override @@ -1312,8 +1335,25 @@ private TableResultInternal executeQueryOperation( } } + private MaterializedTableExecutor getMaterializedTableExecutor() { + if (materializedTableExecutor == null) { + final MaterializedTableJobSubmitter jobSubmitter = + materializedTableJobSubmitter != null + ? materializedTableJobSubmitter + : new DefaultMaterializedTableJobSubmitter( + operationCtx, planner, execEnv); + materializedTableExecutor = + materializedTableExecutorFactory.createExecutor(operationCtx, jobSubmitter); + } + return materializedTableExecutor; + } + @Override public TableResultInternal executeInternal(Operation operation) { + if (operation instanceof MaterializedTableOperation) { + return getMaterializedTableExecutor().execute((MaterializedTableOperation) operation); + } + // delegate execution to Operation if it implements ExecutableOperation if (operation instanceof ExecutableOperation) { return ((ExecutableOperation) operation).execute(operationCtx); diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/materializedtable/DefaultMaterializedTableJobSubmitter.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/materializedtable/DefaultMaterializedTableJobSubmitter.java new file mode 100644 index 00000000000000..eed6b4c4190eb8 --- /dev/null +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/materializedtable/DefaultMaterializedTableJobSubmitter.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.api.internal.materializedtable; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.dag.Pipeline; +import org.apache.flink.api.dag.Transformation; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.api.TableException; +import org.apache.flink.table.delegation.Executor; +import org.apache.flink.table.delegation.ExecutorFactory; +import org.apache.flink.table.delegation.Planner; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableJobSubmitter; +import org.apache.flink.table.delegation.materializedtable.RefreshJobResult; +import org.apache.flink.table.delegation.materializedtable.RefreshJobTarget; +import org.apache.flink.table.delegation.materializedtable.RefreshWorkflowContext; +import org.apache.flink.table.factories.FactoryUtil; +import org.apache.flink.table.factories.PlannerFactoryUtil; +import org.apache.flink.table.operations.ExecutableOperation; +import org.apache.flink.table.operations.ModifyOperation; +import org.apache.flink.table.operations.Operation; +import org.apache.flink.table.workflow.WorkflowScheduler; + +import java.util.List; +import java.util.Optional; + +import static org.apache.flink.api.common.RuntimeExecutionMode.STREAMING; +import static org.apache.flink.configuration.ExecutionOptions.RUNTIME_MODE; + +/** + * The default implementation of a {@link MaterializedTableJobSubmitter}. + * + *

It does not support application targets (see {@link #supportsApplicationTargets()}) and does + * not expose a {@link WorkflowScheduler}, so full-mode materialized tables cannot be created, + * suspended, resumed or dropped here, while a one-time REFRESH still works. + */ +@Internal +public class DefaultMaterializedTableJobSubmitter implements MaterializedTableJobSubmitter { + + private static final String DEFAULT_REFRESH_JOB_NAME = "materialized-table-refresh"; + + private final ExecutableOperation.Context operationCtx; + private final Planner planner; + private final Executor sessionExecutor; + + public DefaultMaterializedTableJobSubmitter( + ExecutableOperation.Context operationCtx, Planner planner, Executor sessionExecutor) { + this.operationCtx = operationCtx; + this.planner = planner; + this.sessionExecutor = sessionExecutor; + } + + @Override + public RefreshJobResult submitRefreshJob( + Configuration executionConfig, String insertStatement) { + final RefreshJobTarget target = RefreshJobTarget.forNewJob(operationCtx.getTableConfig()); + final boolean isStreamingRefresh = STREAMING == executionConfig.get(RUNTIME_MODE); + final JobClient jobClient = + isStreamingRefresh == operationCtx.isStreamingMode() + ? submitWithSessionPlanner(target, executionConfig, insertStatement) + : submitWithThrowawayPlanner(target, executionConfig, insertStatement); + + return new RefreshJobResult(target, jobClient.getJobID().toString(), jobClient); + } + + @Override + public boolean supportsApplicationTargets() { + return false; + } + + @Override + public Optional getRefreshWorkflowContext() { + return Optional.empty(); + } + + private static Configuration refreshConfiguration( + Configuration tableConfiguration, + Configuration executionConfig, + RefreshJobTarget target) { + final Configuration refreshConfig = new Configuration(tableConfiguration); + refreshConfig.addAll(executionConfig); + target.applyTo(refreshConfig); + return refreshConfig; + } + + private JobClient submitWithSessionPlanner( + RefreshJobTarget target, Configuration executionConfig, String insertStatement) { + final ModifyOperation modifyOperation = + parseSingleInsertOperation(planner, insertStatement); + operationCtx.getResourceManager().addJarConfiguration(operationCtx.getTableConfig()); + + final Configuration pipelineConfig = + refreshConfiguration( + operationCtx.getTableConfig().getConfiguration(), executionConfig, target); + return submitPipeline(planner, modifyOperation, createRefreshExecutor(), pipelineConfig); + } + + private JobClient submitWithThrowawayPlanner( + RefreshJobTarget target, Configuration executionConfig, String insertStatement) { + final Configuration refreshConfig = + refreshConfiguration( + operationCtx.getTableConfig().getConfiguration(), executionConfig, target); + final TableConfig refreshTableConfig = TableConfig.getDefault(); + refreshTableConfig.setRootConfiguration( + operationCtx.getTableConfig().getRootConfiguration()); + refreshTableConfig.addConfiguration(refreshConfig); + + final Executor refreshExecutor = createRefreshExecutor(); + final Planner refreshPlanner = + PlannerFactoryUtil.createPlanner( + refreshExecutor, + refreshTableConfig, + operationCtx.getResourceManager().getUserClassLoader(), + operationCtx.getModuleManager(), + operationCtx.getCatalogManager(), + operationCtx.getFunctionCatalog()); + + final ModifyOperation modifyOperation = + parseSingleInsertOperation(refreshPlanner, insertStatement); + operationCtx.getResourceManager().addJarConfiguration(refreshTableConfig); + + return submitPipeline( + refreshPlanner, + modifyOperation, + refreshExecutor, + refreshTableConfig.getConfiguration()); + } + + private Executor createRefreshExecutor() { + final ExecutorFactory executorFactory = + FactoryUtil.discoverFactory( + operationCtx.getResourceManager().getUserClassLoader(), + ExecutorFactory.class, + ExecutorFactory.DEFAULT_IDENTIFIER); + return executorFactory.create( + Configuration.fromMap(sessionExecutor.getConfiguration().toMap())); + } + + private static JobClient submitPipeline( + Planner planner, + ModifyOperation modifyOperation, + Executor refreshExecutor, + Configuration pipelineConfig) { + final List> transformations = planner.translate(List.of(modifyOperation)); + final Pipeline pipeline = + refreshExecutor.createPipeline( + transformations, pipelineConfig, DEFAULT_REFRESH_JOB_NAME); + try { + return refreshExecutor.executeAsync(pipeline); + } catch (Exception e) { + throw new TableException("Failed to submit the materialized table refresh job.", e); + } + } + + private static ModifyOperation parseSingleInsertOperation( + Planner planner, String insertStatement) { + final List operations = planner.getParser().parse(insertStatement); + if (operations.size() != 1 || !(operations.get(0) instanceof ModifyOperation)) { + throw new TableException( + String.format( + "The materialized table refresh statement must translate to a single" + + " INSERT operation, but was: %s", + insertStatement)); + } + return (ModifyOperation) operations.get(0); + } +} diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/MaterializedTableExecutor.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/MaterializedTableExecutor.java new file mode 100644 index 00000000000000..27ea7ec20d6fe7 --- /dev/null +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/MaterializedTableExecutor.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.delegation.materializedtable; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.internal.TableResultInternal; +import org.apache.flink.table.operations.materializedtable.MaterializedTableOperation; + +/** Executes {@link MaterializedTableOperation}s through a single execution path. */ +@Internal +public interface MaterializedTableExecutor { + + /** + * Executes the given materialized table operation. + * + * @param op the materialized table operation to execute. + * @return the execution result. + */ + TableResultInternal execute(MaterializedTableOperation op); +} diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/MaterializedTableExecutorFactory.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/MaterializedTableExecutorFactory.java new file mode 100644 index 00000000000000..1af9a2711175e1 --- /dev/null +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/MaterializedTableExecutorFactory.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.delegation.materializedtable; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.factories.Factory; +import org.apache.flink.table.operations.ExecutableOperation; + +/** + * A {@link Factory} for discovering and creating {@link MaterializedTableExecutor} instances via + * Java's Service Provider Interface (SPI). + */ +@Internal +public interface MaterializedTableExecutorFactory extends Factory { + + String DEFAULT_IDENTIFIER = "default"; + + /** Creates a corresponding {@link MaterializedTableExecutor}. */ + MaterializedTableExecutor createExecutor( + ExecutableOperation.Context operationCtx, MaterializedTableJobSubmitter jobSubmitter); +} diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/MaterializedTableJobSubmitter.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/MaterializedTableJobSubmitter.java new file mode 100644 index 00000000000000..b7fcb5e977ea5a --- /dev/null +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/MaterializedTableJobSubmitter.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.delegation.materializedtable; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.Configuration; + +import java.util.Optional; + +/** + * Provides the caller-specific refresh-job submission and scheduling capabilities a {@link + * MaterializedTableExecutor} needs to run materialized table operations. + */ +@Internal +public interface MaterializedTableJobSubmitter { + + /** + * Resolves the target of a refresh job and submits it there with the given execution + * configuration and INSERT statement. + * + * @param executionConfig the configuration used to submit the refresh job. + * @param insertStatement the INSERT statement executed to refresh the materialized table. + * @return the result identifying the submitted refresh job and the target to record for it. + */ + RefreshJobResult submitRefreshJob(Configuration executionConfig, String insertStatement); + + /** + * Whether refresh jobs can be deployed to application targets, each in its own cluster. + * + *

Returning {@code true} also switches the executor to the SQL Gateway's behaviour: it skips + * the target check before any catalog change and the check that a restart stays on the recorded + * target and cluster. + */ + boolean supportsApplicationTargets(); + + /** + * Returns the periodic-refresh scheduling capability for full-mode materialized tables, or + * {@link Optional#empty()} when this job submitter cannot schedule refresh workflows. + */ + Optional getRefreshWorkflowContext(); +} diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/RefreshJobResult.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/RefreshJobResult.java new file mode 100644 index 00000000000000..44c62a963ae8ff --- /dev/null +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/RefreshJobResult.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.delegation.materializedtable; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.core.execution.JobClient; + +import javax.annotation.Nullable; + +import java.util.Optional; + +import static org.apache.flink.util.Preconditions.checkArgument; +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * The result of submitting a materialized table refresh job, describing where the job was deployed + * and how to identify it. It optionally carries the client returned at submission. + */ +@Internal +public final class RefreshJobResult { + + private final RefreshJobTarget target; + private final String jobId; + @Nullable private final JobClient jobClient; + + public RefreshJobResult(RefreshJobTarget target, String jobId, @Nullable JobClient jobClient) { + this.target = checkNotNull(target); + this.jobId = checkNotNull(jobId); + this.jobClient = jobClient; + checkArgument( + jobClient == null || jobClient.getJobID().toHexString().equals(jobId), + "The job client controls job %s, not refresh job %s.", + jobClient == null ? null : jobClient.getJobID(), + jobId); + } + + public RefreshJobTarget getTarget() { + return target; + } + + public String getJobId() { + return jobId; + } + + public Optional getJobClient() { + return Optional.ofNullable(jobClient); + } +} diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/RefreshJobTarget.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/RefreshJobTarget.java new file mode 100644 index 00000000000000..81d6dfe6cae9d5 --- /dev/null +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/RefreshJobTarget.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.delegation.materializedtable; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.DeploymentOptions; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.ObjectIdentifier; +import org.apache.flink.table.refresh.ContinuousRefreshHandler; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static org.apache.flink.util.Preconditions.checkArgument; +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** The execution target and cluster id a materialized table refresh job is submitted to. */ +@Internal +public final class RefreshJobTarget { + + private static final String YARN_APPLICATION_ID_KEY = "yarn.application.id"; + private static final String KUBERNETES_CLUSTER_ID_KEY = "kubernetes.cluster-id"; + + private static final String LOCAL_TARGET = "local"; + private static final String EMBEDDED_TARGET = "embedded"; + private static final String REMOTE_TARGET = "remote"; + private static final String STANDALONE_CLUSTER_ID = "StandaloneClusterId"; + + private final String executionTarget; + private final String clusterId; + + public RefreshJobTarget(String executionTarget, String clusterId) { + this.executionTarget = checkNotNull(executionTarget); + this.clusterId = checkNotNull(clusterId); + checkArgument(!executionTarget.isEmpty(), "The execution target must not be empty."); + } + + /** The target a new refresh job of a program is submitted to. */ + public static RefreshJobTarget forNewJob(ReadableConfig config) { + final String executionTarget = config.getOptional(DeploymentOptions.TARGET).orElse(null); + if (executionTarget == null + || executionTarget.isEmpty() + || LOCAL_TARGET.equals(executionTarget) + || EMBEDDED_TARGET.equals(executionTarget)) { + throw new ValidationException( + String.format( + "Unsupported execution target detected: %s. Materialized table refresh " + + "jobs need a session cluster: set 'execution.target' to " + + "'remote', 'yarn-session' or 'kubernetes-session' (with " + + "'yarn.application.id' or 'kubernetes.cluster-id').", + executionTarget)); + } + final RefreshJobTarget unresolvedTarget = new RefreshJobTarget(executionTarget, ""); + if (unresolvedTarget.isApplicationTarget()) { + throw new ValidationException( + String.format( + "Application-mode materialized table refresh is not supported in this " + + "environment: execution target '%s' requires the SQL Gateway. Use a " + + "session cluster target such as 'remote', 'yarn-session' or " + + "'kubernetes-session'.", + executionTarget)); + } + return new RefreshJobTarget(executionTarget, unresolvedTarget.resolveClusterId(config)); + } + + /** The target for restarting a recorded refresh job. */ + public static RefreshJobTarget forRestart( + ReadableConfig config, + ObjectIdentifier tableIdentifier, + ContinuousRefreshHandler refreshHandler) { + final RefreshJobTarget recordedTarget = recordedIn(refreshHandler); + if (recordedTarget.isApplicationTarget()) { + throw new ValidationException( + String.format( + "The continuous refresh job of materialized table %s runs in its own " + + "application cluster (%s). Restarting it deploys a new " + + "application cluster, which only the SQL Gateway can do, so " + + "resume, alter or redeploy the table through the SQL Gateway. " + + "Suspending, refreshing and dropping it work here.", + tableIdentifier, recordedTarget.describe())); + } + + final RefreshJobTarget programTarget = forNewJob(config); + final boolean sameTarget = + recordedTarget.executionTarget.equals(programTarget.executionTarget); + final boolean sameCluster = + !recordedTarget.identifiesClusterById() + || recordedTarget.clusterId.equals(programTarget.clusterId); + if (!sameTarget || !sameCluster) { + throw new ValidationException( + String.format( + "The continuous refresh job of materialized table %s runs on %s, but " + + "this program submits refresh jobs to %s. RESUME, ALTER and a " + + "redeploy never move a refresh job to another cluster. Set " + + "'execution.target' and its cluster id to the recorded values " + + "to restart it here.", + tableIdentifier, recordedTarget.describe(), programTarget.describe())); + } + return programTarget; + } + + /** The target a refresh handler records. */ + public static RefreshJobTarget recordedIn(ContinuousRefreshHandler refreshHandler) { + final String clusterId = refreshHandler.getClusterId(); + return new RefreshJobTarget( + refreshHandler.getExecutionTarget(), clusterId == null ? "" : clusterId); + } + + public String getExecutionTarget() { + return executionTarget; + } + + /** The cluster id, or an empty string when it is not known. */ + public String getClusterId() { + return clusterId; + } + + /** Whether the target deploys each job in its own application cluster. */ + public boolean isApplicationTarget() { + return executionTarget.endsWith("application"); + } + + public void applyTo(Configuration configuration) { + configuration.set(DeploymentOptions.TARGET, executionTarget); + if (identifiesClusterById()) { + configuration.setString(clusterIdKey().get(), clusterId); + } + } + + public Map toClusterInfo() { + final Map clusterInfo = new HashMap<>(); + clusterInfo.put(DeploymentOptions.TARGET.key(), executionTarget); + clusterIdKey().ifPresent(key -> clusterInfo.put(key, clusterId)); + return Collections.unmodifiableMap(clusterInfo); + } + + private Optional clusterIdKey() { + if (executionTarget.startsWith("yarn")) { + return Optional.of(YARN_APPLICATION_ID_KEY); + } else if (executionTarget.startsWith("kubernetes")) { + return Optional.of(KUBERNETES_CLUSTER_ID_KEY); + } + return Optional.empty(); + } + + private boolean identifiesClusterById() { + return !clusterId.isEmpty() && clusterIdKey().isPresent(); + } + + private String describe() { + if (identifiesClusterById()) { + return String.format( + "execution.target '%s', %s '%s'", + executionTarget, clusterIdKey().get(), clusterId); + } + return String.format("execution.target '%s'", executionTarget); + } + + private String resolveClusterId(ReadableConfig config) { + final Optional clusterIdKey = clusterIdKey(); + if (clusterIdKey.isEmpty()) { + // record StandaloneClusterId#toString for 'remote' + return REMOTE_TARGET.equals(executionTarget) ? STANDALONE_CLUSTER_ID : ""; + } + return config.getOptional( + ConfigOptions.key(clusterIdKey.get()).stringType().noDefaultValue()) + .filter(configuredClusterId -> !configuredClusterId.isBlank()) + .orElseThrow( + () -> + new ValidationException( + String.format( + "Execution target '%s' requires '%s' to be configured to submit or control a materialized table refresh job.", + executionTarget, clusterIdKey.get()))); + } +} diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/RefreshWorkflowContext.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/RefreshWorkflowContext.java new file mode 100644 index 00000000000000..2a815c68cb5fec --- /dev/null +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/delegation/materializedtable/RefreshWorkflowContext.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.delegation.materializedtable; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.refresh.RefreshHandler; +import org.apache.flink.table.workflow.WorkflowScheduler; + +import java.util.Map; + +/** + * The capability to schedule periodic refresh workflows for full-mode materialized tables, exposed + * by a {@link MaterializedTableJobSubmitter} only when the deployment can provide it. + */ +@Internal +public interface RefreshWorkflowContext { + + /** The scheduler that creates, modifies and deletes periodic refresh workflows. */ + WorkflowScheduler scheduler(); + + /** The REST endpoint URL a scheduled workflow uses to reach the cluster managing refreshes. */ + String restEndpointUrl(); + + /** The session configuration a scheduled refresh job is initialized with. */ + Map sessionInitializationConf(); +} diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableAsQueryOperation.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableAsQueryOperation.java index 0dcbfd72926658..21354a5ab0c89c 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableAsQueryOperation.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableAsQueryOperation.java @@ -32,7 +32,7 @@ * Operation to describe an ALTER MATERIALIZED TABLE AS query operation. The operation is not * executable and only serves as a wrapper. * - *

As an example see {@link MaterializedTableManager} where it is transformed to {@link + *

As an example see {@code DefaultMaterializedTableExecutor} where it is transformed to {@link * AlterMaterializedTableChangeOperation} before execution. */ @Internal diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableRefreshOperation.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableRefreshOperation.java index 5212cf58c7ff48..c98f112f076e1b 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableRefreshOperation.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableRefreshOperation.java @@ -23,6 +23,8 @@ import org.apache.flink.table.catalog.ObjectIdentifier; import org.apache.flink.table.operations.OperationUtils; +import javax.annotation.Nullable; + import java.util.Map; /** @@ -33,17 +35,48 @@ public class AlterMaterializedTableRefreshOperation extends AlterMaterializedTableOperation { private final Map partitionSpec; + private final boolean isPeriodic; + private final @Nullable String scheduleTime; + private final Map dynamicOptions; public AlterMaterializedTableRefreshOperation( ObjectIdentifier tableIdentifier, Map partitionSpec) { + this(tableIdentifier, partitionSpec, false, null, Map.of()); + } + + private AlterMaterializedTableRefreshOperation( + ObjectIdentifier tableIdentifier, + Map partitionSpec, + boolean isPeriodic, + @Nullable String scheduleTime, + Map dynamicOptions) { super(tableIdentifier); this.partitionSpec = partitionSpec; + this.isPeriodic = isPeriodic; + this.scheduleTime = scheduleTime; + this.dynamicOptions = dynamicOptions; + } + + public static AlterMaterializedTableRefreshOperation oneTime( + ObjectIdentifier tableIdentifier, + Map partitionSpec, + Map dynamicOptions) { + return new AlterMaterializedTableRefreshOperation( + tableIdentifier, partitionSpec, false, null, dynamicOptions); + } + + public static AlterMaterializedTableRefreshOperation periodic( + ObjectIdentifier tableIdentifier, + @Nullable String scheduleTime, + Map dynamicOptions) { + return new AlterMaterializedTableRefreshOperation( + tableIdentifier, Map.of(), true, scheduleTime, dynamicOptions); } @Override public TableResultInternal execute(Context ctx) { - // execute AlterMaterializedTableRefreshOperation in SqlGateway OperationExecutor. - // see more at MaterializedTableManager#callAlterMaterializedTableRefreshOperation + // TableEnvironmentImpl#executeInternal routes every MaterializedTableOperation + // to the MaterializedTableExecutor before ExecutableOperation dispatch. throw new UnsupportedOperationException( "AlterMaterializedTableRefreshOperation does not support ExecutableOperation yet."); } @@ -52,6 +85,18 @@ public Map getPartitionSpec() { return partitionSpec; } + public boolean isPeriodic() { + return isPeriodic; + } + + public @Nullable String getScheduleTime() { + return scheduleTime; + } + + public Map getDynamicOptions() { + return dynamicOptions; + } + @Override public String asSummaryString() { StringBuilder sb = diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/planner/loader/PlannerModule.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/planner/loader/PlannerModule.java index 1a8375b5039efe..1e457a571533cc 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/planner/loader/PlannerModule.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/planner/loader/PlannerModule.java @@ -27,6 +27,7 @@ import org.apache.flink.table.api.TableException; import org.apache.flink.table.delegation.ExecutorFactory; import org.apache.flink.table.delegation.PlannerFactory; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutorFactory; import org.apache.flink.table.factories.FactoryUtil; import org.apache.flink.util.FileUtils; import org.apache.flink.util.IOUtils; @@ -169,6 +170,13 @@ public PlannerFactory loadPlannerFactory() { this.submoduleClassLoader, PlannerFactory.class, PlannerFactory.DEFAULT_IDENTIFIER); } + public MaterializedTableExecutorFactory loadMaterializedTableExecutorFactory() { + return FactoryUtil.discoverFactory( + this.submoduleClassLoader, + MaterializedTableExecutorFactory.class, + MaterializedTableExecutorFactory.DEFAULT_IDENTIFIER); + } + /** * A class loader extending {@link ComponentClassLoader} which overwrites method{@link #addURL} * to enable it can add url to component classloader. diff --git a/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/utils/MaterializedTableExecutorFactoryMock.java b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/utils/MaterializedTableExecutorFactoryMock.java new file mode 100644 index 00000000000000..36fe04ac2b500a --- /dev/null +++ b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/utils/MaterializedTableExecutorFactoryMock.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.utils; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutor; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutorFactory; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableJobSubmitter; +import org.apache.flink.table.operations.ExecutableOperation; + +import java.util.Set; + +/** Mocking {@link MaterializedTableExecutorFactory} for tests without the planner. */ +public class MaterializedTableExecutorFactoryMock implements MaterializedTableExecutorFactory { + + @Override + public MaterializedTableExecutor createExecutor( + ExecutableOperation.Context operationCtx, MaterializedTableJobSubmitter jobSubmitter) { + return operation -> { + throw new UnsupportedOperationException( + "Materialized table execution is not supported by this test mock."); + }; + } + + @Override + public String factoryIdentifier() { + return DEFAULT_IDENTIFIER; + } + + @Override + public Set> requiredOptions() { + return Set.of(); + } + + @Override + public Set> optionalOptions() { + return Set.of(); + } +} diff --git a/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/utils/TableEnvironmentMock.java b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/utils/TableEnvironmentMock.java index ff046ed48581f8..08930c5f888e4c 100644 --- a/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/utils/TableEnvironmentMock.java +++ b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/utils/TableEnvironmentMock.java @@ -57,7 +57,9 @@ protected TableEnvironmentMock( executor, functionCatalog, planner, - isStreamingMode); + isStreamingMode, + new MaterializedTableExecutorFactoryMock(), + null); this.catalogManager = catalogManager; this.executor = executor; this.functionCatalog = functionCatalog; diff --git a/flink-table/flink-table-api-scala-bridge/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-table/flink-table-api-scala-bridge/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory new file mode 100644 index 00000000000000..bf08ae4ef0730e --- /dev/null +++ b/flink-table/flink-table-api-scala-bridge/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The Scala StreamTableEnvironmentImpl looks this factory up when it is created, and this module's +# tests run without the planner. +org.apache.flink.table.utils.MaterializedTableExecutorFactoryMock diff --git a/flink-table/flink-table-planner-loader/src/main/java/org/apache/flink/table/planner/loader/DelegateMaterializedTableExecutorFactory.java b/flink-table/flink-table-planner-loader/src/main/java/org/apache/flink/table/planner/loader/DelegateMaterializedTableExecutorFactory.java new file mode 100644 index 00000000000000..10583e967bbc38 --- /dev/null +++ b/flink-table/flink-table-planner-loader/src/main/java/org/apache/flink/table/planner/loader/DelegateMaterializedTableExecutorFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.loader; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutor; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutorFactory; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableJobSubmitter; +import org.apache.flink.table.operations.ExecutableOperation; + +/** Delegate of {@link MaterializedTableExecutorFactory}. */ +@Internal +public class DelegateMaterializedTableExecutorFactory + extends BaseDelegateFactory + implements MaterializedTableExecutorFactory { + + public DelegateMaterializedTableExecutorFactory() { + super(PlannerModule.getInstance().loadMaterializedTableExecutorFactory()); + } + + @Override + public MaterializedTableExecutor createExecutor( + ExecutableOperation.Context operationCtx, MaterializedTableJobSubmitter jobSubmitter) { + return delegate.createExecutor(operationCtx, jobSubmitter); + } +} diff --git a/flink-table/flink-table-planner-loader/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-table/flink-table-planner-loader/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory index a932494d960ed8..415099ac45d2d7 100644 --- a/flink-table/flink-table-planner-loader/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory +++ b/flink-table/flink-table-planner-loader/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -15,3 +15,4 @@ org.apache.flink.table.planner.loader.DelegateExecutorFactory org.apache.flink.table.planner.loader.DelegatePlannerFactory +org.apache.flink.table.planner.loader.DelegateMaterializedTableExecutorFactory diff --git a/flink-table/flink-table-planner-loader/src/test/java/org/apache/flink/table/planner/loader/LoaderITCase.java b/flink-table/flink-table-planner-loader/src/test/java/org/apache/flink/table/planner/loader/LoaderITCase.java index f245fcf4cf2850..325aaa972066fb 100644 --- a/flink-table/flink-table-planner-loader/src/test/java/org/apache/flink/table/planner/loader/LoaderITCase.java +++ b/flink-table/flink-table-planner-loader/src/test/java/org/apache/flink/table/planner/loader/LoaderITCase.java @@ -22,6 +22,7 @@ import org.apache.flink.configuration.ConfigurationUtils; import org.apache.flink.table.delegation.ExecutorFactory; import org.apache.flink.table.delegation.PlannerFactory; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutorFactory; import org.apache.flink.table.factories.FactoryUtil; import org.apache.flink.util.FileUtils; import org.apache.flink.util.TestLogger; @@ -66,6 +67,21 @@ public void testExecutorFactory() { .isEqualTo(ExecutorFactory.DEFAULT_IDENTIFIER); } + @Test + public void testMaterializedTableExecutorFactory() { + MaterializedTableExecutorFactory materializedTableExecutorFactory = + FactoryUtil.discoverFactory( + LoaderITCase.class.getClassLoader(), + MaterializedTableExecutorFactory.class, + MaterializedTableExecutorFactory.DEFAULT_IDENTIFIER); + + assertThat(materializedTableExecutorFactory) + .isNotNull() + .isInstanceOf(DelegateMaterializedTableExecutorFactory.class); + assertThat(materializedTableExecutorFactory.factoryIdentifier()) + .isEqualTo(MaterializedTableExecutorFactory.DEFAULT_IDENTIFIER); + } + @Test public void testPlannerFactory() { assertThat( diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/delegation/materializedtable/DefaultMaterializedTableExecutor.java similarity index 51% rename from flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java rename to flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/delegation/materializedtable/DefaultMaterializedTableExecutor.java index cc01f96e7ece27..bb9eb9fc49acb1 100644 --- a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/delegation/materializedtable/DefaultMaterializedTableExecutor.java @@ -16,21 +16,20 @@ * limitations under the License. */ -package org.apache.flink.table.gateway.service.materializedtable; +package org.apache.flink.table.planner.delegation.materializedtable; import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.VisibleForTesting; -import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.JobStatus; -import org.apache.flink.client.deployment.DefaultClusterClientServiceLoader; -import org.apache.flink.client.deployment.application.ApplicationConfiguration; -import org.apache.flink.client.deployment.application.cli.ApplicationClusterDeployer; import org.apache.flink.configuration.CheckpointingOptions; import org.apache.flink.configuration.Configuration; import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.ResultKind; +import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.ValidationException; -import org.apache.flink.table.api.config.TableConfigOptions; -import org.apache.flink.table.api.internal.TableEnvironmentInternal; +import org.apache.flink.table.api.internal.StaticResultProvider; +import org.apache.flink.table.api.internal.TableResultImpl; +import org.apache.flink.table.api.internal.TableResultInternal; import org.apache.flink.table.catalog.CatalogMaterializedTable; import org.apache.flink.table.catalog.CatalogMaterializedTable.RefreshMode; import org.apache.flink.table.catalog.CatalogMaterializedTable.RefreshStatus; @@ -45,17 +44,12 @@ import org.apache.flink.table.data.GenericRowData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; -import org.apache.flink.table.factories.WorkflowSchedulerFactoryUtil; -import org.apache.flink.table.gateway.api.operation.OperationHandle; -import org.apache.flink.table.gateway.api.results.ResultSet; -import org.apache.flink.table.gateway.api.utils.SqlGatewayException; -import org.apache.flink.table.gateway.rest.SqlGatewayRestEndpointFactory; -import org.apache.flink.table.gateway.rest.util.SqlGatewayRestOptions; -import org.apache.flink.table.gateway.service.operation.OperationExecutor; -import org.apache.flink.table.gateway.service.result.ResultFetcher; -import org.apache.flink.table.gateway.service.utils.SqlExecutionException; -import org.apache.flink.table.operations.command.DescribeJobOperation; -import org.apache.flink.table.operations.command.StopJobOperation; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutor; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableJobSubmitter; +import org.apache.flink.table.delegation.materializedtable.RefreshJobResult; +import org.apache.flink.table.delegation.materializedtable.RefreshJobTarget; +import org.apache.flink.table.delegation.materializedtable.RefreshWorkflowContext; +import org.apache.flink.table.operations.ExecutableOperation; import org.apache.flink.table.operations.materializedtable.AlterMaterializedTableChangeOperation; import org.apache.flink.table.operations.materializedtable.AlterMaterializedTableRefreshOperation; import org.apache.flink.table.operations.materializedtable.AlterMaterializedTableResumeOperation; @@ -68,7 +62,6 @@ import org.apache.flink.table.refresh.ContinuousRefreshHandlerSerializer; import org.apache.flink.table.refresh.RefreshHandler; import org.apache.flink.table.refresh.RefreshHandlerSerializer; -import org.apache.flink.table.runtime.application.SqlDriver; import org.apache.flink.table.types.logical.LogicalTypeFamily; import org.apache.flink.table.workflow.CreatePeriodicRefreshWorkflow; import org.apache.flink.table.workflow.CreateRefreshWorkflow; @@ -77,18 +70,15 @@ import org.apache.flink.table.workflow.ResumeRefreshWorkflow; import org.apache.flink.table.workflow.SuspendRefreshWorkflow; import org.apache.flink.table.workflow.WorkflowScheduler; +import org.apache.flink.types.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; - import java.io.IOException; -import java.net.URLClassLoader; import java.time.Duration; import java.time.ZoneId; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -100,11 +90,8 @@ import static org.apache.flink.api.common.RuntimeExecutionMode.BATCH; import static org.apache.flink.api.common.RuntimeExecutionMode.STREAMING; -import static org.apache.flink.configuration.CheckpointingOptions.SAVEPOINT_DIRECTORY; -import static org.apache.flink.configuration.DeploymentOptions.TARGET; import static org.apache.flink.configuration.ExecutionOptions.RUNTIME_MODE; import static org.apache.flink.configuration.PipelineOptions.NAME; -import static org.apache.flink.configuration.PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID; import static org.apache.flink.configuration.StateRecoveryOptions.SAVEPOINT_PATH; import static org.apache.flink.table.api.config.MaterializedTableConfigOptions.DATE_FORMATTER; import static org.apache.flink.table.api.config.MaterializedTableConfigOptions.PARTITION_FIELDS; @@ -112,115 +99,87 @@ import static org.apache.flink.table.api.internal.TableResultInternal.TABLE_RESULT_OK; import static org.apache.flink.table.catalog.CatalogBaseTable.TableKind.MATERIALIZED_TABLE; import static org.apache.flink.table.catalog.IntervalFreshness.convertFreshnessToCron; -import static org.apache.flink.table.factories.WorkflowSchedulerFactoryUtil.WORKFLOW_SCHEDULER_PREFIX; -import static org.apache.flink.table.gateway.api.endpoint.SqlGatewayEndpointFactoryUtils.getEndpointConfig; -import static org.apache.flink.table.gateway.service.utils.Constants.CLUSTER_INFO; -import static org.apache.flink.table.gateway.service.utils.Constants.JOB_ID; import static org.apache.flink.table.utils.DateTimeUtils.formatTimestampStringWithOffset; -/** Manager is responsible for execute the {@link MaterializedTableOperation}. */ +/** Default executor for {@link MaterializedTableOperation}s. */ @Internal -public class MaterializedTableManager { - - private static final Logger LOG = LoggerFactory.getLogger(MaterializedTableManager.class); - - private final URLClassLoader userCodeClassLoader; - - private final @Nullable WorkflowScheduler workflowScheduler; - - private final String restEndpointUrl; - - public MaterializedTableManager( - Configuration configuration, URLClassLoader userCodeClassLoader) { - this.userCodeClassLoader = userCodeClassLoader; - this.restEndpointUrl = buildRestEndpointUrl(configuration); - this.workflowScheduler = buildWorkflowScheduler(configuration, userCodeClassLoader); - } - - private String buildRestEndpointUrl(Configuration configuration) { - Configuration restEndpointConfig = - Configuration.fromMap( - getEndpointConfig(configuration, SqlGatewayRestEndpointFactory.IDENTIFIER)); - String address = restEndpointConfig.get(SqlGatewayRestOptions.ADDRESS); - int port = restEndpointConfig.get(SqlGatewayRestOptions.PORT); - - return String.format("http://%s:%s", address, port); - } - - private WorkflowScheduler buildWorkflowScheduler( - Configuration configuration, URLClassLoader userCodeClassLoader) { - return WorkflowSchedulerFactoryUtil.createWorkflowScheduler( - configuration, userCodeClassLoader); - } - - public void open() throws Exception { - if (workflowScheduler != null) { - workflowScheduler.open(); - } +public class DefaultMaterializedTableExecutor implements MaterializedTableExecutor { + + private static final Logger LOG = + LoggerFactory.getLogger(DefaultMaterializedTableExecutor.class); + + private static final String LEFT_IN_INITIALIZING_STATUS = + " The table was left in INITIALIZING status. No materialized table statement can drop a" + + " table in this status or move it out of it; remove it from the catalog directly."; + + private final ExecutableOperation.Context operationCtx; + private final MaterializedTableJobSubmitter jobSubmitter; + private final RefreshJobController refreshJobController; + + public DefaultMaterializedTableExecutor( + ExecutableOperation.Context operationCtx, MaterializedTableJobSubmitter jobSubmitter) { + this.operationCtx = operationCtx; + this.jobSubmitter = jobSubmitter; + this.refreshJobController = + new RefreshJobController( + operationCtx.getTableConfig(), + operationCtx.getResourceManager().getUserClassLoader()); } - public void close() throws Exception { - if (workflowScheduler != null) { - workflowScheduler.close(); - } - } + @Override + public TableResultInternal execute(MaterializedTableOperation op) { + try { + if (op instanceof CreateMaterializedTableOperation) { + return callCreateMaterializedTableOperation((CreateMaterializedTableOperation) op); + } else if (op instanceof AlterMaterializedTableRefreshOperation) { + return callAlterMaterializedTableRefreshOperation( + (AlterMaterializedTableRefreshOperation) op); + } else if (op instanceof AlterMaterializedTableSuspendOperation) { + return callAlterMaterializedTableSuspend( + (AlterMaterializedTableSuspendOperation) op); + } else if (op instanceof AlterMaterializedTableResumeOperation) { + return callAlterMaterializedTableResume((AlterMaterializedTableResumeOperation) op); + } else if (op instanceof DropMaterializedTableOperation) { + return callDropMaterializedTableOperation((DropMaterializedTableOperation) op); + } else if (op instanceof AlterMaterializedTableChangeOperation) { + return callAlterMaterializedTableChangeOperation( + (AlterMaterializedTableChangeOperation) op); + } else if (op instanceof ConvertTableToMaterializedTableOperation) { + return callConvertTableToMaterializedTableOperation( + (ConvertTableToMaterializedTableOperation) op); + } - public ResultFetcher callMaterializedTableOperation( - OperationExecutor operationExecutor, - OperationHandle handle, - MaterializedTableOperation op) { - if (op instanceof CreateMaterializedTableOperation) { - return callCreateMaterializedTableOperation( - operationExecutor, handle, (CreateMaterializedTableOperation) op); - } else if (op instanceof AlterMaterializedTableRefreshOperation) { - return callAlterMaterializedTableRefreshOperation( - operationExecutor, handle, (AlterMaterializedTableRefreshOperation) op); - } else if (op instanceof AlterMaterializedTableSuspendOperation) { - return callAlterMaterializedTableSuspend( - operationExecutor, handle, (AlterMaterializedTableSuspendOperation) op); - } else if (op instanceof AlterMaterializedTableResumeOperation) { - return callAlterMaterializedTableResume( - operationExecutor, handle, (AlterMaterializedTableResumeOperation) op); - } else if (op instanceof DropMaterializedTableOperation) { - return callDropMaterializedTableOperation( - operationExecutor, handle, (DropMaterializedTableOperation) op); - } else if (op instanceof AlterMaterializedTableChangeOperation) { - return callAlterMaterializedTableChangeOperation( - operationExecutor, handle, (AlterMaterializedTableChangeOperation) op); - } else if (op instanceof ConvertTableToMaterializedTableOperation) { - return callConvertTableToMaterializedTableOperation( - operationExecutor, handle, (ConvertTableToMaterializedTableOperation) op); + throw new TableException( + String.format( + "Unsupported Operation %s for materialized table.", + op.asSummaryString())); + } finally { + refreshJobController.releaseRetrievedJobClients(); } - - throw new SqlExecutionException( - String.format( - "Unsupported Operation %s for materialized table.", op.asSummaryString())); } - private ResultFetcher callCreateMaterializedTableOperation( - OperationExecutor operationExecutor, - OperationHandle handle, + private TableResultInternal callCreateMaterializedTableOperation( CreateMaterializedTableOperation createMaterializedTableOperation) { ResolvedCatalogMaterializedTable materializedTable = createMaterializedTableOperation.getCatalogMaterializedTable(); if (RefreshMode.CONTINUOUS == materializedTable.getRefreshMode()) { - createMaterializedTableInContinuousMode( - operationExecutor, handle, createMaterializedTableOperation); + checkSubmissionTargetBeforeChanges(); + createMaterializedTableInContinuousMode(createMaterializedTableOperation); } else { createMaterializedTableInFullMode( - operationExecutor, handle, createMaterializedTableOperation); + createMaterializedTableOperation, + requireRefreshWorkflowContext( + "The workflow scheduler must be configured when creating materialized table in full refresh mode.")); } // Just return ok to unify different refresh job info of continuous and full mode, user // should get the refresh job info via desc table. - return ResultFetcher.fromTableResult(handle, TABLE_RESULT_OK, false); + return TABLE_RESULT_OK; } private void createMaterializedTableInContinuousMode( - OperationExecutor operationExecutor, - OperationHandle handle, CreateMaterializedTableOperation createMaterializedTableOperation) { // create materialized table first - operationExecutor.callExecutableOperation(handle, createMaterializedTableOperation); + createMaterializedTableOperation.execute(operationCtx); ObjectIdentifier materializedTableIdentifier = createMaterializedTableOperation.getTableIdentifier(); @@ -228,20 +187,21 @@ private void createMaterializedTableInContinuousMode( createMaterializedTableOperation.getCatalogMaterializedTable(); try { - executeContinuousRefreshJob( - operationExecutor, - handle, - catalogMaterializedTable, - materializedTableIdentifier, - Map.of(), - Optional.empty()); + executeContinuousRefreshJobOrCancelIt( + catalogMaterializedTable, materializedTableIdentifier); } catch (Exception e) { // drop materialized table if submitting the Flink streaming job encounters an - // exception. Thus, weak - // atomicity is guaranteed - operationExecutor.callExecutableOperation( - handle, new DropMaterializedTableOperation(materializedTableIdentifier, true)); - throw new SqlExecutionException( + // exception. Thus, weak atomicity is guaranteed + cleanUpAfterFailure( + () -> + new DropMaterializedTableOperation(materializedTableIdentifier, true) + .execute(operationCtx), + e, + String.format( + "Failed to submit continuous refresh job for materialized table %s, and dropping the table failed as well." + + LEFT_IN_INITIALIZING_STATUS, + materializedTableIdentifier)); + throw new TableException( String.format( "Failed to submit continuous refresh job for materialized table %s.", materializedTableIdentifier), @@ -250,15 +210,10 @@ private void createMaterializedTableInContinuousMode( } private void createMaterializedTableInFullMode( - OperationExecutor operationExecutor, - OperationHandle handle, - CreateMaterializedTableOperation createMaterializedTableOperation) { - if (workflowScheduler == null) { - throw new SqlExecutionException( - "The workflow scheduler must be configured when creating materialized table in full refresh mode."); - } + CreateMaterializedTableOperation createMaterializedTableOperation, + RefreshWorkflowContext refreshWorkflowContext) { // create materialized table first - operationExecutor.callExecutableOperation(handle, createMaterializedTableOperation); + createMaterializedTableOperation.execute(operationCtx); ObjectIdentifier materializedTableIdentifier = createMaterializedTableOperation.getTableIdentifier(); @@ -267,16 +222,20 @@ private void createMaterializedTableInFullMode( try { createPeriodicRefreshWorkflow( - operationExecutor, - handle, - materializedTableIdentifier, - catalogMaterializedTable); + materializedTableIdentifier, catalogMaterializedTable, refreshWorkflowContext); } catch (Exception e) { // drop materialized table if creating the refresh workflow encounters an exception, so // weak atomicity is guaranteed - operationExecutor.callExecutableOperation( - handle, new DropMaterializedTableOperation(materializedTableIdentifier, true)); - throw new SqlExecutionException( + cleanUpAfterFailure( + () -> + new DropMaterializedTableOperation(materializedTableIdentifier, true) + .execute(operationCtx), + e, + String.format( + "Failed to create refresh workflow for materialized table %s, and dropping the table failed as well." + + LEFT_IN_INITIALIZING_STATUS, + materializedTableIdentifier)); + throw new TableException( String.format( "Failed to create refresh workflow for materialized table %s.", materializedTableIdentifier), @@ -284,29 +243,28 @@ private void createMaterializedTableInFullMode( } } - private ResultFetcher callConvertTableToMaterializedTableOperation( - OperationExecutor operationExecutor, - OperationHandle handle, + private TableResultInternal callConvertTableToMaterializedTableOperation( ConvertTableToMaterializedTableOperation convertOperation) { ResolvedCatalogMaterializedTable materializedTable = convertOperation.getMaterializedTable(); if (RefreshMode.CONTINUOUS == materializedTable.getRefreshMode()) { - convertTableToMaterializedTableInContinuousMode( - operationExecutor, handle, convertOperation); + convertTableToMaterializedTableInContinuousMode(convertOperation); } else { - convertTableToMaterializedTableInFullMode(operationExecutor, handle, convertOperation); + convertTableToMaterializedTableInFullMode( + convertOperation, + requireRefreshWorkflowContext( + "The workflow scheduler must be configured when converting a table to a materialized table in full refresh mode.")); } // Just return ok for unify different refresh job info of continuous and full mode, user // should get the refresh job info via desc table. - return ResultFetcher.fromTableResult(handle, TABLE_RESULT_OK, false); + return TABLE_RESULT_OK; } private void convertTableToMaterializedTableInContinuousMode( - OperationExecutor operationExecutor, - OperationHandle handle, ConvertTableToMaterializedTableOperation convertOperation) { + checkSubmissionTargetBeforeChanges(); // swap the catalog entry from a regular table to a materialized table first - operationExecutor.callExecutableOperation(handle, convertOperation); + convertOperation.execute(operationCtx); ObjectIdentifier materializedTableIdentifier = convertOperation.getTableIdentifier(); ResolvedCatalogMaterializedTable catalogMaterializedTable = @@ -314,19 +272,13 @@ private void convertTableToMaterializedTableInContinuousMode( try { executeContinuousRefreshJob( - operationExecutor, - handle, catalogMaterializedTable, materializedTableIdentifier, Map.of(), Optional.empty()); } catch (Exception e) { - suspendMaterializedTable( - operationExecutor, - handle, - materializedTableIdentifier, - catalogMaterializedTable); - throw new SqlExecutionException( + suspendMaterializedTable(materializedTableIdentifier, catalogMaterializedTable); + throw new TableException( String.format( "Failed to start the continuous refresh job when converting table %s to a materialized table. " + "The table was converted and left in SUSPENDED status; resume it once the issue is resolved.", @@ -336,15 +288,10 @@ private void convertTableToMaterializedTableInContinuousMode( } private void convertTableToMaterializedTableInFullMode( - OperationExecutor operationExecutor, - OperationHandle handle, - ConvertTableToMaterializedTableOperation convertOperation) { - if (workflowScheduler == null) { - throw new SqlExecutionException( - "The workflow scheduler must be configured when converting a table to a materialized table in full refresh mode."); - } + ConvertTableToMaterializedTableOperation convertOperation, + RefreshWorkflowContext refreshWorkflowContext) { // swap the catalog entry from a regular table to a materialized table first - operationExecutor.callExecutableOperation(handle, convertOperation); + convertOperation.execute(operationCtx); ObjectIdentifier materializedTableIdentifier = convertOperation.getTableIdentifier(); ResolvedCatalogMaterializedTable catalogMaterializedTable = @@ -352,17 +299,10 @@ private void convertTableToMaterializedTableInFullMode( try { createPeriodicRefreshWorkflow( - operationExecutor, - handle, - materializedTableIdentifier, - catalogMaterializedTable); + materializedTableIdentifier, catalogMaterializedTable, refreshWorkflowContext); } catch (Exception e) { - suspendMaterializedTable( - operationExecutor, - handle, - materializedTableIdentifier, - catalogMaterializedTable); - throw new SqlExecutionException( + suspendMaterializedTable(materializedTableIdentifier, catalogMaterializedTable); + throw new TableException( String.format( "Failed to create the refresh workflow when converting table %s to a materialized table. " + "The table was converted and left in SUSPENDED status; resume it once the issue is resolved.", @@ -376,11 +316,12 @@ private void convertTableToMaterializedTableInFullMode( * resulting handler with {@code ACTIVATED} status. */ private void createPeriodicRefreshWorkflow( - OperationExecutor operationExecutor, - OperationHandle handle, ObjectIdentifier materializedTableIdentifier, - ResolvedCatalogMaterializedTable catalogMaterializedTable) + ResolvedCatalogMaterializedTable catalogMaterializedTable, + RefreshWorkflowContext refreshWorkflowContext) throws Exception { + final WorkflowScheduler workflowScheduler = + refreshWorkflowContext.scheduler(); final IntervalFreshness freshness = catalogMaterializedTable.getDefinitionFreshness(); final String cronExpression = convertFreshnessToCron(freshness); final CreateRefreshWorkflow createRefreshWorkflow = @@ -388,9 +329,9 @@ private void createPeriodicRefreshWorkflow( materializedTableIdentifier, catalogMaterializedTable.getExpandedQuery(), cronExpression, - getSessionInitializationConf(operationExecutor), + refreshWorkflowContext.sessionInitializationConf(), Map.of(), - restEndpointUrl); + refreshWorkflowContext.restEndpointUrl()); final RefreshHandler refreshHandler = workflowScheduler.createRefreshWorkflow(createRefreshWorkflow); @@ -399,8 +340,6 @@ private void createPeriodicRefreshWorkflow( final byte[] serializedRefreshHandler = refreshHandlerSerializer.serialize(refreshHandler); updateRefreshHandler( - operationExecutor, - handle, materializedTableIdentifier, catalogMaterializedTable, RefreshStatus.ACTIVATED, @@ -410,8 +349,6 @@ private void createPeriodicRefreshWorkflow( /** Sets a materialized table refresh status to {@code SUSPENDED}. */ private void suspendMaterializedTable( - OperationExecutor operationExecutor, - OperationHandle handle, ObjectIdentifier materializedTableIdentifier, ResolvedCatalogMaterializedTable catalogMaterializedTable) { AlterMaterializedTableChangeOperation alterOperation = @@ -420,53 +357,60 @@ private void suspendMaterializedTable( oldTable -> List.of(TableChange.modifyRefreshStatus(RefreshStatus.SUSPENDED)), catalogMaterializedTable); - operationExecutor.callExecutableOperation(handle, alterOperation); + alterOperation.execute(operationCtx); } - private ResultFetcher callAlterMaterializedTableSuspend( - OperationExecutor operationExecutor, - OperationHandle handle, + private TableResultInternal callAlterMaterializedTableSuspend( AlterMaterializedTableSuspendOperation op) { ObjectIdentifier tableIdentifier = op.getTableIdentifier(); ResolvedCatalogMaterializedTable materializedTable = - getCatalogMaterializedTable(operationExecutor, tableIdentifier); + getCatalogMaterializedTable(tableIdentifier); - // Initialization phase doesn't support resume operation. + // Initialization phase doesn't support suspend operation. if (RefreshStatus.INITIALIZING == materializedTable.getRefreshStatus()) { - throw new SqlExecutionException( + throw new TableException( String.format( "Materialized table %s is being initialized and does not support suspend operation.", tableIdentifier)); } if (RefreshMode.CONTINUOUS == materializedTable.getRefreshMode()) { - suspendContinuousRefreshJob( - operationExecutor, handle, tableIdentifier, materializedTable); + suspendContinuousRefreshJob(tableIdentifier, materializedTable); } else { - suspendRefreshWorkflow(operationExecutor, handle, tableIdentifier, materializedTable); + suspendRefreshWorkflow(tableIdentifier, materializedTable); } - return ResultFetcher.fromTableResult(handle, TABLE_RESULT_OK, false); + return TABLE_RESULT_OK; } private ResolvedCatalogMaterializedTable suspendContinuousRefreshJob( - OperationExecutor operationExecutor, - OperationHandle handle, - ObjectIdentifier tableIdentifier, - ResolvedCatalogMaterializedTable materializedTable) { + ObjectIdentifier tableIdentifier, ResolvedCatalogMaterializedTable materializedTable) { + final ContinuousRefreshHandler refreshHandler; + final String savepointPath; try { - ContinuousRefreshHandler refreshHandler = + refreshHandler = deserializeContinuousHandler(materializedTable.getSerializedRefreshHandler()); if (RefreshStatus.SUSPENDED == materializedTable.getRefreshStatus()) { - throw new SqlExecutionException( + throw new TableException( String.format( "Materialized table %s continuous refresh job has been suspended, jobId is %s.", tableIdentifier, refreshHandler.getJobId())); } - String savepointPath = stopJobWithSavepoint(operationExecutor, handle, refreshHandler); + final String savepointDirectory = requireSavepointDirectory(); + savepointPath = + refreshJobController.stopWithSavepoint(refreshHandler, savepointDirectory); + } catch (Exception e) { + throw new TableException( + String.format( + "Failed to suspend the continuous refresh job for materialized table %s.", + tableIdentifier), + e); + } - ContinuousRefreshHandler updateRefreshHandler = + // The job has stopped, so from here on a failure must name the savepoint. + try { + final ContinuousRefreshHandler updateRefreshHandler = new ContinuousRefreshHandler( refreshHandler.getExecutionTarget(), refreshHandler.getClusterId(), @@ -474,59 +418,60 @@ private ResolvedCatalogMaterializedTable suspendContinuousRefreshJob( savepointPath); return updateRefreshHandler( - operationExecutor, - handle, tableIdentifier, materializedTable, RefreshStatus.SUSPENDED, updateRefreshHandler.asSummaryString(), serializeContinuousHandler(updateRefreshHandler)); } catch (Exception e) { - throw new SqlExecutionException( + final String jobId = refreshHandler.getJobId(); + LOG.error( + "Stopped the continuous refresh job {} of materialized table {} with savepoint {}, but failed to record it in the catalog.", + jobId, + tableIdentifier, + savepointPath, + e); + throw new TableException( String.format( - "Failed to suspend the continuous refresh job for materialized table %s.", - tableIdentifier), + "Stopped the continuous refresh job %s of materialized table %s with savepoint %s, but failed to record the savepoint in the catalog, which still reports the table as ACTIVATED. To resume from the savepoint, set 'execution.state-recovery.path' to %s before RESUME, and unset it afterwards (RESET in SQL); otherwise RESUME starts the refresh job without restoring state.", + jobId, tableIdentifier, savepointPath, savepointPath), e); } } private void suspendRefreshWorkflow( - OperationExecutor operationExecutor, - OperationHandle handle, - ObjectIdentifier tableIdentifier, - ResolvedCatalogMaterializedTable materializedTable) { + ObjectIdentifier tableIdentifier, ResolvedCatalogMaterializedTable materializedTable) { if (RefreshStatus.SUSPENDED == materializedTable.getRefreshStatus()) { - throw new SqlExecutionException( + throw new TableException( String.format( "Materialized table %s refresh workflow has been suspended.", tableIdentifier)); } - if (workflowScheduler == null) { - throw new SqlExecutionException( - "The workflow scheduler must be configured when suspending materialized table in full refresh mode."); - } + WorkflowScheduler workflowScheduler = + requireRefreshWorkflowContext( + "The workflow scheduler must be configured when suspending materialized table in full refresh mode.") + .scheduler(); try { RefreshHandlerSerializer refreshHandlerSerializer = workflowScheduler.getRefreshHandlerSerializer(); RefreshHandler refreshHandler = refreshHandlerSerializer.deserialize( - materializedTable.getSerializedRefreshHandler(), userCodeClassLoader); + materializedTable.getSerializedRefreshHandler(), + operationCtx.getResourceManager().getUserClassLoader()); ModifyRefreshWorkflow modifyRefreshWorkflow = new SuspendRefreshWorkflow(refreshHandler); workflowScheduler.modifyRefreshWorkflow(modifyRefreshWorkflow); updateRefreshHandler( - operationExecutor, - handle, tableIdentifier, materializedTable, RefreshStatus.SUSPENDED, refreshHandler.asSummaryString(), materializedTable.getSerializedRefreshHandler()); } catch (Exception e) { - throw new SqlExecutionException( + throw new TableException( String.format( "Failed to suspend the refresh workflow for materialized table %s.", tableIdentifier), @@ -534,17 +479,15 @@ private void suspendRefreshWorkflow( } } - private ResultFetcher callAlterMaterializedTableResume( - OperationExecutor operationExecutor, - OperationHandle handle, + private TableResultInternal callAlterMaterializedTableResume( AlterMaterializedTableResumeOperation op) { ObjectIdentifier tableIdentifier = op.getTableIdentifier(); ResolvedCatalogMaterializedTable catalogMaterializedTable = - getCatalogMaterializedTable(operationExecutor, tableIdentifier); + getCatalogMaterializedTable(tableIdentifier); // Initialization phase doesn't support resume operation. if (RefreshStatus.INITIALIZING == catalogMaterializedTable.getRefreshStatus()) { - throw new SqlExecutionException( + throw new TableException( String.format( "Materialized table %s is being initialized and does not support resume operation.", tableIdentifier)); @@ -552,55 +495,50 @@ private ResultFetcher callAlterMaterializedTableResume( if (RefreshMode.CONTINUOUS == catalogMaterializedTable.getRefreshMode()) { resumeContinuousRefreshJob( - operationExecutor, - handle, - tableIdentifier, - catalogMaterializedTable, - op.getDynamicOptions()); + tableIdentifier, catalogMaterializedTable, op.getDynamicOptions()); } else { resumeRefreshWorkflow( - operationExecutor, - handle, - tableIdentifier, - catalogMaterializedTable, - op.getDynamicOptions()); + tableIdentifier, catalogMaterializedTable, op.getDynamicOptions()); } - return ResultFetcher.fromTableResult(handle, TABLE_RESULT_OK, false); + return TABLE_RESULT_OK; } private void resumeContinuousRefreshJob( - OperationExecutor operationExecutor, - OperationHandle handle, ObjectIdentifier tableIdentifier, ResolvedCatalogMaterializedTable catalogMaterializedTable, Map dynamicOptions) { - ContinuousRefreshHandler refreshHandler = + final ContinuousRefreshHandler refreshHandler = deserializeContinuousHandler( catalogMaterializedTable.getSerializedRefreshHandler()); // Repeated resume continuous refresh job is not supported if (RefreshStatus.ACTIVATED == catalogMaterializedTable.getRefreshStatus()) { - JobStatus jobStatus = getJobStatus(operationExecutor, handle, refreshHandler); + final JobStatus jobStatus = refreshJobController.getJobStatus(refreshHandler); if (!jobStatus.isGloballyTerminalState()) { - throw new SqlExecutionException( + throw new TableException( String.format( "Materialized table %s continuous refresh job has been resumed, jobId is %s.", tableIdentifier, refreshHandler.getJobId())); } + // an ACTIVATED handler never carries a savepoint, so only a configured path keeps state + if (operationCtx.getTableConfig().getOptional(SAVEPOINT_PATH).isEmpty()) { + LOG.warn( + "Materialized table {} is ACTIVATED, but its continuous refresh job {} has already reached {}. Starting a new refresh job without restoring state; to restore a savepoint instead, set 'execution.state-recovery.path' to it before RESUME, and unset it afterwards (RESET in SQL).", + tableIdentifier, + refreshHandler.getJobId(), + jobStatus); + } } - Optional restorePath = refreshHandler.getRestorePath(); + checkRestartTargetBeforeChanges( + tableIdentifier, catalogMaterializedTable.getSerializedRefreshHandler()); + final Optional restorePath = refreshHandler.getRestorePath(); try { executeContinuousRefreshJob( - operationExecutor, - handle, - catalogMaterializedTable, - tableIdentifier, - dynamicOptions, - restorePath); + catalogMaterializedTable, tableIdentifier, dynamicOptions, restorePath); } catch (Exception e) { - throw new SqlExecutionException( + throw new TableException( String.format( "Failed to resume the continuous refresh job for materialized table %s.", tableIdentifier), @@ -609,44 +547,40 @@ private void resumeContinuousRefreshJob( } private void resumeRefreshWorkflow( - OperationExecutor operationExecutor, - OperationHandle handle, ObjectIdentifier tableIdentifier, ResolvedCatalogMaterializedTable catalogMaterializedTable, Map dynamicOptions) { // Repeated resume refresh workflow is not supported if (RefreshStatus.ACTIVATED == catalogMaterializedTable.getRefreshStatus()) { - throw new SqlExecutionException( + throw new TableException( String.format( "Materialized table %s refresh workflow has been resumed.", tableIdentifier)); } - if (workflowScheduler == null) { - throw new SqlExecutionException( - "The workflow scheduler must be configured when resuming materialized table in full refresh mode."); - } + WorkflowScheduler workflowScheduler = + requireRefreshWorkflowContext( + "The workflow scheduler must be configured when resuming materialized table in full refresh mode.") + .scheduler(); try { RefreshHandlerSerializer refreshHandlerSerializer = workflowScheduler.getRefreshHandlerSerializer(); RefreshHandler refreshHandler = refreshHandlerSerializer.deserialize( catalogMaterializedTable.getSerializedRefreshHandler(), - userCodeClassLoader); + operationCtx.getResourceManager().getUserClassLoader()); ModifyRefreshWorkflow modifyRefreshWorkflow = new ResumeRefreshWorkflow(refreshHandler, dynamicOptions); workflowScheduler.modifyRefreshWorkflow(modifyRefreshWorkflow); updateRefreshHandler( - operationExecutor, - handle, tableIdentifier, catalogMaterializedTable, RefreshStatus.ACTIVATED, refreshHandler.asSummaryString(), catalogMaterializedTable.getSerializedRefreshHandler()); } catch (Exception e) { - throw new SqlExecutionException( + throw new TableException( String.format( "Failed to resume the refresh workflow for materialized table %s.", tableIdentifier), @@ -655,8 +589,38 @@ private void resumeRefreshWorkflow( } private void executeContinuousRefreshJob( - OperationExecutor operationExecutor, - OperationHandle handle, + CatalogMaterializedTable catalogMaterializedTable, + ObjectIdentifier materializedTableIdentifier, + Map dynamicOptions, + Optional restorePath) { + recordContinuousRefreshJob( + materializedTableIdentifier, + catalogMaterializedTable, + submitContinuousRefreshJob( + catalogMaterializedTable, + materializedTableIdentifier, + dynamicOptions, + restorePath)); + } + + private void executeContinuousRefreshJobOrCancelIt( + CatalogMaterializedTable catalogMaterializedTable, + ObjectIdentifier materializedTableIdentifier) { + final ContinuousRefreshHandler refreshHandler = + submitContinuousRefreshJob( + catalogMaterializedTable, + materializedTableIdentifier, + Map.of(), + Optional.empty()); + try { + recordContinuousRefreshJob( + materializedTableIdentifier, catalogMaterializedTable, refreshHandler); + } catch (Exception e) { + throw cancelUnrecordedRefreshJob(materializedTableIdentifier, refreshHandler, e); + } + } + + private ContinuousRefreshHandler submitContinuousRefreshJob( CatalogMaterializedTable catalogMaterializedTable, ObjectIdentifier materializedTableIdentifier, Map dynamicOptions, @@ -673,10 +637,10 @@ private void executeContinuousRefreshJob( restorePath.ifPresent(s -> customConfig.set(SAVEPOINT_PATH, s)); // Do not override the user-defined checkpoint interval - if (!operationExecutor - .getSessionContext() - .getSessionConf() - .contains(CheckpointingOptions.CHECKPOINTING_INTERVAL)) { + if (operationCtx + .getTableConfig() + .getOptional(CheckpointingOptions.CHECKPOINTING_INTERVAL) + .isEmpty()) { final Duration freshness = validateAndGetIntervalFreshness(catalogMaterializedTable).toDuration(); @@ -689,67 +653,83 @@ private void executeContinuousRefreshJob( catalogMaterializedTable.getExpandedQuery(), dynamicOptions); - JobExecutionResult result = - executeRefreshJob(insertStatement, customConfig, operationExecutor, handle); - ContinuousRefreshHandler continuousRefreshHandler = - new ContinuousRefreshHandler( - result.executionTarget, result.clusterId, result.jobId); - byte[] serializedBytes = serializeContinuousHandler(continuousRefreshHandler); + final RefreshJobResult result = + jobSubmitter.submitRefreshJob(customConfig, insertStatement); + refreshJobController.register(result); + return new ContinuousRefreshHandler( + result.getTarget().getExecutionTarget(), + result.getTarget().getClusterId(), + result.getJobId()); + } + private void recordContinuousRefreshJob( + ObjectIdentifier materializedTableIdentifier, + CatalogMaterializedTable catalogMaterializedTable, + ContinuousRefreshHandler refreshHandler) { updateRefreshHandler( - operationExecutor, - handle, materializedTableIdentifier, - resolveCatalogMaterializedTable(operationExecutor, catalogMaterializedTable), + resolveCatalogMaterializedTable(catalogMaterializedTable), RefreshStatus.ACTIVATED, - continuousRefreshHandler.asSummaryString(), - serializedBytes); + refreshHandler.asSummaryString(), + serializeContinuousHandler(refreshHandler)); } - private ResultFetcher callAlterMaterializedTableRefreshOperation( - OperationExecutor operationExecutor, - OperationHandle handle, - AlterMaterializedTableRefreshOperation alterMaterializedTableRefreshOperation) { - ObjectIdentifier materializedTableIdentifier = - alterMaterializedTableRefreshOperation.getTableIdentifier(); - - Map partitionSpec = - alterMaterializedTableRefreshOperation.getPartitionSpec(); - - return refreshMaterializedTable( - operationExecutor, - handle, - materializedTableIdentifier, - partitionSpec, - Map.of(), - false, - null); + private TableException cancelUnrecordedRefreshJob( + ObjectIdentifier tableIdentifier, + ContinuousRefreshHandler refreshHandler, + Exception recordFailure) { + final String jobId = refreshHandler.getJobId(); + LOG.error( + "Failed to record the continuous refresh job {} of materialized table {} in the catalog; cancelling it.", + jobId, + tableIdentifier, + recordFailure); + try { + refreshJobController.cancel(refreshHandler); + } catch (Exception cancelFailure) { + LOG.error( + "Failed to cancel the continuous refresh job {} of materialized table {} (execution.target={}, cluster id={}), which the catalog does not record. It may still be running and must be cancelled manually.", + jobId, + tableIdentifier, + refreshHandler.getExecutionTarget(), + refreshHandler.getClusterId(), + cancelFailure); + return withSuppressed( + String.format( + "Failed to record the continuous refresh job %s of materialized table %s in the catalog, and cancelling it failed as well. The job may still be running (execution.target=%s, cluster id=%s) and must be cancelled manually.", + jobId, + tableIdentifier, + refreshHandler.getExecutionTarget(), + refreshHandler.getClusterId()), + cancelFailure, + recordFailure); + } + return new TableException( + String.format( + "Failed to record the continuous refresh job %s of materialized table %s in the catalog. The job was cancelled.", + jobId, tableIdentifier), + recordFailure); } - public ResultFetcher refreshMaterializedTable( - OperationExecutor operationExecutor, - OperationHandle handle, - ObjectIdentifier materializedTableIdentifier, - Map staticPartitions, - Map dynamicOptions, - boolean isPeriodic, - @Nullable String scheduleTime) { + private TableResultInternal callAlterMaterializedTableRefreshOperation( + AlterMaterializedTableRefreshOperation op) { + ObjectIdentifier materializedTableIdentifier = op.getTableIdentifier(); ResolvedCatalogMaterializedTable materializedTable = - getCatalogMaterializedTable(operationExecutor, materializedTableIdentifier); + getCatalogMaterializedTable(materializedTableIdentifier); + + boolean isPeriodic = op.isPeriodic(); Map refreshPartitions = isPeriodic ? getPeriodRefreshPartition( - scheduleTime, + op.getScheduleTime(), materializedTable.getDefinitionFreshness(), materializedTableIdentifier, materializedTable.getOptions(), - operationExecutor - .getTableEnvironment() - .getConfig() - .getLocalTimeZone()) - : staticPartitions; + operationCtx.getTableConfig().getLocalTimeZone()) + : op.getPartitionSpec(); validatePartitionSpec(refreshPartitions, materializedTable); + checkSubmissionTargetBeforeChanges(); // Set job name, runtime mode Configuration customConfig = new Configuration(); @@ -770,40 +750,34 @@ public ResultFetcher refreshMaterializedTable( materializedTableIdentifier, materializedTable.getExpandedQuery(), refreshPartitions, - dynamicOptions); + op.getDynamicOptions()); try { LOG.info( "Starting refresh of the materialized table {}, statement: {}", materializedTableIdentifier, insertStatement); - JobExecutionResult result = - executeRefreshJob(insertStatement, customConfig, operationExecutor, handle); - - Map clusterInfo = new HashMap<>(); - clusterInfo.put( - StringData.fromString(TARGET.key()), - StringData.fromString(result.executionTarget)); - Optional clusterIdKeyName = getClusterIdKeyName(result.executionTarget); - clusterIdKeyName.ifPresent( - s -> - clusterInfo.put( - StringData.fromString(s), - StringData.fromString(result.clusterId))); - - return ResultFetcher.fromResults( - handle, + final RefreshJobResult result = + jobSubmitter.submitRefreshJob(customConfig, insertStatement); + + final Row row = Row.of(result.getJobId(), result.getTarget().toClusterInfo()); + ResolvedSchema schema = ResolvedSchema.of( - Column.physical(JOB_ID, DataTypes.STRING()), + Column.physical("job id", DataTypes.STRING()), Column.physical( - CLUSTER_INFO, - DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()))), - Collections.singletonList( - GenericRowData.of( - StringData.fromString(result.jobId), - new GenericMapData(clusterInfo)))); + "cluster info", + DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()))); + + return TableResultImpl.builder() + .resultKind(ResultKind.SUCCESS_WITH_CONTENT) + .schema(schema) + .resultProvider( + new StaticResultProvider( + List.of(row), + DefaultMaterializedTableExecutor::refreshRowToInternalRow)) + .build(); } catch (Exception e) { - throw new SqlExecutionException( + throw new TableException( String.format( "Failed to refresh the materialized table %s.", materializedTableIdentifier), @@ -845,7 +819,7 @@ static Map getPeriodRefreshPartition( TimeZone.getTimeZone(localZoneId), -freshness.toDuration().toMillis()); if (partFiledValue == null) { - throw new SqlExecutionException( + throw new TableException( String.format( "Failed to parse a valid partition value for the field '%s' in materialized table %s using the scheduler time '%s' based on the date format '%s'.", partField, @@ -902,7 +876,7 @@ private void validatePartitionSpec( } @VisibleForTesting - protected static String getRefreshStatement( + static String getRefreshStatement( ObjectIdentifier tableIdentifier, String definitionQuery, Map partitionSpec, @@ -929,103 +903,127 @@ protected static String getRefreshStatement( return insertStatement.toString(); } - private ResultFetcher callAlterMaterializedTableChangeOperation( - OperationExecutor operationExecutor, - OperationHandle handle, + private TableResultInternal callAlterMaterializedTableChangeOperation( AlterMaterializedTableChangeOperation op) { ObjectIdentifier tableIdentifier = op.getTableIdentifier(); ResolvedCatalogMaterializedTable oldMaterializedTable = - getCatalogMaterializedTable(operationExecutor, tableIdentifier); + getCatalogMaterializedTable(tableIdentifier); if (RefreshMode.FULL == oldMaterializedTable.getRefreshMode()) { // directly apply the alter operation - return operationExecutor.callExecutableOperation( - handle, op.copyAsTableChangeOperation()); + return op.copyAsTableChangeOperation().execute(operationCtx); } if (RefreshStatus.ACTIVATED == oldMaterializedTable.getRefreshStatus()) { - // 1. suspend the materialized table - ResolvedCatalogMaterializedTable suspendMaterializedTable = - suspendContinuousRefreshJob( - operationExecutor, handle, tableIdentifier, oldMaterializedTable); - - // 2. alter materialized table schema & query definition - AlterMaterializedTableChangeOperation alterMaterializedTableChangeOperation = - new AlterMaterializedTableChangeOperation( - op.getTableIdentifier(), - oldTable -> op.getTableChanges(), - suspendMaterializedTable, - op.getAsQueryOperation()); - operationExecutor.callExecutableOperation( - handle, alterMaterializedTableChangeOperation); - - // 3. resume the materialized table - try { - executeContinuousRefreshJob( - operationExecutor, - handle, - alterMaterializedTableChangeOperation.getNewTable(), - tableIdentifier, - Map.of(), - Optional.empty()); - } catch (Exception e) { - // Roll back the changes to the materialized table and restore the continuous - // refresh job - LOG.warn( - "Failed to start the continuous refresh job for materialized table {} using new query {}, rollback to origin query {}.", - tableIdentifier, - op.getNewTable().getExpandedQuery(), - suspendMaterializedTable.getExpandedQuery(), - e); + alterActivatedContinuousMaterializedTable(op, oldMaterializedTable); + } else if (RefreshStatus.SUSPENDED == oldMaterializedTable.getRefreshStatus()) { + alterSuspendedContinuousMaterializedTable(op, oldMaterializedTable); + } else { + throw new TableException( + String.format( + "Materialized table %s is being initialized and does not support alter operation.", + tableIdentifier)); + } - AlterMaterializedTableChangeOperation rollbackChangeOperation = - generateRollbackAlterMaterializedTableOperation( - suspendMaterializedTable, alterMaterializedTableChangeOperation); - operationExecutor.callExecutableOperation(handle, rollbackChangeOperation); + return TABLE_RESULT_OK; + } - ContinuousRefreshHandler continuousRefreshHandler = - deserializeContinuousHandler( - suspendMaterializedTable.getSerializedRefreshHandler()); - executeContinuousRefreshJob( - operationExecutor, - handle, + /** Stops the refresh job, applies the change and restarts it, rolling back on failure. */ + private void alterActivatedContinuousMaterializedTable( + AlterMaterializedTableChangeOperation op, + ResolvedCatalogMaterializedTable oldMaterializedTable) { + final ObjectIdentifier tableIdentifier = op.getTableIdentifier(); + // check the target before the stop, so a restart that could never run keeps the job + // running and the table untouched + checkRestartTargetBeforeChanges( + tableIdentifier, oldMaterializedTable.getSerializedRefreshHandler()); + + // 1. suspend the materialized table + ResolvedCatalogMaterializedTable suspendMaterializedTable = + suspendContinuousRefreshJob(tableIdentifier, oldMaterializedTable); + + // 2. alter materialized table schema & query definition + final AlterMaterializedTableChangeOperation alterMaterializedTableChangeOperation = + new AlterMaterializedTableChangeOperation( + op.getTableIdentifier(), + oldTable -> op.getTableChanges(), suspendMaterializedTable, - tableIdentifier, - Map.of(), - continuousRefreshHandler.getRestorePath()); + op.getAsQueryOperation()); + alterMaterializedTableChangeOperation.execute(operationCtx); - throw new SqlExecutionException( - String.format( - "Failed to start the continuous refresh job using new query %s when altering materialized table %s select query.", - op.getNewTable().getExpandedQuery(), tableIdentifier), - e); - } - } else if (RefreshStatus.SUSPENDED == oldMaterializedTable.getRefreshStatus()) { - // alter schema & definition query & refresh handler (reset savepoint path of refresh - // handler) - List tableChanges = new ArrayList<>(op.getTableChanges()); - TableChange.ModifyRefreshHandler modifyRefreshHandler = - generateResetSavepointTableChange( - oldMaterializedTable.getSerializedRefreshHandler()); - tableChanges.add(modifyRefreshHandler); - - AlterMaterializedTableChangeOperation alterMaterializedTableChangeOperation = - new AlterMaterializedTableChangeOperation( - tableIdentifier, - oldTable -> tableChanges, - oldMaterializedTable, - op.getAsQueryOperation()); + // 3. resume the materialized table + try { + executeContinuousRefreshJob( + alterMaterializedTableChangeOperation.getNewTable(), + tableIdentifier, + Map.of(), + Optional.empty()); + } catch (Exception e) { + final ContinuousRefreshHandler suspendedRefreshHandler = + deserializeContinuousHandler( + suspendMaterializedTable.getSerializedRefreshHandler()); + LOG.warn( + "Failed to start the continuous refresh job for materialized table {} using new query {}, rollback to origin query {} with savepoint {}.", + tableIdentifier, + op.getNewTable().getExpandedQuery(), + suspendMaterializedTable.getExpandedQuery(), + suspendedRefreshHandler.getRestorePath().orElse(null), + e); - operationExecutor.callExecutableOperation( - handle, alterMaterializedTableChangeOperation); - } else { - throw new SqlExecutionException( + // Roll back the changes to the materialized table and restore the continuous + // refresh job + generateRollbackAlterMaterializedTableOperation( + suspendMaterializedTable, alterMaterializedTableChangeOperation) + .execute(operationCtx); + executeContinuousRefreshJob( + suspendMaterializedTable, + tableIdentifier, + Map.of(), + suspendedRefreshHandler.getRestorePath()); + + throw new TableException( String.format( - "Materialized table %s is being initialized and does not support alter operation.", - tableIdentifier)); + "Failed to start the continuous refresh job using new query %s when altering materialized table %s select query.", + op.getNewTable().getExpandedQuery(), tableIdentifier), + e); } + } + + private void alterSuspendedContinuousMaterializedTable( + AlterMaterializedTableChangeOperation op, + ResolvedCatalogMaterializedTable oldMaterializedTable) { + // alter schema & definition query & refresh handler (reset savepoint path of refresh + // handler) + List tableChanges = new ArrayList<>(op.getTableChanges()); + tableChanges.add( + generateResetSavepointTableChange( + deserializeContinuousHandler( + oldMaterializedTable.getSerializedRefreshHandler()))); - return ResultFetcher.fromTableResult(handle, TABLE_RESULT_OK, false); + AlterMaterializedTableChangeOperation alterMaterializedTableChangeOperation = + new AlterMaterializedTableChangeOperation( + op.getTableIdentifier(), + oldTable -> tableChanges, + oldMaterializedTable, + op.getAsQueryOperation()); + + alterMaterializedTableChangeOperation.execute(operationCtx); + } + + private static TableException withSuppressed( + String message, Exception cause, Exception suppressed) { + final TableException failure = new TableException(message, cause); + failure.addSuppressed(suppressed); + return failure; + } + + private static void cleanUpAfterFailure( + Runnable cleanup, Exception failure, String cleanupFailureMessage) { + try { + cleanup.run(); + } catch (Exception cleanupFailure) { + throw withSuppressed(cleanupFailureMessage, cleanupFailure, failure); + } } private AlterMaterializedTableChangeOperation generateRollbackAlterMaterializedTableOperation( @@ -1040,9 +1038,7 @@ private AlterMaterializedTableChangeOperation generateRollbackAlterMaterializedT } private TableChange.ModifyRefreshHandler generateResetSavepointTableChange( - byte[] serializedContinuousHandler) { - ContinuousRefreshHandler continuousRefreshHandler = - deserializeContinuousHandler(serializedContinuousHandler); + ContinuousRefreshHandler continuousRefreshHandler) { ContinuousRefreshHandler resetContinuousRefreshHandler = new ContinuousRefreshHandler( continuousRefreshHandler.getExecutionTarget(), @@ -1054,18 +1050,21 @@ private TableChange.ModifyRefreshHandler generateResetSavepointTableChange( serializeContinuousHandler(resetContinuousRefreshHandler)); } - private ResultFetcher callDropMaterializedTableOperation( - OperationExecutor operationExecutor, - OperationHandle handle, + private TableResultInternal callDropMaterializedTableOperation( DropMaterializedTableOperation dropMaterializedTableOperation) { ObjectIdentifier tableIdentifier = dropMaterializedTableOperation.getTableIdentifier(); - boolean tableExists = operationExecutor.tableExists(tableIdentifier); + boolean tableExists = + operationCtx + .getCatalogManager() + .getCatalog(tableIdentifier.getCatalogName()) + .map(catalog -> catalog.tableExists(tableIdentifier.toObjectPath())) + .orElse(false); if (!tableExists) { if (dropMaterializedTableOperation.isIfExists()) { LOG.info( "Materialized table {} does not exists, skip the drop operation.", tableIdentifier); - return ResultFetcher.fromTableResult(handle, TABLE_RESULT_OK, false); + return TABLE_RESULT_OK; } else { throw new ValidationException( String.format( @@ -1075,7 +1074,7 @@ private ResultFetcher callDropMaterializedTableOperation( } ResolvedCatalogMaterializedTable materializedTable = - getCatalogMaterializedTable(operationExecutor, tableIdentifier); + getCatalogMaterializedTable(tableIdentifier); RefreshStatus refreshStatus = materializedTable.getRefreshStatus(); if (RefreshStatus.ACTIVATED == refreshStatus || RefreshStatus.SUSPENDED == refreshStatus) { RefreshMode refreshMode = materializedTable.getRefreshMode(); @@ -1083,8 +1082,7 @@ private ResultFetcher callDropMaterializedTableOperation( deleteRefreshWorkflow(tableIdentifier, materializedTable); } else if (RefreshMode.CONTINUOUS == refreshMode && RefreshStatus.ACTIVATED == refreshStatus) { - cancelContinuousRefreshJob( - operationExecutor, handle, tableIdentifier, materializedTable); + cancelContinuousRefreshJob(tableIdentifier, materializedTable); } } else if (RefreshStatus.INITIALIZING == refreshStatus) { throw new ValidationException( @@ -1093,65 +1091,65 @@ private ResultFetcher callDropMaterializedTableOperation( tableIdentifier.asSerializableString())); } - operationExecutor.callExecutableOperation(handle, dropMaterializedTableOperation); + dropMaterializedTableOperation.execute(operationCtx); - return ResultFetcher.fromTableResult(handle, TABLE_RESULT_OK, false); + return TABLE_RESULT_OK; } private void cancelContinuousRefreshJob( - OperationExecutor operationExecutor, - OperationHandle handle, - ObjectIdentifier tableIdentifier, - CatalogMaterializedTable materializedTable) { - ContinuousRefreshHandler refreshHandler = + ObjectIdentifier tableIdentifier, CatalogMaterializedTable materializedTable) { + final ContinuousRefreshHandler refreshHandler = deserializeContinuousHandler(materializedTable.getSerializedRefreshHandler()); - // get job running status - JobStatus jobStatus = getJobStatus(operationExecutor, handle, refreshHandler); - if (!jobStatus.isTerminalState()) { - try { - cancelJob(operationExecutor, handle, refreshHandler); - } catch (Exception e) { - jobStatus = getJobStatus(operationExecutor, handle, refreshHandler); - if (!jobStatus.isTerminalState()) { - throw new SqlExecutionException( - String.format( - "Failed to drop the materialized table %s because the continuous refresh job %s could not be canceled." - + " The current status of the continuous refresh job is %s.", - tableIdentifier, refreshHandler.getJobId(), jobStatus), - e); - } else { - LOG.warn( - "An exception occurred while canceling the continuous refresh job {} for materialized table {}," - + " but since the job is in a terminal state, skip the cancel operation.", - refreshHandler.getJobId(), - tableIdentifier); - } - } - } else { + if (refreshJobController.getJobStatus(refreshHandler).isTerminalState()) { LOG.info( "No need to cancel the continuous refresh job {} for materialized table {} as it is not currently running.", refreshHandler.getJobId(), tableIdentifier); + return; + } + + final Exception cancelFailure; + try { + refreshJobController.cancel(refreshHandler); + return; + } catch (Exception e) { + cancelFailure = e; } + + final JobStatus jobStatus = refreshJobController.getJobStatus(refreshHandler); + if (!jobStatus.isTerminalState()) { + throw new TableException( + String.format( + "Failed to drop the materialized table %s because the continuous refresh job %s could not be canceled." + + " The current status of the continuous refresh job is %s.", + tableIdentifier, refreshHandler.getJobId(), jobStatus), + cancelFailure); + } + LOG.warn( + "An exception occurred while canceling the continuous refresh job {} for materialized table {}," + + " but since the job is in a terminal state, skip the cancel operation.", + refreshHandler.getJobId(), + tableIdentifier, + cancelFailure); } private void deleteRefreshWorkflow( ObjectIdentifier tableIdentifier, CatalogMaterializedTable catalogMaterializedTable) { - if (workflowScheduler == null) { - throw new SqlExecutionException( - "The workflow scheduler must be configured when dropping materialized table in full refresh mode."); - } + WorkflowScheduler workflowScheduler = + requireRefreshWorkflowContext( + "The workflow scheduler must be configured when dropping materialized table in full refresh mode.") + .scheduler(); try { RefreshHandlerSerializer refreshHandlerSerializer = workflowScheduler.getRefreshHandlerSerializer(); RefreshHandler refreshHandler = refreshHandlerSerializer.deserialize( catalogMaterializedTable.getSerializedRefreshHandler(), - userCodeClassLoader); + operationCtx.getResourceManager().getUserClassLoader()); DeleteRefreshWorkflow deleteRefreshWorkflow = new DeleteRefreshWorkflow(refreshHandler); workflowScheduler.deleteRefreshWorkflow(deleteRefreshWorkflow); } catch (Exception e) { - throw new SqlExecutionException( + throw new TableException( String.format( "Failed to delete the refresh workflow for materialized table %s.", tableIdentifier), @@ -1159,106 +1157,13 @@ private void deleteRefreshWorkflow( } } - /** - * Retrieves the session configuration for initializing the periodic refresh job. The function - * filters out default context configurations and removes unnecessary configurations such as - * resources download directory and workflow scheduler related configurations. - * - * @param operationExecutor The OperationExecutor instance used to access the session context. - * @return A Map containing the session configurations for initializing session for executing - * the periodic refresh job. - */ - private Map getSessionInitializationConf(OperationExecutor operationExecutor) { - Map sessionConf = - operationExecutor.getSessionContext().getSessionConf().toMap(); - - // we only keep the session conf that is not in the default context or the conf value is - // different from the default context. - Map defaultContextConf = - operationExecutor.getSessionContext().getDefaultContext().getFlinkConfig().toMap(); - sessionConf - .entrySet() - .removeIf( - entry -> { - String key = entry.getKey(); - String value = entry.getValue(); - return defaultContextConf.containsKey(key) - && defaultContextConf.get(key).equals(value); - }); - - // remove useless conf - sessionConf.remove(TableConfigOptions.RESOURCES_DOWNLOAD_DIR.key()); - sessionConf.keySet().removeIf(key -> key.startsWith(WORKFLOW_SCHEDULER_PREFIX)); - - return sessionConf; - } - - private static JobStatus getJobStatus( - OperationExecutor operationExecutor, - OperationHandle handle, - ContinuousRefreshHandler refreshHandler) { - ResultFetcher resultFetcher = - operationExecutor.callDescribeJobOperation( - getTableEnvironment(operationExecutor, refreshHandler), - handle, - new DescribeJobOperation(refreshHandler.getJobId())); - List result = fetchAllResults(resultFetcher); - String jobStatus = result.get(0).getString(2).toString(); - return JobStatus.valueOf(jobStatus); - } - - private static void cancelJob( - OperationExecutor operationExecutor, - OperationHandle handle, - ContinuousRefreshHandler refreshHandler) { - operationExecutor.callStopJobOperation( - getTableEnvironment(operationExecutor, refreshHandler), - handle, - new StopJobOperation(refreshHandler.getJobId(), false, false)); - } - - private static String stopJobWithSavepoint( - OperationExecutor executor, - OperationHandle handle, - ContinuousRefreshHandler refreshHandler) { - // check savepoint dir is configured - Optional savepointDir = - executor.getSessionContext().getSessionConf().getOptional(SAVEPOINT_DIRECTORY); - if (savepointDir.isEmpty()) { - throw new ValidationException( - "Savepoint directory is not configured, can't stop job with savepoint."); - } - String jobId = refreshHandler.getJobId(); - ResultFetcher resultFetcher = - executor.callStopJobOperation( - getTableEnvironment(executor, refreshHandler), - handle, - new StopJobOperation(jobId, true, false)); - List results = fetchAllResults(resultFetcher); - return results.get(0).getString(0).toString(); - } - - private static TableEnvironmentInternal getTableEnvironment( - OperationExecutor executor, ContinuousRefreshHandler refreshHandler) { - - String target = refreshHandler.getExecutionTarget(); - Configuration sessionConfiguration = new Configuration(); - sessionConfiguration.set(TARGET, target); - Optional clusterIdKeyName = getClusterIdKeyName(target); - clusterIdKeyName.ifPresent( - s -> sessionConfiguration.setString(s, refreshHandler.getClusterId())); - - return executor.getTableEnvironment( - executor.getSessionContext().getSessionState().resourceManager, - sessionConfiguration); - } - private ContinuousRefreshHandler deserializeContinuousHandler(byte[] serializedRefreshHandler) { try { return ContinuousRefreshHandlerSerializer.INSTANCE.deserialize( - serializedRefreshHandler, userCodeClassLoader); + serializedRefreshHandler, + operationCtx.getResourceManager().getUserClassLoader()); } catch (IOException | ClassNotFoundException e) { - throw new SqlExecutionException("Failed to deserialize ContinuousRefreshHandler.", e); + throw new TableException("Failed to deserialize ContinuousRefreshHandler.", e); } } @@ -1266,14 +1171,17 @@ private byte[] serializeContinuousHandler(ContinuousRefreshHandler refreshHandle try { return ContinuousRefreshHandlerSerializer.INSTANCE.serialize(refreshHandler); } catch (IOException e) { - throw new SqlExecutionException("Failed to serialize ContinuousRefreshHandler.", e); + throw new TableException("Failed to serialize ContinuousRefreshHandler.", e); } } private ResolvedCatalogMaterializedTable getCatalogMaterializedTable( - OperationExecutor operationExecutor, ObjectIdentifier tableIdentifier) { + ObjectIdentifier tableIdentifier) { ResolvedCatalogBaseTable resolvedCatalogBaseTable = - operationExecutor.getTable(tableIdentifier); + operationCtx + .getCatalogManager() + .getTableOrError(tableIdentifier) + .getResolvedTable(); if (MATERIALIZED_TABLE != resolvedCatalogBaseTable.getTableKind()) { throw new ValidationException( String.format( @@ -1285,17 +1193,11 @@ private ResolvedCatalogMaterializedTable getCatalogMaterializedTable( } private ResolvedCatalogMaterializedTable resolveCatalogMaterializedTable( - OperationExecutor operationExecutor, CatalogMaterializedTable materializedTable) { - return operationExecutor - .getSessionContext() - .getSessionState() - .catalogManager - .resolveCatalogMaterializedTable(materializedTable); + CatalogMaterializedTable materializedTable) { + return operationCtx.getCatalogManager().resolveCatalogMaterializedTable(materializedTable); } private ResolvedCatalogMaterializedTable updateRefreshHandler( - OperationExecutor operationExecutor, - OperationHandle operationHandle, ObjectIdentifier materializedTableIdentifier, ResolvedCatalogMaterializedTable catalogMaterializedTable, RefreshStatus refreshStatus, @@ -1311,16 +1213,14 @@ private ResolvedCatalogMaterializedTable updateRefreshHandler( oldTable -> tableChanges, catalogMaterializedTable); // update RefreshHandler to Catalog - operationExecutor.callExecutableOperation( - operationHandle, alterMaterializedTableChangeOperation); + alterMaterializedTableChangeOperation.execute(operationCtx); - return resolveCatalogMaterializedTable( - operationExecutor, alterMaterializedTableChangeOperation.getNewTable()); + return resolveCatalogMaterializedTable(alterMaterializedTableChangeOperation.getNewTable()); } /** Generate insert statement for materialized table. */ @VisibleForTesting - protected static String getInsertStatement( + static String getInsertStatement( ObjectIdentifier materializedTableIdentifier, String definitionQuery, Map dynamicOptions) { @@ -1346,123 +1246,52 @@ private static String generateTableWithDynamicOptions( return builder.toString(); } - private static List fetchAllResults(ResultFetcher resultFetcher) { - Long token = 0L; - List results = new ArrayList<>(); - while (token != null) { - ResultSet result = resultFetcher.fetchResults(token, Integer.MAX_VALUE); - results.addAll(result.getData()); - token = result.getNextToken(); + private void checkSubmissionTargetBeforeChanges() { + if (!jobSubmitter.supportsApplicationTargets()) { + RefreshJobTarget.forNewJob(operationCtx.getTableConfig()); } - return results; } - private static JobExecutionResult executeRefreshJob( - String script, - Configuration executionConfig, - OperationExecutor operationExecutor, - OperationHandle operationHandle) { - String executeTarget = operationExecutor.getSessionContext().getSessionConf().get(TARGET); - if (executeTarget == null || executeTarget.isEmpty() || "local".equals(executeTarget)) { - String errorMessage = - String.format( - "Unsupported execution target detected: %s." - + "Currently, only the following execution targets are supported: " - + "'remote', 'yarn-session', 'yarn-application', 'kubernetes-session', 'kubernetes-application'. ", - executeTarget); - LOG.error(errorMessage); - throw new ValidationException(errorMessage); - } - - if (executeTarget.endsWith("application")) { - return executeApplicationJob(script, executionConfig, operationExecutor); - } else { - return executeNonApplicationJob( - script, executionConfig, operationExecutor, operationHandle); + private void checkRestartTargetBeforeChanges( + ObjectIdentifier tableIdentifier, byte[] serializedRefreshHandler) { + if (!jobSubmitter.supportsApplicationTargets()) { + RefreshJobTarget.forRestart( + operationCtx.getTableConfig(), + tableIdentifier, + deserializeContinuousHandler(serializedRefreshHandler)); } } - private static JobExecutionResult executeNonApplicationJob( - String script, - Configuration executionConfig, - OperationExecutor operationExecutor, - OperationHandle operationHandle) { - String executeTarget = operationExecutor.getSessionContext().getSessionConf().get(TARGET); - String clusterId = - operationExecutor - .getSessionClusterId() - .orElseThrow( - () -> { - String errorMessage = - String.format( - "No cluster ID found when executing materialized table refresh job. Execution target is : %s", - executeTarget); - LOG.error(errorMessage); - return new ValidationException(errorMessage); - }); - - ResultFetcher resultFetcher = - operationExecutor.executeStatement(operationHandle, executionConfig, script); - List results = fetchAllResults(resultFetcher); - String jobId = results.get(0).getString(0).toString(); - - return new JobExecutionResult(executeTarget, clusterId, jobId); - } - - private static JobExecutionResult executeApplicationJob( - String script, Configuration executionConfig, OperationExecutor operationExecutor) { - List arguments = new ArrayList<>(); - arguments.add("--" + SqlDriver.OPTION_SQL_SCRIPT.getLongOpt()); - arguments.add(script); - - Configuration mergedConfig = - new Configuration(operationExecutor.getSessionContext().getSessionConf()); - mergedConfig.addAll(executionConfig); - JobID jobId = new JobID(); - mergedConfig.set(PIPELINE_FIXED_JOB_ID, jobId.toString()); - - ApplicationConfiguration applicationConfiguration = - new ApplicationConfiguration( - arguments.toArray(new String[0]), SqlDriver.class.getName()); - try { - String clusterId = - new ApplicationClusterDeployer(new DefaultClusterClientServiceLoader()) - .run(mergedConfig, applicationConfiguration) - .toString(); - - return new JobExecutionResult(mergedConfig.get(TARGET), clusterId, jobId.toString()); - } catch (Throwable t) { - LOG.error("Failed to deploy script {} to application cluster.", script, t); - throw new SqlGatewayException("Failed to deploy script to cluster.", t); - } + private String requireSavepointDirectory() { + return operationCtx + .getTableConfig() + .getOptional(CheckpointingOptions.SAVEPOINT_DIRECTORY) + .orElseThrow( + () -> + new ValidationException( + "Savepoint directory is not configured, can't stop job with savepoint.")); } - private static class JobExecutionResult { - - private final String executionTarget; - private final String clusterId; - private final String jobId; - - public JobExecutionResult(String executionTarget, String clusterId, String jobId) { - this.executionTarget = executionTarget; - this.clusterId = clusterId; - this.jobId = jobId; - } + private static IntervalFreshness validateAndGetIntervalFreshness( + final CatalogMaterializedTable catalogMaterializedTable) { + return Optional.ofNullable(catalogMaterializedTable.getDefinitionFreshness()) + .orElseThrow(() -> new TableException("Freshness cannot be null")); } - private static Optional getClusterIdKeyName(String targetName) { - if (targetName.startsWith("yarn")) { - return Optional.of("yarn.application.id"); - } else if (targetName.startsWith("kubernetes")) { - return Optional.of("kubernetes.cluster-id"); - } else { - return Optional.empty(); - } + private RefreshWorkflowContext requireRefreshWorkflowContext(String missingSchedulerMessage) { + return jobSubmitter + .getRefreshWorkflowContext() + .orElseThrow(() -> new TableException(missingSchedulerMessage)); } - private static IntervalFreshness validateAndGetIntervalFreshness( - final CatalogMaterializedTable catalogMaterializedTable) { - return Optional.ofNullable(catalogMaterializedTable.getDefinitionFreshness()) - .orElseThrow(() -> new SqlExecutionException("Freshness cannot be null")); + private static RowData refreshRowToInternalRow(Row row) { + final Map internalClusterInfo = new HashMap<>(); + row.>getFieldAs(1) + .forEach( + (key, value) -> + internalClusterInfo.put( + StringData.fromString(key), StringData.fromString(value))); + return GenericRowData.of( + StringData.fromString(row.getFieldAs(0)), new GenericMapData(internalClusterInfo)); } } diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/delegation/materializedtable/DefaultMaterializedTableExecutorFactory.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/delegation/materializedtable/DefaultMaterializedTableExecutorFactory.java new file mode 100644 index 00000000000000..f488278d21de7e --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/delegation/materializedtable/DefaultMaterializedTableExecutorFactory.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.delegation.materializedtable; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutor; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutorFactory; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableJobSubmitter; +import org.apache.flink.table.operations.ExecutableOperation; + +import java.util.Set; + +/** Factory that creates the default {@link DefaultMaterializedTableExecutor}. */ +@Internal +public class DefaultMaterializedTableExecutorFactory implements MaterializedTableExecutorFactory { + + @Override + public MaterializedTableExecutor createExecutor( + ExecutableOperation.Context operationCtx, MaterializedTableJobSubmitter jobSubmitter) { + return new DefaultMaterializedTableExecutor(operationCtx, jobSubmitter); + } + + @Override + public String factoryIdentifier() { + return MaterializedTableExecutorFactory.DEFAULT_IDENTIFIER; + } + + @Override + public Set> requiredOptions() { + return Set.of(); + } + + @Override + public Set> optionalOptions() { + return Set.of(); + } +} diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/delegation/materializedtable/RefreshJobController.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/delegation/materializedtable/RefreshJobController.java new file mode 100644 index 00000000000000..65fc18c8dfc709 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/delegation/materializedtable/RefreshJobController.java @@ -0,0 +1,381 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.delegation.materializedtable; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.JobStatus; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.DeploymentOptions; +import org.apache.flink.core.execution.DefaultExecutorServiceLoader; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.core.execution.JobRetrievalSupportedPipelineExecutor; +import org.apache.flink.core.execution.PipelineExecutor; +import org.apache.flink.core.execution.SavepointFormatType; +import org.apache.flink.runtime.messages.FlinkJobNotFoundException; +import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.api.TableException; +import org.apache.flink.table.delegation.materializedtable.RefreshJobResult; +import org.apache.flink.table.delegation.materializedtable.RefreshJobTarget; +import org.apache.flink.table.refresh.ContinuousRefreshHandler; +import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.TimeUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** Controls the continuous refresh jobs of materialized tables. */ +@Internal +final class RefreshJobController { + + private static final Logger LOG = LoggerFactory.getLogger(RefreshJobController.class); + + static final ConfigOption CLIENT_TIMEOUT = + ConfigOptions.key("client.timeout").durationType().defaultValue(Duration.ofSeconds(60)); + + private static final String CONFIGURED_COORDINATES_HINT = + "The job was looked up with coordinates the catalog does not record, such as " + + "'rest.address', 'kubernetes.namespace' and high-availability settings, " + + "which are taken from this program's configuration and must point at the " + + "cluster the job runs on."; + + private static final String REQUEST_MAY_STILL_COMPLETE = + "The request may still complete on the cluster, so check the status of the refresh " + + "job before retrying."; + + private static final String SAVEPOINT_RECOVERY_HINT = + "If the job still stops, its savepoint is written under '%s' while the materialized " + + "table stays ACTIVATED; set 'execution.state-recovery.path' to that savepoint " + + "before resuming, and unset it afterwards (RESET in SQL), otherwise RESUME " + + "starts the refresh job without restoring state."; + + private final TableConfig tableConfig; + private final ClassLoader userClassLoader; + private final Function> + executorLookup; + + private final Map submittedJobClients = new HashMap<>(); + private final Map retrievedJobClients = new HashMap<>(); + + RefreshJobController(TableConfig tableConfig, ClassLoader userClassLoader) { + this(tableConfig, userClassLoader, RefreshJobController::findRetrievalExecutor); + } + + @VisibleForTesting + RefreshJobController( + TableConfig tableConfig, + ClassLoader userClassLoader, + Function> + executorLookup) { + this.tableConfig = checkNotNull(tableConfig); + this.userClassLoader = checkNotNull(userClassLoader); + this.executorLookup = checkNotNull(executorLookup); + } + + void register(RefreshJobResult result) { + result.getJobClient() + .ifPresent(jobClient -> submittedJobClients.put(result.getJobId(), jobClient)); + } + + JobStatus getJobStatus(ContinuousRefreshHandler refreshHandler) { + final JobStatus jobStatus; + try { + jobStatus = await(refreshHandler, JobRequest.GET_STATUS, JobClient::getJobStatus, ""); + } catch (TableException e) { + if (isUnknownJob(e)) { + throw new TableException( + String.format( + "Described job %s does not exist in the cluster.", + refreshHandler.getJobId())); + } + throw e; + } + if (jobStatus.isGloballyTerminalState()) { + submittedJobClients.remove(refreshHandler.getJobId()); + } + return jobStatus; + } + + void cancel(ContinuousRefreshHandler refreshHandler) { + await(refreshHandler, JobRequest.CANCEL, JobClient::cancel, ""); + submittedJobClients.remove(refreshHandler.getJobId()); + } + + String stopWithSavepoint(ContinuousRefreshHandler refreshHandler, String savepointDirectory) { + final String savepointPath = + await( + refreshHandler, + JobRequest.STOP_WITH_SAVEPOINT, + jobClient -> + jobClient.stopWithSavepoint( + false, savepointDirectory, SavepointFormatType.DEFAULT), + String.format(SAVEPOINT_RECOVERY_HINT, savepointDirectory)); + submittedJobClients.remove(refreshHandler.getJobId()); + return savepointPath; + } + + void releaseRetrievedJobClients() { + for (JobClient jobClient : retrievedJobClients.values()) { + if (jobClient instanceof AutoCloseable) { + try { + ((AutoCloseable) jobClient).close(); + } catch (Exception e) { + LOG.warn( + "Failed to close the client of refresh job {}.", + jobClient.getJobID(), + e); + } + } + } + retrievedJobClients.clear(); + } + + private JobClient lookup(ContinuousRefreshHandler refreshHandler) { + final JobClient submittedJobClient = submittedJobClients.get(refreshHandler.getJobId()); + if (submittedJobClient != null) { + return submittedJobClient; + } + final JobClient cachedJobClient = retrievedJobClients.get(refreshHandler.getJobId()); + if (cachedJobClient != null) { + return cachedJobClient; + } + + final Optional retrievedJobClient; + try { + retrievedJobClient = retrieve(refreshHandler); + } catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + + throw new TableException( + "Failed to run cluster action.", ExceptionUtils.stripExecutionException(e)); + } + final JobClient jobClient = + retrievedJobClient.orElseThrow( + () -> noRetrievalExecutor(refreshHandler, retrievalTarget(refreshHandler))); + retrievedJobClients.put(refreshHandler.getJobId(), jobClient); + return jobClient; + } + + private Optional retrieve(ContinuousRefreshHandler refreshHandler) throws Exception { + final Configuration retrievalConfiguration = programConfiguration(); + RefreshJobTarget.recordedIn(refreshHandler).applyTo(retrievalConfiguration); + retrievalConfiguration.set(DeploymentOptions.TARGET, retrievalTarget(refreshHandler)); + final Optional executor = + executorLookup.apply(retrievalConfiguration); + if (executor.isEmpty()) { + return Optional.empty(); + } + return Optional.of( + executor.get() + .retrieveJobClient( + JobID.fromHexString(refreshHandler.getJobId()), + retrievalConfiguration, + userClassLoader) + .get()); + } + + private static Optional findRetrievalExecutor( + Configuration configuration) { + final PipelineExecutor executor; + try { + executor = + new DefaultExecutorServiceLoader() + .getExecutorFactory(configuration) + .getExecutor(configuration); + } catch (IllegalStateException e) { + LOG.info( + "No usable pipeline executor for execution target '{}'.", + configuration.get(DeploymentOptions.TARGET)); + return Optional.empty(); + } + return executor instanceof JobRetrievalSupportedPipelineExecutor + ? Optional.of((JobRetrievalSupportedPipelineExecutor) executor) + : Optional.empty(); + } + + private static String retrievalTarget(ContinuousRefreshHandler refreshHandler) { + final String recordedTarget = refreshHandler.getExecutionTarget(); + if (RefreshJobTarget.recordedIn(refreshHandler).isApplicationTarget()) { + return recordedTarget.substring(0, recordedTarget.length() - "application".length()) + + "session"; + } + return recordedTarget; + } + + private Configuration programConfiguration() { + final Configuration configuration = + Configuration.fromMap(tableConfig.getRootConfiguration().toMap()); + configuration.addAll(tableConfig.getConfiguration()); + return configuration; + } + + private static boolean isUnknownJob(Throwable failure) { + return ExceptionUtils.findThrowable(failure, FlinkJobNotFoundException.class).isPresent() + || ExceptionUtils.findThrowableWithMessage( + failure, FlinkJobNotFoundException.class.getName()) + .isPresent(); + } + + /** Sends a request to the job and waits for its result. */ + private T await( + ContinuousRefreshHandler refreshHandler, + JobRequest jobRequest, + Function> request, + String recoveryHint) { + final String action = jobRequest.action; + final String coordinatesHint = + submittedJobClients.containsKey(refreshHandler.getJobId()) + ? "" + : CONFIGURED_COORDINATES_HINT; + final JobClient jobClient = lookup(refreshHandler); + + final CompletableFuture response; + try { + response = request.apply(jobClient); + } catch (Exception e) { + if (jobRequest == JobRequest.GET_STATUS) { + throw statusLookupFailure(refreshHandler, e); + } + throw new TableException( + withHints(failureMessage(action, refreshHandler), coordinatesHint), e); + } + + final String pendingRequestHint = + jobRequest.mayStillComplete + ? withHints(REQUEST_MAY_STILL_COMPLETE, recoveryHint) + : ""; + final Duration timeout = tableConfig.get(CLIENT_TIMEOUT); + try { + return response.get(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + final TableException timeoutFailure = + new TableException( + withHints( + timeoutMessage(action, refreshHandler, timeout), + coordinatesHint, + pendingRequestHint), + e); + throw jobRequest == JobRequest.GET_STATUS + ? statusLookupFailure(refreshHandler, timeoutFailure) + : timeoutFailure; + } catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + + if (jobRequest == JobRequest.GET_STATUS) { + throw statusLookupFailure(refreshHandler, e); + } + throw new TableException( + withHints( + failureMessage(action, refreshHandler), + coordinatesHint, + pendingRequestHint), + e); + } + } + + private static TableException statusLookupFailure( + ContinuousRefreshHandler refreshHandler, Throwable cause) { + return new TableException( + String.format("Failed to get job %s in the cluster.", refreshHandler.getJobId()), + cause); + } + + private static String failureMessage(String action, ContinuousRefreshHandler refreshHandler) { + return String.format("Failed to %s.", describe(action, refreshHandler)); + } + + private static TableException noRetrievalExecutor( + ContinuousRefreshHandler refreshHandler, String lookedUpTarget) { + return new TableException( + String.format( + "Cannot reach refresh job %s: no pipeline executor that can reach " + + "existing jobs supports execution target '%s'%s. Either the " + + "executor for this target is missing from this program's class " + + "path (flink-clients, plus flink-yarn with HADOOP_CLASSPATH or " + + "flink-kubernetes), more than one PipelineExecutorFactory on the " + + "class path supports the target, or jobs on this target can only " + + "be controlled from the TableEnvironment that submitted them.", + refreshHandler.getJobId(), + lookedUpTarget, + lookedUpTarget.equals(refreshHandler.getExecutionTarget()) + ? "" + : String.format( + " (used for the recorded target '%s')", + refreshHandler.getExecutionTarget()))); + } + + private static String timeoutMessage( + String action, ContinuousRefreshHandler refreshHandler, Duration timeout) { + return String.format( + "Timed out after %s ('%s') waiting to %s.", + TimeUtils.formatWithHighestUnit(timeout), + CLIENT_TIMEOUT.key(), + describe(action, refreshHandler)); + } + + private static String withHints(String message, String... hints) { + final StringBuilder messageWithHints = new StringBuilder(message); + for (String hint : hints) { + if (!hint.isEmpty()) { + messageWithHints.append(' ').append(hint); + } + } + return messageWithHints.toString(); + } + + private static String describe(String action, ContinuousRefreshHandler refreshHandler) { + return String.format( + "%s (execution.target=%s, cluster id=%s)", + String.format(action, refreshHandler.getJobId()), + refreshHandler.getExecutionTarget(), + RefreshJobTarget.recordedIn(refreshHandler).getClusterId()); + } + + private enum JobRequest { + GET_STATUS("get the status of refresh job %s", false), + CANCEL("cancel refresh job %s", true), + STOP_WITH_SAVEPOINT("stop refresh job %s with a savepoint", true); + + private final String action; + private final boolean mayStillComplete; + + JobRequest(String action, boolean mayStillComplete) { + this.action = action; + this.mayStillComplete = mayStillComplete; + } + } +} diff --git a/flink-table/flink-table-planner/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-table/flink-table-planner/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory index ac5f9b6bb17677..6c1075fd0d52a8 100644 --- a/flink-table/flink-table-planner/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory +++ b/flink-table/flink-table-planner/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -16,3 +16,4 @@ org.apache.flink.table.planner.delegation.DefaultExecutorFactory org.apache.flink.table.planner.delegation.DefaultParserFactory org.apache.flink.table.planner.delegation.DefaultPlannerFactory +org.apache.flink.table.planner.delegation.materializedtable.DefaultMaterializedTableExecutorFactory diff --git a/flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManagerTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/delegation/materializedtable/DefaultMaterializedTableExecutorTest.java similarity index 66% rename from flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManagerTest.java rename to flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/delegation/materializedtable/DefaultMaterializedTableExecutorTest.java index 993e60db1562dc..79b85304a69833 100644 --- a/flink-table/flink-sql-gateway/src/test/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManagerTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/delegation/materializedtable/DefaultMaterializedTableExecutorTest.java @@ -16,10 +16,45 @@ * limitations under the License. */ -package org.apache.flink.table.gateway.service.materializedtable; +package org.apache.flink.table.planner.delegation.materializedtable; +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.JobStatus; +import org.apache.flink.configuration.CheckpointingOptions; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.DeploymentOptions; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableException; +import org.apache.flink.table.api.bridge.internal.AbstractStreamTableEnvironmentImpl; +import org.apache.flink.table.api.bridge.java.internal.StreamTableEnvironmentImpl; +import org.apache.flink.table.catalog.Catalog; +import org.apache.flink.table.catalog.CatalogBaseTable; +import org.apache.flink.table.catalog.CatalogManager; +import org.apache.flink.table.catalog.CatalogMaterializedTable; +import org.apache.flink.table.catalog.CatalogMaterializedTable.RefreshStatus; +import org.apache.flink.table.catalog.FunctionCatalog; +import org.apache.flink.table.catalog.GenericInMemoryCatalog; import org.apache.flink.table.catalog.IntervalFreshness; import org.apache.flink.table.catalog.ObjectIdentifier; +import org.apache.flink.table.catalog.ObjectPath; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.catalog.exceptions.TableNotExistException; +import org.apache.flink.table.delegation.Executor; +import org.apache.flink.table.delegation.Planner; +import org.apache.flink.table.delegation.materializedtable.MaterializedTableJobSubmitter; +import org.apache.flink.table.delegation.materializedtable.RefreshJobResult; +import org.apache.flink.table.delegation.materializedtable.RefreshJobTarget; +import org.apache.flink.table.delegation.materializedtable.RefreshWorkflowContext; +import org.apache.flink.table.factories.PlannerFactoryUtil; +import org.apache.flink.table.module.ModuleManager; +import org.apache.flink.table.refresh.ContinuousRefreshHandler; +import org.apache.flink.table.refresh.ContinuousRefreshHandlerSerializer; +import org.apache.flink.table.resource.ResourceManager; +import org.apache.flink.table.utils.CatalogManagerMocks; +import org.apache.flink.util.concurrent.FutureUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -27,18 +62,37 @@ import javax.annotation.Nullable; +import java.net.URL; import java.time.ZoneId; +import java.util.ArrayDeque; +import java.util.Arrays; import java.util.Collections; +import java.util.Deque; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; -/** Test for {@link MaterializedTableManager}. */ -class MaterializedTableManagerTest { +/** Test for {@link DefaultMaterializedTableExecutor}. */ +class DefaultMaterializedTableExecutorTest { + + private static final String CATALOG = "mt_cat"; + private static final String DATABASE = "mt_db"; + private static final String SAVEPOINT_DIRECTORY = "file:///savepoints"; + + private static final String CREATE_MATERIALIZED_TABLE = + "CREATE MATERIALIZED TABLE my_mt WITH ('connector' = 'values')" + + "\n FRESHNESS = INTERVAL '30' SECOND" + + "\n REFRESH_MODE = CONTINUOUS" + + "\n AS SELECT k, v FROM datagen_source"; + + private final FailingCatalog catalog = new FailingCatalog(CATALOG, DATABASE); @Test void testGetManuallyRefreshStatement() { @@ -46,7 +100,7 @@ void testGetManuallyRefreshStatement() { ObjectIdentifier.of("catalog", "database", "my_materialized_table"); String query = "SELECT * FROM my_source_table"; assertThat( - MaterializedTableManager.getRefreshStatement( + DefaultMaterializedTableExecutor.getRefreshStatement( tableIdentifier, query, Collections.emptyMap(), @@ -59,7 +113,7 @@ void testGetManuallyRefreshStatement() { partitionSpec.put("k1", "v1"); partitionSpec.put("k2", "v2"); assertThat( - MaterializedTableManager.getRefreshStatement( + DefaultMaterializedTableExecutor.getRefreshStatement( tableIdentifier, query, partitionSpec, Collections.emptyMap())) .isEqualTo( "INSERT OVERWRITE `catalog`.`database`.`my_materialized_table`\n" @@ -69,7 +123,6 @@ void testGetManuallyRefreshStatement() { @Test void testGenerateInsertStatement() { - // Test generate insert crate statement ObjectIdentifier materializedTableIdentifier = ObjectIdentifier.of("catalog", "database", "table"); String definitionQuery = "SELECT * FROM source_table"; @@ -77,7 +130,7 @@ void testGenerateInsertStatement() { "INSERT INTO `catalog`.`database`.`table`\n" + "SELECT * FROM source_table"; String actualStatement = - MaterializedTableManager.getInsertStatement( + DefaultMaterializedTableExecutor.getInsertStatement( materializedTableIdentifier, definitionQuery, Collections.emptyMap()); assertThat(actualStatement).isEqualTo(expectedStatement); @@ -98,7 +151,7 @@ void testGenerateInsertStatementWithDynamicOptions() { + "SELECT * FROM source_table"; String actualStatement = - MaterializedTableManager.getInsertStatement( + DefaultMaterializedTableExecutor.getInsertStatement( materializedTableIdentifier, definitionQuery, dynamicOptions); assertThat(actualStatement).isEqualTo(expectedStatement); } @@ -110,7 +163,7 @@ void testGetPeriodRefreshPartition(TestSpec testSpec) { if (testSpec.errorMessage == null) { Map actualRefreshPartition = - MaterializedTableManager.getPeriodRefreshPartition( + DefaultMaterializedTableExecutor.getPeriodRefreshPartition( testSpec.schedulerTime, testSpec.freshness, objectIdentifier, @@ -121,7 +174,7 @@ void testGetPeriodRefreshPartition(TestSpec testSpec) { } else { assertThatThrownBy( () -> - MaterializedTableManager.getPeriodRefreshPartition( + DefaultMaterializedTableExecutor.getPeriodRefreshPartition( testSpec.schedulerTime, testSpec.freshness, objectIdentifier, @@ -408,6 +461,205 @@ static Stream testData() { "Failed to parse a valid partition value for the field 'day' in materialized table `catalog`.`database`.`table` using the scheduler time '2024-01-01' based on the date format 'yyyy-MM-dd HH:mm:ss'.")); } + @Test + void resumeDoesNotRestartARefreshJobThatIsOnlyLocallySuspended() { + final TestingJobClient suspendedJob = + new TestingJobClient(new JobID(), JobStatus.SUSPENDED); + final ScriptedJobSubmitter jobSubmitter = new ScriptedJobSubmitter(suspendedJob); + final TableEnvironment tableEnv = createTableEnvironment(jobSubmitter); + tableEnv.executeSql(CREATE_MATERIALIZED_TABLE); + + assertThatThrownBy(() -> tableEnv.executeSql("ALTER MATERIALIZED TABLE my_mt RESUME")) + .isInstanceOf(TableException.class) + .hasMessageContaining( + "continuous refresh job has been resumed, jobId is " + + suspendedJob.getJobID()); + assertThat(jobSubmitter.submissions).isEqualTo(1); + } + + @Test + void createReportsARefreshJobItCouldNeitherRecordNorCancel() throws Exception { + final JobID jobId = new JobID(); + final TestingJobClient uncancellableJob = + new TestingJobClient(jobId, JobStatus.RUNNING) + .withCancelRequest( + () -> + FutureUtils.completedExceptionally( + new RuntimeException("injected cancel failure"))); + final TableEnvironment tableEnv = + createTableEnvironment(new ScriptedJobSubmitter(uncancellableJob)); + catalog.failNextAlterTable(hasRefreshStatus(RefreshStatus.ACTIVATED)); + + final Throwable createFailure = + catchThrowable(() -> tableEnv.executeSql(CREATE_MATERIALIZED_TABLE)); + + assertThat(createFailure) + .isInstanceOf(TableException.class) + .hasMessageContaining("Failed to submit continuous refresh job") + .hasRootCauseMessage("injected cancel failure"); + final Throwable unrecordedJobFailure = createFailure.getCause(); + assertThat(unrecordedJobFailure) + .hasMessageContaining(jobId.toHexString()) + .hasMessageContaining("and cancelling it failed as well") + .hasMessageContaining("must be cancelled manually"); + assertThat(unrecordedJobFailure.getSuppressed()) + .singleElement() + .satisfies( + catalogFailure -> + assertThat(catalogFailure) + .hasRootCauseMessage("injected alter failure")); + assertThat(catalog.tableExists(new ObjectPath(DATABASE, "my_mt"))).isFalse(); + } + + @Test + void createRecordsTheClusterIdOfAKubernetesSessionTarget() throws Exception { + final TableEnvironment tableEnv = + createTableEnvironment( + new ScriptedJobSubmitter( + new TestingJobClient(new JobID(), JobStatus.RUNNING))); + tableEnv.getConfig().set("execution.target", "kubernetes-session"); + tableEnv.getConfig().set("kubernetes.cluster-id", "c1"); + + tableEnv.executeSql(CREATE_MATERIALIZED_TABLE); + + final ContinuousRefreshHandler refreshHandler = + getRefreshHandler(catalog, new ObjectPath(DATABASE, "my_mt")); + assertThat(refreshHandler.getExecutionTarget()).isEqualTo("kubernetes-session"); + assertThat(refreshHandler.getClusterId()).isEqualTo("c1"); + } + + private TableEnvironment createTableEnvironment(ScriptedJobSubmitter jobSubmitter) { + final Configuration rootConfiguration = new Configuration(); + rootConfiguration.set(DeploymentOptions.TARGET, "remote"); + rootConfiguration.set(CheckpointingOptions.SAVEPOINT_DIRECTORY, SAVEPOINT_DIRECTORY); + + final StreamExecutionEnvironment executionEnvironment = + new StreamExecutionEnvironment(rootConfiguration); + final ResourceManager resourceManager = + ResourceManager.createResourceManager( + new URL[0], getClass().getClassLoader(), rootConfiguration); + final ClassLoader userClassLoader = resourceManager.getUserClassLoader(); + final Executor executor = + AbstractStreamTableEnvironmentImpl.lookupExecutor( + userClassLoader, executionEnvironment); + + final TableConfig tableConfig = TableConfig.getDefault(); + tableConfig.setRootConfiguration(executor.getConfiguration()); + jobSubmitter.tableConfig = tableConfig; + + final ModuleManager moduleManager = new ModuleManager(); + final CatalogManager catalogManager = + CatalogManagerMocks.preparedCatalogManager() + .classLoader(userClassLoader) + .config(tableConfig) + .defaultCatalog(CATALOG, catalog) + .executionConfig(executionEnvironment.getConfig()) + .build(); + final FunctionCatalog functionCatalog = + new FunctionCatalog(tableConfig, resourceManager, catalogManager, moduleManager); + final Planner planner = + PlannerFactoryUtil.createPlanner( + executor, + tableConfig, + userClassLoader, + moduleManager, + catalogManager, + functionCatalog); + + final TableEnvironment tableEnv = + new StreamTableEnvironmentImpl( + catalogManager, + moduleManager, + resourceManager, + functionCatalog, + tableConfig, + executionEnvironment, + planner, + executor, + true, + new DefaultMaterializedTableExecutorFactory(), + jobSubmitter); + tableEnv.executeSql( + "CREATE TABLE datagen_source (k INT, v BIGINT) WITH ('connector' = 'datagen')"); + return tableEnv; + } + + private static ContinuousRefreshHandler getRefreshHandler(Catalog catalog, ObjectPath tablePath) + throws Exception { + return ContinuousRefreshHandlerSerializer.INSTANCE.deserialize( + ((CatalogMaterializedTable) catalog.getTable(tablePath)) + .getSerializedRefreshHandler(), + DefaultMaterializedTableExecutorTest.class.getClassLoader()); + } + + private static Predicate hasRefreshStatus(RefreshStatus refreshStatus) { + return table -> + table instanceof CatalogMaterializedTable + && ((CatalogMaterializedTable) table).getRefreshStatus() == refreshStatus; + } + + /** Fails a selected table alteration once, so tests can observe a rejected catalog write. */ + private static final class FailingCatalog extends GenericInMemoryCatalog { + + @Nullable private Predicate failingAlterMatcher; + + private FailingCatalog(String name, String defaultDatabase) { + super(name, defaultDatabase); + } + + /** The next {@code alterTable} whose new table matches fails; the hook then clears. */ + private void failNextAlterTable(Predicate newTableMatches) { + this.failingAlterMatcher = newTableMatches; + } + + @Override + public void alterTable( + ObjectPath tablePath, CatalogBaseTable newTable, boolean ignoreIfNotExists) + throws TableNotExistException { + if (failingAlterMatcher != null && failingAlterMatcher.test(newTable)) { + failingAlterMatcher = null; + throw new CatalogException("injected alter failure"); + } + super.alterTable(tablePath, newTable, ignoreIfNotExists); + } + } + + private static final class ScriptedJobSubmitter implements MaterializedTableJobSubmitter { + + private final Deque jobClients; + private int submissions; + private TableConfig tableConfig; + + private ScriptedJobSubmitter(JobClient... jobClients) { + this.jobClients = new ArrayDeque<>(Arrays.asList(jobClients)); + } + + @Override + public RefreshJobResult submitRefreshJob( + Configuration executionConfig, String insertStatement) { + submissions++; + final JobClient jobClient = jobClients.poll(); + if (jobClient == null) { + throw new IllegalStateException( + "Unexpected refresh job submission: " + insertStatement); + } + return new RefreshJobResult( + RefreshJobTarget.forNewJob(tableConfig), + jobClient.getJobID().toHexString(), + jobClient); + } + + @Override + public boolean supportsApplicationTargets() { + return false; + } + + @Override + public Optional getRefreshWorkflowContext() { + return Optional.empty(); + } + } + private static class TestSpec { private String schedulerTime; private IntervalFreshness freshness; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/delegation/materializedtable/RefreshJobControllerTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/delegation/materializedtable/RefreshJobControllerTest.java new file mode 100644 index 00000000000000..4e767ee3479435 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/delegation/materializedtable/RefreshJobControllerTest.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.delegation.materializedtable; + +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.JobStatus; +import org.apache.flink.api.dag.Pipeline; +import org.apache.flink.client.deployment.StandaloneClusterId; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.core.execution.JobRetrievalSupportedPipelineExecutor; +import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.api.TableException; +import org.apache.flink.table.refresh.ContinuousRefreshHandler; + +import org.junit.jupiter.api.Test; + +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link RefreshJobController}. */ +class RefreshJobControllerTest { + + private final JobID jobId = new JobID(); + private final TableConfig tableConfig = TableConfig.getDefault(); + + @Test + void interruptedWaitRestoresTheInterruptFlag() { + Thread.currentThread().interrupt(); + try { + assertThatThrownBy( + () -> + controllerReaching( + new TestingJobClient( + jobId, new CompletableFuture<>())) + .getJobStatus(handler())) + .isInstanceOf(TableException.class) + .hasCauseInstanceOf(InterruptedException.class); + assertThat(Thread.currentThread().isInterrupted()).isTrue(); + } finally { + Thread.interrupted(); + } + } + + @Test + void stopThatWasNeverSentIsNotReportedAsPending() { + final RuntimeException unreachable = + new RuntimeException("Couldn't retrieve standalone cluster"); + final JobClient jobClient = + new TestingJobClient(jobId, JobStatus.RUNNING) + .withStopRequest( + savepointDirectory -> { + throw unreachable; + }); + + assertThatThrownBy( + () -> + controllerReaching(jobClient) + .stopWithSavepoint(handler(), "file:///savepoints/mt")) + .isInstanceOf(TableException.class) + .hasCauseReference(unreachable) + .hasMessageContaining("'rest.address'") + .hasMessageNotContaining("may still complete") + .hasMessageNotContaining("'execution.state-recovery.path'"); + } + + @Test + void retrievedJobClientIsReusedWithinOneOperationAndClosedAtItsEnd() { + final CloseableTestingJobClient jobClient = + new CloseableTestingJobClient(jobId, JobStatus.RUNNING); + final CountingRetriever retriever = new CountingRetriever(jobClient); + final RefreshJobController controller = controller(retriever); + + controller.getJobStatus(handler()); + controller.cancel(handler()); + controller.getJobStatus(handler()); + + assertThat(retriever.retrievals).isEqualTo(1); + assertThat(jobClient.isCancelRequested()).isTrue(); + assertThat(jobClient.closed).isFalse(); + + controller.releaseRetrievedJobClients(); + + assertThat(jobClient.closed).isTrue(); + controller.getJobStatus(handler()); + assertThat(retriever.retrievals).isEqualTo(2); + } + + private RefreshJobController controller(CountingRetriever retriever) { + return new RefreshJobController( + tableConfig, getClass().getClassLoader(), configuration -> Optional.of(retriever)); + } + + private RefreshJobController controllerReaching(JobClient jobClient) { + return controller(new CountingRetriever(jobClient)); + } + + private ContinuousRefreshHandler handler() { + return new ContinuousRefreshHandler( + "remote", StandaloneClusterId.getInstance().toString(), jobId.toHexString()); + } + + private static final class CountingRetriever implements JobRetrievalSupportedPipelineExecutor { + + private final JobClient jobClient; + private int retrievals; + + private CountingRetriever(JobClient jobClient) { + this.jobClient = jobClient; + } + + @Override + public CompletableFuture execute( + Pipeline pipeline, Configuration configuration, ClassLoader userCodeClassloader) { + throw new UnsupportedOperationException(); + } + + @Override + public CompletableFuture retrieveJobClient( + JobID jobId, Configuration configuration, ClassLoader userCodeClassloader) { + retrievals++; + return CompletableFuture.completedFuture(jobClient); + } + } + + private static final class CloseableTestingJobClient extends TestingJobClient + implements AutoCloseable { + + private boolean closed; + + private CloseableTestingJobClient(JobID jobId, JobStatus jobStatus) { + super(jobId, jobStatus); + } + + @Override + public void close() { + closed = true; + } + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/delegation/materializedtable/TestingJobClient.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/delegation/materializedtable/TestingJobClient.java new file mode 100644 index 00000000000000..b27f563732528e --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/delegation/materializedtable/TestingJobClient.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.delegation.materializedtable; + +import org.apache.flink.api.common.JobExecutionResult; +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.JobStatus; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.core.execution.SavepointFormatType; + +import javax.annotation.Nullable; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import java.util.function.Supplier; + +/** A {@link JobClient} with a controllable status. */ +class TestingJobClient implements JobClient { + + private final JobID jobId; + private final Supplier> jobStatusRequest; + private Supplier> cancelRequest = + () -> CompletableFuture.completedFuture(null); + private Function> stopRequest = + savepointDirectory -> + CompletableFuture.completedFuture(savepointDirectory + "/savepoint-1"); + private boolean cancelRequested; + + TestingJobClient(JobID jobId, JobStatus jobStatus) { + this(jobId, CompletableFuture.completedFuture(jobStatus)); + } + + TestingJobClient(JobID jobId, CompletableFuture jobStatus) { + this.jobId = jobId; + this.jobStatusRequest = () -> jobStatus; + } + + TestingJobClient withCancelRequest(Supplier> cancelRequest) { + this.cancelRequest = cancelRequest; + return this; + } + + TestingJobClient withStopRequest(Function> stopRequest) { + this.stopRequest = stopRequest; + return this; + } + + boolean isCancelRequested() { + return cancelRequested; + } + + @Override + public JobID getJobID() { + return jobId; + } + + @Override + public CompletableFuture getJobStatus() { + return jobStatusRequest.get(); + } + + @Override + public CompletableFuture cancel() { + cancelRequested = true; + return cancelRequest.get(); + } + + @Override + public CompletableFuture stopWithSavepoint( + boolean advanceToEndOfEventTime, + @Nullable String savepointDirectory, + SavepointFormatType formatType) { + return stopRequest.apply(savepointDirectory); + } + + @Override + public CompletableFuture triggerSavepoint( + @Nullable String savepointDirectory, SavepointFormatType formatType) { + throw new UnsupportedOperationException(); + } + + @Override + public CompletableFuture> getAccumulators() { + throw new UnsupportedOperationException(); + } + + @Override + public CompletableFuture getJobExecutionResult() { + throw new UnsupportedOperationException(); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/ExplainTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/ExplainTest.java index e6b4024fae91f7..a4455b12a41881 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/ExplainTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/stream/sql/ExplainTest.java @@ -21,6 +21,8 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.table.api.TableConfig; import org.apache.flink.table.api.config.TableConfigOptions; +import org.apache.flink.table.api.internal.TableEnvironmentInternal; +import org.apache.flink.table.operations.materializedtable.CreateMaterializedTableOperation; import org.apache.flink.table.planner.utils.StreamTableTestUtil; import org.apache.flink.table.planner.utils.TableTestBase; import org.apache.flink.table.planner.utils.TableTestUtil; @@ -97,17 +99,16 @@ void testExplainCreateOrAlterMaterializedTable() { @Test void testExplainAlterMaterializedTable() { - util.getTableEnv() - .executeSql( - "CREATE OR ALTER MATERIALIZED TABLE MyMTTable\n" - + " WITH (\n" - + " 'connector' = 'values'\n" - + ") AS\n" - + " SELECT\n" - + " `a`,\n" - + " `b`\n" - + " FROM\n" - + " MyTable"); + createMaterializedTableInCatalogOnly( + "CREATE OR ALTER MATERIALIZED TABLE MyMTTable\n" + + " WITH (\n" + + " 'connector' = 'values'\n" + + ") AS\n" + + " SELECT\n" + + " `a`,\n" + + " `b`\n" + + " FROM\n" + + " MyTable"); verifyExplain( "ALTER MATERIALIZED TABLE MyMTTable\n" + "AS\n" @@ -121,17 +122,16 @@ void testExplainAlterMaterializedTable() { @Test void testExplainFullAlterMaterializedTable() { - util.getTableEnv() - .executeSql( - "CREATE OR ALTER MATERIALIZED TABLE MyMTTable\n" - + " WITH (\n" - + " 'connector' = 'values'\n" - + ") AS\n" - + " SELECT\n" - + " `a`,\n" - + " `b`\n" - + " FROM\n" - + " MyTable"); + createMaterializedTableInCatalogOnly( + "CREATE OR ALTER MATERIALIZED TABLE MyMTTable\n" + + " WITH (\n" + + " 'connector' = 'values'\n" + + ") AS\n" + + " SELECT\n" + + " `a`,\n" + + " `b`\n" + + " FROM\n" + + " MyTable"); verifyExplain( "CREATE OR ALTER MATERIALIZED TABLE MyMTTable(\n" + " `b`,\n" @@ -250,6 +250,18 @@ void testExplainReplaceTableAsSelectWithColumnsInCreateAndQueryParts() { "testExplainCtasWithColumnsInCreateAndQueryParts"); } + private void createMaterializedTableInCatalogOnly(String createStatement) { + TableEnvironmentInternal tableEnv = (TableEnvironmentInternal) util.getTableEnv(); + CreateMaterializedTableOperation operation = + (CreateMaterializedTableOperation) + tableEnv.getParser().parse(createStatement).get(0); + tableEnv.getCatalogManager() + .createTable( + operation.getCatalogMaterializedTable(), + operation.getTableIdentifier(), + false); + } + private void verifyExplain(final String statement) { final String displayName = this.testInfo.getDisplayName(); verifyExplain(statement, displayName.substring(0, displayName.length() - 2)); diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/stream/sql/MaterializedTableTableApiITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/stream/sql/MaterializedTableTableApiITCase.java new file mode 100644 index 00000000000000..0b9058b82c730f --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/stream/sql/MaterializedTableTableApiITCase.java @@ -0,0 +1,851 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.runtime.stream.sql; + +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.JobStatus; +import org.apache.flink.client.program.rest.RestClusterClient; +import org.apache.flink.configuration.CheckpointingOptions; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.DeploymentOptions; +import org.apache.flink.configuration.RestOptions; +import org.apache.flink.core.testutils.CommonTestUtils; +import org.apache.flink.runtime.client.JobStatusMessage; +import org.apache.flink.runtime.jobgraph.JobType; +import org.apache.flink.runtime.rest.messages.EmptyRequestBody; +import org.apache.flink.runtime.rest.messages.JobMessageParameters; +import org.apache.flink.runtime.rest.messages.MessageHeaders; +import org.apache.flink.runtime.rest.messages.ResponseBody; +import org.apache.flink.runtime.rest.messages.checkpoints.CheckpointConfigHeaders; +import org.apache.flink.runtime.rest.messages.checkpoints.CheckpointConfigInfo; +import org.apache.flink.runtime.rest.messages.checkpoints.CheckpointingStatistics.RestoredCheckpointStatistics; +import org.apache.flink.runtime.rest.messages.checkpoints.CheckpointingStatisticsHeaders; +import org.apache.flink.runtime.rest.util.RestMapperUtils; +import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.util.TestStreamEnvironment; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableException; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.table.api.internal.TableEnvironmentInternal; +import org.apache.flink.table.catalog.CatalogBaseTable.TableKind; +import org.apache.flink.table.catalog.CatalogMaterializedTable.LogicalRefreshMode; +import org.apache.flink.table.catalog.CatalogMaterializedTable.RefreshMode; +import org.apache.flink.table.catalog.CatalogMaterializedTable.RefreshStatus; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.GenericInMemoryCatalog; +import org.apache.flink.table.catalog.IntervalFreshness; +import org.apache.flink.table.catalog.ObjectIdentifier; +import org.apache.flink.table.catalog.ResolvedCatalogMaterializedTable; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.planner.factories.TestValuesTableFactory; +import org.apache.flink.table.refresh.ContinuousRefreshHandler; +import org.apache.flink.table.refresh.ContinuousRefreshHandlerSerializer; +import org.apache.flink.test.junit5.InjectClusterClient; +import org.apache.flink.test.junit5.MiniClusterExtension; +import org.apache.flink.types.Row; +import org.apache.flink.util.CollectionUtil; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import javax.annotation.Nullable; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.flink.test.util.TestUtils.waitUntilAllTasksAreRunning; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** IT case for materialized table related statement via {@link TableEnvironment}. */ +class MaterializedTableTableApiITCase { + + private static final String CATALOG = "test_catalog"; + private static final String DATABASE = "test_db"; + private static final ObjectIdentifier USERS_SHOPS = + ObjectIdentifier.of(CATALOG, DATABASE, "users_shops"); + + private static final Duration TIMEOUT = Duration.ofSeconds(60); + private static final Duration PAUSE = Duration.ofMillis(200); + + private static final ConfigOption KUBERNETES_CLUSTER_ID = + ConfigOptions.key("kubernetes.cluster-id").stringType().noDefaultValue(); + + private static final String UPDATING_VALUES_SINK = + " WITH(\n" + + " 'connector' = 'values',\n" + + " 'sink-insert-only' = 'false'\n" + + " )\n"; + + private static final String USERS_SHOPS_QUERY = + " AS SELECT \n" + + " user_id,\n" + + " shop_id,\n" + + " ds,\n" + + " SUM (payment_amount_cents) AS payed_buy_fee_sum,\n" + + " SUM (1) AS pv\n" + + " FROM (\n" + + " SELECT user_id, shop_id, DATE_FORMAT(order_created_at, 'yyyy-MM-dd') AS ds, payment_amount_cents FROM datagenSource" + + " ) AS tmp\n" + + " GROUP BY (user_id, shop_id, ds)"; + + private static final String USERS_SHOPS_DDL = + "CREATE MATERIALIZED TABLE users_shops" + + " PARTITIONED BY (ds)\n" + + UPDATING_VALUES_SINK + + " FRESHNESS = INTERVAL '30' SECOND\n" + + USERS_SHOPS_QUERY; + + private static final ResolvedSchema USERS_SHOPS_SCHEMA = + ResolvedSchema.of( + Arrays.asList( + Column.physical("user_id", DataTypes.BIGINT()), + Column.physical("shop_id", DataTypes.BIGINT()), + Column.physical("ds", DataTypes.STRING()), + Column.physical("payed_buy_fee_sum", DataTypes.BIGINT()), + Column.physical("pv", DataTypes.INT().notNull()))); + + @RegisterExtension + private static final MiniClusterExtension MINI_CLUSTER = + new MiniClusterExtension( + new MiniClusterResourceConfiguration.Builder() + .setNumberTaskManagers(2) + .setNumberSlotsPerTaskManager(4) + .build()); + + private StreamTableEnvironment tEnv; + private GenericInMemoryCatalog catalog; + private RestClusterClient clusterClient; + + @BeforeEach + void before(@InjectClusterClient RestClusterClient injectedClusterClient) { + TestStreamEnvironment.unsetAsContext(); + clusterClient = injectedClusterClient; + + final StreamExecutionEnvironment executionEnvironment = + StreamExecutionEnvironment.getExecutionEnvironment( + new Configuration(MINI_CLUSTER.getClientConfiguration())); + executionEnvironment.setParallelism(1); + tEnv = StreamTableEnvironment.create(executionEnvironment); + tEnv.getConfig().set(DeploymentOptions.TARGET, "remote"); + tEnv.getConfig() + .set( + RestOptions.ADDRESS, + MINI_CLUSTER.getClientConfiguration().get(RestOptions.ADDRESS)); + + catalog = new GenericInMemoryCatalog(CATALOG, DATABASE); + tEnv.registerCatalog(CATALOG, catalog); + tEnv.useCatalog(CATALOG); + tEnv.useDatabase(DATABASE); + + tEnv.executeSql( + "CREATE TABLE datagenSource (\n" + + " order_id BIGINT,\n" + + " order_number VARCHAR(20),\n" + + " user_id BIGINT,\n" + + " shop_id BIGINT,\n" + + " product_id BIGINT,\n" + + " status BIGINT,\n" + + " order_type BIGINT,\n" + + " order_created_at TIMESTAMP,\n" + + " payment_amount_cents BIGINT\n" + + ")\n" + + "WITH (\n" + + " 'connector' = 'datagen',\n" + + " 'rows-per-second' = '10'\n" + + ")"); + } + + @AfterEach + void after() throws Exception { + for (JobStatusMessage job : + clusterClient.listJobs().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)) { + if (!job.getJobState().isTerminalState()) { + cancelUnlessEnded(job.getJobId()); + } + } + TestValuesTableFactory.clearAllData(); + } + + private void cancelUnlessEnded(JobID jobId) throws Exception { + try { + clusterClient.cancel(jobId).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } catch (Exception cancelFailure) { + if (!getJobStatus(jobId).isGloballyTerminalState()) { + throw cancelFailure; + } + } + } + + @Test + void testCreateMaterializedTableInContinuousMode() throws Exception { + tEnv.executeSql(USERS_SHOPS_DDL); + + final ResolvedCatalogMaterializedTable actualMaterializedTable = getUsersShops(); + assertThat(actualMaterializedTable.getResolvedSchema()).isEqualTo(USERS_SHOPS_SCHEMA); + assertThat(actualMaterializedTable.getLogicalRefreshMode()) + .isSameAs(LogicalRefreshMode.AUTOMATIC); + assertThat(actualMaterializedTable.getRefreshMode()).isSameAs(RefreshMode.CONTINUOUS); + assertThat(actualMaterializedTable.getRefreshStatus()).isSameAs(RefreshStatus.ACTIVATED); + assertThat(actualMaterializedTable.getRefreshHandlerDescription()).isNotEmpty(); + assertThat(actualMaterializedTable.getSerializedRefreshHandler()).isNotEmpty(); + + final JobID refreshJobId = getRefreshJobId(); + waitUntilAllTasksAreRunning(clusterClient, refreshJobId); + assertThat(getJobStatus(refreshJobId)).isEqualTo(JobStatus.RUNNING); + assertThat(getCheckpointInterval(refreshJobId)).isEqualTo(30 * 1000); + } + + @Test + void testCreateMaterializedTableInContinuousModeWithoutFreshnessAndRefreshMode() + throws Exception { + tEnv.executeSql( + "CREATE MATERIALIZED TABLE users_shops" + + " PARTITIONED BY (ds)\n" + + UPDATING_VALUES_SINK + + USERS_SHOPS_QUERY); + + final ResolvedCatalogMaterializedTable actualMaterializedTable = getUsersShops(); + assertThat(actualMaterializedTable.getResolvedSchema()).isEqualTo(USERS_SHOPS_SCHEMA); + assertThat(actualMaterializedTable.getDefinitionFreshness()) + .isEqualTo(IntervalFreshness.ofMinute(3)); + assertThat(actualMaterializedTable.getLogicalRefreshMode()) + .isSameAs(LogicalRefreshMode.AUTOMATIC); + assertThat(actualMaterializedTable.getRefreshMode()).isSameAs(RefreshMode.CONTINUOUS); + assertThat(actualMaterializedTable.getRefreshStatus()).isSameAs(RefreshStatus.ACTIVATED); + assertThat(actualMaterializedTable.getRefreshHandlerDescription()).isNotEmpty(); + assertThat(actualMaterializedTable.getSerializedRefreshHandler()).isNotEmpty(); + + final JobID refreshJobId = getRefreshJobId(); + waitUntilAllTasksAreRunning(clusterClient, refreshJobId); + assertThat(getJobStatus(refreshJobId)).isEqualTo(JobStatus.RUNNING); + // default checkpoint interval is 3 minutes + assertThat(getCheckpointInterval(refreshJobId)).isEqualTo(60 * 3 * 1000); + } + + @Test + void testCreateMaterializedTableInContinuousModeWithCustomCheckpointInterval() + throws Exception { + final long checkpointInterval = 60 * 1000; + tEnv.getConfig() + .set( + CheckpointingOptions.CHECKPOINTING_INTERVAL, + Duration.ofMillis(checkpointInterval)); + + tEnv.executeSql(USERS_SHOPS_DDL); + + final JobID refreshJobId = getRefreshJobId(); + waitUntilAllTasksAreRunning(clusterClient, refreshJobId); + assertThat(getCheckpointInterval(refreshJobId)).isEqualTo(checkpointInterval); + } + + @Test + void testCreateMaterializedTableFailedInContinuousMode() { + // an insert-only sink cannot consume the updates of the aggregation + assertThatThrownBy( + () -> + tEnv.executeSql( + "CREATE MATERIALIZED TABLE users_shops" + + " PARTITIONED BY (ds)\n" + + " WITH(\n" + + " 'connector' = 'values',\n" + + " 'sink-insert-only' = 'true'\n" + + " )\n" + + " FRESHNESS = INTERVAL '30' SECOND\n" + + USERS_SHOPS_QUERY)) + .isInstanceOf(TableException.class) + .hasMessageContaining( + String.format( + "Failed to submit continuous refresh job for materialized table %s.", + USERS_SHOPS.asSerializableString())); + + assertThat(catalog.tableExists(USERS_SHOPS.toObjectPath())).isFalse(); + } + + @Test + void testAlterMaterializedTableRefresh() throws Exception { + final List data = new ArrayList<>(); + data.add(Row.of(1L, 1L, 1L, "2024-01-01")); + data.add(Row.of(2L, 2L, 2L, "2024-01-02")); + data.add(Row.of(3L, 3L, 3L, "2024-01-02")); + createBoundedValuesSource(data); + + tEnv.executeSql( + "CREATE MATERIALIZED TABLE users_shops" + + " PARTITIONED BY (ds)\n" + + UPDATING_VALUES_SINK + + " FRESHNESS = INTERVAL '30' SECOND\n" + + " REFRESH_MODE = CONTINUOUS\n" + + " AS SELECT \n" + + " user_id,\n" + + " shop_id,\n" + + " ds,\n" + + " COUNT(order_id) AS order_cnt\n" + + " FROM (\n" + + " SELECT user_id, shop_id, order_created_at AS ds, order_id FROM my_source" + + " ) AS tmp\n" + + " GROUP BY (user_id, shop_id, ds)"); + CommonTestUtils.waitUtil( + () -> getUsersShopsResults().size() == data.size(), + TIMEOUT, + PAUSE, + "Failed to verify the data in materialized table."); + + data.remove(2); + + final long currentTime = System.currentTimeMillis(); + final List result = + CollectionUtil.iteratorToList( + tEnv.executeSql( + "ALTER MATERIALIZED TABLE users_shops REFRESH PARTITION (ds = '2024-01-02')") + .collect()); + assertThat(result).hasSize(1); + final JobID refreshJobId = JobID.fromHexString(result.get(0).getFieldAs(0)); + + verifyRefreshJobCreated(refreshJobId, currentTime); + + assertThat(getUsersShopsResults()).containsExactly("+I[2, 2, 2024-01-02, 1]"); + } + + @Test + void testAlterMaterializedTableRefreshWithInvalidPartitionSpec() { + tEnv.executeSql( + "CREATE MATERIALIZED TABLE users_shops" + + " PARTITIONED BY (ds1, ds2)\n" + + UPDATING_VALUES_SINK + + " FRESHNESS = INTERVAL '30' SECOND\n" + + " AS SELECT \n" + + " user_id,\n" + + " shop_id,\n" + + " ds1,\n" + + " ds2,\n" + + " SUM (payment_amount_cents) AS payed_buy_fee_sum,\n" + + " SUM (1) AS pv\n" + + " FROM (\n" + + " SELECT user_id, shop_id, DATE_FORMAT(order_created_at, 'yyyy-MM-dd') AS ds1, user_id % 10 as ds2, payment_amount_cents FROM datagenSource" + + " ) AS tmp\n" + + " GROUP BY (user_id, shop_id, ds1, ds2)"); + + // CASE 1: check unknown partition keys + assertThatThrownBy( + () -> + tEnv.executeSql( + "ALTER MATERIALIZED TABLE users_shops REFRESH PARTITION (ds3 = '2024-01-01')")) + .isInstanceOf(ValidationException.class) + .hasMessage( + "The partition spec contains unknown partition keys:\n" + + "\n" + + "ds3\n" + + "\n" + + "All known partition keys are:\n" + + "\n" + + "ds2\n" + + "ds1"); + + // CASE 2: check specific non-string partition keys as partition spec to refresh + assertThatThrownBy( + () -> + tEnv.executeSql( + "ALTER MATERIALIZED TABLE users_shops REFRESH PARTITION (ds2 = 5)")) + .isInstanceOf(ValidationException.class) + .hasMessage( + "Currently, refreshing materialized table only supports referring to char, varchar and string type partition keys. All specified partition keys in partition specs with unsupported types are:\n" + + "\n" + + "ds2"); + } + + @Test + void testAlterMaterializedTableSuspendAndResumeInContinuousMode(@TempDir Path temporaryPath) + throws Exception { + tEnv.executeSql(USERS_SHOPS_DDL); + + assertThat(getUsersShops().getRefreshStatus()).isSameAs(RefreshStatus.ACTIVATED); + waitUntilAllTasksAreRunning(clusterClient, getRefreshJobId()); + + setupSavepointDir(temporaryPath); + + tEnv.executeSql("ALTER MATERIALIZED TABLE users_shops SUSPEND"); + + assertThat(getUsersShops().getRefreshStatus()).isSameAs(RefreshStatus.SUSPENDED); + final ContinuousRefreshHandler suspendRefreshHandler = getRefreshHandler(); + assertThat(getJobStatus(JobID.fromHexString(suspendRefreshHandler.getJobId()))) + .isEqualTo(JobStatus.FINISHED); + assertThat(suspendRefreshHandler.getRestorePath()).isNotEmpty(); + final String actualSavepointPath = suspendRefreshHandler.getRestorePath().get(); + + tEnv.executeSql( + "ALTER MATERIALIZED TABLE users_shops RESUME WITH ('sink.drop-late-event' = 'false')"); + + final ResolvedCatalogMaterializedTable resumedMaterializedTable = getUsersShops(); + assertThat(resumedMaterializedTable.getOptions()).doesNotContainKey("sink.drop-late-event"); + assertThat(resumedMaterializedTable.getRefreshStatus()).isSameAs(RefreshStatus.ACTIVATED); + + final JobID resumeJobId = getRefreshJobId(); + waitUntilAllTasksAreRunning(clusterClient, resumeJobId); + assertThat(getJobStatus(resumeJobId)).isEqualTo(JobStatus.RUNNING); + assertThat(getJobRestoreSavepointPath(resumeJobId)).hasValue(actualSavepointPath); + } + + @Test + void testAlterMaterializedTableWithoutSavepointDirConfiguredInContinuousMode() + throws Exception { + tEnv.executeSql(USERS_SHOPS_DDL); + waitUntilAllTasksAreRunning(clusterClient, getRefreshJobId()); + + assertThatThrownBy(() -> tEnv.executeSql("ALTER MATERIALIZED TABLE users_shops SUSPEND")) + .rootCause() + .isInstanceOf(ValidationException.class) + .hasMessageContaining( + "Savepoint directory is not configured, can't stop job with savepoint."); + } + + @Test + void testAlterMaterializedTableWithRepeatedSuspendAndResumeInContinuousMode( + @TempDir Path temporaryPath) throws Exception { + tEnv.executeSql(USERS_SHOPS_DDL); + waitUntilAllTasksAreRunning(clusterClient, getRefreshJobId()); + + setupSavepointDir(temporaryPath); + + tEnv.executeSql("ALTER MATERIALIZED TABLE users_shops SUSPEND"); + + assertThatThrownBy(() -> tEnv.executeSql("ALTER MATERIALIZED TABLE users_shops SUSPEND")) + .rootCause() + .isInstanceOf(TableException.class) + .hasMessageContaining( + String.format( + "Materialized table %s continuous refresh job has been suspended", + USERS_SHOPS)); + + tEnv.executeSql("ALTER MATERIALIZED TABLE users_shops RESUME"); + + assertThatThrownBy(() -> tEnv.executeSql("ALTER MATERIALIZED TABLE users_shops RESUME")) + .isInstanceOf(TableException.class) + .hasMessageContaining( + String.format( + "Materialized table %s continuous refresh job has been resumed", + USERS_SHOPS)); + } + + @Test + void testCreateOrAlterMaterializedTableColumnListReordersSchemaOnCreatePath() { + createBoundedValuesSource(List.of()); + + tEnv.executeSql( + "CREATE OR ALTER MATERIALIZED TABLE users_shops (shop_id, user_id, oca, order_id)" + + UPDATING_VALUES_SINK + + " AS SELECT \n" + + " user_id,\n" + + " shop_id,\n" + + " order_created_at AS oca,\n" + + " order_id" + + " FROM my_source"); + + assertThat(getUsersShops().getResolvedSchema().getColumnNames()) + .containsExactly("shop_id", "user_id", "oca", "order_id"); + } + + @Test + void testCreateMaterializedTableWithDistribution() { + assertThatThrownBy( + () -> + tEnv.executeSql( + "CREATE MATERIALIZED TABLE users_shops (" + + " PRIMARY KEY (shop_id, user_id) not enforced)" + + " DISTRIBUTED BY HASH (user_id) INTO 7 BUCKETS\n" + + " WITH(\n" + + " 'connector' = 'blackhole'\n" + + " )\n" + + " FRESHNESS = INTERVAL '30' SECOND\n" + + " AS SELECT 1 as shop_id, 2 as user_id")) + .isInstanceOf(TableException.class) + .hasRootCauseMessage( + String.format( + "Table '%s' is distributed into buckets," + + " but the underlying DynamicTableSink doesn't implement the SupportsBucketing interface.", + USERS_SHOPS.asSummaryString())); + } + + @Test + void testAlterMaterializedTableAsQueryInContinuousMode(@TempDir Path temporaryPath) + throws Exception { + tEnv.executeSql( + "CREATE MATERIALIZED TABLE users_shops (" + + " PRIMARY KEY (ds, user_id) not enforced)" + + " PARTITIONED BY (ds)\n" + + " WITH(\n" + + " 'connector' = 'values',\n" + + " 'sink-insert-only' = 'false',\n" + + " 'sink-changelog-mode-enforced' = 'I,UB,UA,D'\n" + + " )\n" + + " FRESHNESS = INTERVAL '30' SECOND\n" + + " AS SELECT \n" + + " coalesce(user_id, 0) as user_id,\n" + + " shop_id,\n" + + " coalesce(ds, '') as ds,\n" + + " SUM (payment_amount_cents) AS payed_buy_fee_sum\n" + + " FROM (\n" + + " SELECT user_id, shop_id, DATE_FORMAT(order_created_at, 'yyyy-MM-dd') AS ds, payment_amount_cents FROM datagenSource" + + " ) AS tmp\n" + + " GROUP BY (user_id, shop_id, ds)"); + final ResolvedCatalogMaterializedTable oldTable = getUsersShops(); + waitUntilAllTasksAreRunning(clusterClient, getRefreshJobId()); + + setupSavepointDir(temporaryPath); + + tEnv.executeSql( + "ALTER MATERIALIZED TABLE users_shops" + + " AS SELECT \n" + + " coalesce(user_id, 0) as user_id,\n" + + " shop_id,\n" + + " coalesce(ds, '') as ds,\n" + + " SUM (payment_amount_cents) AS payed_buy_fee_sum,\n" + + " SUM (1) AS pv\n" + + " FROM (\n" + + " SELECT user_id, shop_id, DATE_FORMAT(order_created_at, 'yyyy-MM-dd') AS ds, payment_amount_cents FROM datagenSource" + + " ) AS tmp\n" + + " GROUP BY (user_id, shop_id, ds)"); + final ResolvedCatalogMaterializedTable newTable = getUsersShops(); + + assertThat(getAddedColumns(newTable.getResolvedSchema(), oldTable.getResolvedSchema())) + .containsExactly(Column.physical("pv", DataTypes.INT())); + assertThat(newTable.getResolvedSchema().getPrimaryKey()) + .isEqualTo(oldTable.getResolvedSchema().getPrimaryKey()); + assertThat(newTable.getResolvedSchema().getWatermarkSpecs()) + .isEqualTo(oldTable.getResolvedSchema().getWatermarkSpecs()); + assertThat(newTable.getExpandedQuery()) + .hasToString( + String.format( + "SELECT COALESCE(`tmp`.`user_id`, 0) AS `user_id`, `tmp`.`shop_id`, COALESCE(`tmp`.`ds`, '') AS `ds`, SUM(`tmp`.`payment_amount_cents`) AS `payed_buy_fee_sum`, SUM(1) AS `pv`\n" + + "FROM (SELECT `datagenSource`.`user_id`, `datagenSource`.`shop_id`, DATE_FORMAT(`datagenSource`.`order_created_at`, 'yyyy-MM-dd') AS `ds`, `datagenSource`.`payment_amount_cents`\n" + + "FROM `%s`.`%s`.`datagenSource` AS `datagenSource`) AS `tmp`\n" + + "GROUP BY ROW(`tmp`.`user_id`, `tmp`.`shop_id`, `tmp`.`ds`)", + CATALOG, DATABASE)); + assertThat(oldTable.getSerializedRefreshHandler()) + .isNotEqualTo(newTable.getSerializedRefreshHandler()); + assertThat(getJobRestoreSavepointPath(getRefreshJobId())).isEmpty(); + } + + @Test + void testAlterMaterializedTableAsQueryInContinuousModeWithSuspendStatus( + @TempDir Path temporaryPath) throws Exception { + tEnv.executeSql( + "CREATE MATERIALIZED TABLE users_shops" + + " PARTITIONED BY (ds)\n" + + UPDATING_VALUES_SINK + + " FRESHNESS = INTERVAL '30' SECOND\n" + + " AS SELECT \n" + + " user_id,\n" + + " shop_id,\n" + + " ds,\n" + + " SUM (payment_amount_cents) AS payed_buy_fee_sum\n" + + " FROM (\n" + + " SELECT user_id, shop_id, DATE_FORMAT(order_created_at, 'yyyy-MM-dd') AS ds, payment_amount_cents FROM datagenSource" + + " ) AS tmp\n" + + " GROUP BY (user_id, shop_id, ds)"); + final ResolvedCatalogMaterializedTable oldTable = getUsersShops(); + waitUntilAllTasksAreRunning(clusterClient, getRefreshJobId()); + + setupSavepointDir(temporaryPath); + + tEnv.executeSql("ALTER MATERIALIZED TABLE users_shops SUSPEND"); + + tEnv.executeSql( + "ALTER MATERIALIZED TABLE users_shops" + + " AS SELECT \n" + + " user_id,\n" + + " shop_id,\n" + + " ds,\n" + + " SUM (payment_amount_cents) AS payed_buy_fee_sum,\n" + + " SUM (1) AS pv\n" + + " FROM (\n" + + " SELECT user_id, shop_id, DATE_FORMAT(order_created_at, 'yyyy-MM-dd') AS ds, payment_amount_cents FROM datagenSource" + + " ) AS tmp\n" + + " GROUP BY (user_id, shop_id, ds)"); + final ResolvedCatalogMaterializedTable newTable = getUsersShops(); + + assertThat(getAddedColumns(newTable.getResolvedSchema(), oldTable.getResolvedSchema())) + .containsExactly(Column.physical("pv", DataTypes.INT())); + assertThat(newTable.getExpandedQuery()) + .hasToString( + String.format( + "SELECT `tmp`.`user_id`, `tmp`.`shop_id`, `tmp`.`ds`, SUM(`tmp`.`payment_amount_cents`) AS `payed_buy_fee_sum`, SUM(1) AS `pv`\n" + + "FROM (SELECT `datagenSource`.`user_id`, `datagenSource`.`shop_id`, DATE_FORMAT(`datagenSource`.`order_created_at`, 'yyyy-MM-dd') AS `ds`, `datagenSource`.`payment_amount_cents`\n" + + "FROM `%s`.`%s`.`datagenSource` AS `datagenSource`) AS `tmp`\n" + + "GROUP BY ROW(`tmp`.`user_id`, `tmp`.`shop_id`, `tmp`.`ds`)", + CATALOG, DATABASE)); + assertThat(oldTable.getSerializedRefreshHandler()) + .isEqualTo(newTable.getSerializedRefreshHandler()); + assertThat(oldTable.getDefinitionFreshness()).isEqualTo(newTable.getDefinitionFreshness()); + + // verify the restore path is empty + assertThat(getRefreshHandler().getRestorePath()).isEmpty(); + } + + @Test + void testDropMaterializedTableInContinuousMode() throws Exception { + tEnv.executeSql(USERS_SHOPS_DDL); + + assertThat(catalog.getTable(USERS_SHOPS.toObjectPath()).getTableKind()) + .isEqualTo(TableKind.MATERIALIZED_TABLE); + final JobID refreshJobId = getRefreshJobId(); + waitUntilAllTasksAreRunning(clusterClient, refreshJobId); + assertThat(getJobStatus(refreshJobId)).isEqualTo(JobStatus.RUNNING); + + // Drop materialized table using drop table statement + assertThatThrownBy(() -> tEnv.executeSql("DROP TABLE users_shops")) + .isInstanceOf(ValidationException.class) + .hasMessage( + String.format( + "Table with identifier '%s' does not exist.", + USERS_SHOPS.asSummaryString())); + + tEnv.executeSql("DROP MATERIALIZED TABLE IF EXISTS users_shops"); + + assertThat(catalog.tableExists(USERS_SHOPS.toObjectPath())).isFalse(); + awaitJobStatus(refreshJobId, JobStatus.CANCELED); + + // verify drop materialized table that doesn't exist + assertThatThrownBy(() -> tEnv.executeSql("DROP MATERIALIZED TABLE users_shops")) + .isInstanceOf(ValidationException.class) + .hasMessage( + String.format( + "Materialized table with identifier %s does not exist.", + USERS_SHOPS.asSerializableString())); + + tEnv.executeSql("DROP MATERIALIZED TABLE IF EXISTS users_shops"); + + // Drop a table using drop materialized table statement + assertThatThrownBy(() -> tEnv.executeSql("DROP MATERIALIZED TABLE IF EXISTS datagenSource")) + .isInstanceOf(ValidationException.class) + .hasMessage( + String.format( + "Table %s is not a materialized table, does not support materialized table related operation.", + ObjectIdentifier.of(CATALOG, DATABASE, "datagenSource") + .asSerializableString())); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("unsupportedTargets") + void unsupportedTargetIsRejectedBeforeAnyCatalogChange( + String description, @Nullable String tableConfigTarget, String expectedMessage) + throws Exception { + final TableEnvironment tableEnv; + if (tableConfigTarget == null) { + tableEnv = TableEnvironment.create(EnvironmentSettings.inStreamingMode()); + tableEnv.registerCatalog(CATALOG, catalog); + tableEnv.useCatalog(CATALOG); + tableEnv.useDatabase(DATABASE); + } else { + tableEnv = tEnv; + tableEnv.getConfig().set(DeploymentOptions.TARGET, tableConfigTarget); + } + final Set jobsBefore = listJobIds(); + + assertThatThrownBy(() -> tableEnv.executeSql(USERS_SHOPS_DDL)) + .isInstanceOf(ValidationException.class) + .hasMessage(expectedMessage); + + assertThat(catalog.tableExists(USERS_SHOPS.toObjectPath())).isFalse(); + assertThat(listJobIds()).isEqualTo(jobsBefore); + } + + private static Stream unsupportedTargets() { + return Stream.of( + Arguments.of( + "default environment", + null, + "Unsupported execution target detected: local. Materialized table" + + " refresh jobs need a session cluster: set 'execution.target'" + + " to 'remote', 'yarn-session' or 'kubernetes-session' (with" + + " 'yarn.application.id' or 'kubernetes.cluster-id')."), + Arguments.of( + "kubernetes-application", + "kubernetes-application", + "Application-mode materialized table refresh is not supported in" + + " this environment: execution target 'kubernetes-application'" + + " requires the SQL Gateway. Use a session cluster target such" + + " as 'remote', 'yarn-session' or 'kubernetes-session'."), + Arguments.of( + "kubernetes-session without its cluster id", + "kubernetes-session", + "Execution target 'kubernetes-session' requires" + + " 'kubernetes.cluster-id' to be configured to submit or" + + " control a materialized table refresh job.")); + } + + @Test + void resumeOntoAnotherClusterIsRejectedBeforeAnythingChanges(@TempDir Path temporaryPath) + throws Exception { + tEnv.executeSql(USERS_SHOPS_DDL); + waitUntilAllTasksAreRunning(clusterClient, getRefreshJobId()); + setupSavepointDir(temporaryPath); + tEnv.executeSql("ALTER MATERIALIZED TABLE users_shops SUSPEND"); + final ResolvedCatalogMaterializedTable suspendedTable = getUsersShops(); + final Set jobsBefore = listJobIds(); + + tEnv.getConfig().set(DeploymentOptions.TARGET, "kubernetes-session"); + tEnv.getConfig().set(KUBERNETES_CLUSTER_ID, "other"); + + assertThatThrownBy(() -> tEnv.executeSql("ALTER MATERIALIZED TABLE users_shops RESUME")) + .isInstanceOf(ValidationException.class) + .hasMessageContaining( + "RESUME, ALTER and a redeploy never move a refresh job to another cluster"); + + final ResolvedCatalogMaterializedTable unchangedTable = getUsersShops(); + assertThat(unchangedTable.getRefreshStatus()).isSameAs(RefreshStatus.SUSPENDED); + assertThat(unchangedTable.getSerializedRefreshHandler()) + .isEqualTo(suspendedTable.getSerializedRefreshHandler()); + assertThat(listJobIds()).isEqualTo(jobsBefore); + } + + private void createBoundedValuesSource(List data) { + tEnv.executeSql( + String.format( + "CREATE TABLE IF NOT EXISTS my_source (\n" + + " order_id BIGINT,\n" + + " user_id BIGINT,\n" + + " shop_id BIGINT,\n" + + " order_created_at STRING\n" + + ")\n" + + "WITH (\n" + + " 'connector' = 'values',\n" + + " 'bounded' = 'true',\n" + + " 'data-id' = '%s'\n" + + ")", + TestValuesTableFactory.registerData(data))); + } + + private void setupSavepointDir(Path temporaryPath) { + tEnv.getConfig() + .set( + CheckpointingOptions.SAVEPOINT_DIRECTORY, + "file://" + temporaryPath.toAbsolutePath()); + } + + private ResolvedCatalogMaterializedTable getUsersShops() { + return (ResolvedCatalogMaterializedTable) + ((TableEnvironmentInternal) tEnv) + .getCatalogManager() + .getTableOrError(USERS_SHOPS) + .getResolvedTable(); + } + + private ContinuousRefreshHandler getRefreshHandler() throws Exception { + return ContinuousRefreshHandlerSerializer.INSTANCE.deserialize( + getUsersShops().getSerializedRefreshHandler(), getClass().getClassLoader()); + } + + private JobID getRefreshJobId() throws Exception { + return JobID.fromHexString(getRefreshHandler().getJobId()); + } + + private static List getUsersShopsResults() { + return TestValuesTableFactory.getResultsAsStrings(USERS_SHOPS.getObjectName()); + } + + private static List getAddedColumns( + ResolvedSchema newSchema, ResolvedSchema oldSchema) { + return newSchema.getColumns().stream() + .filter(column -> !oldSchema.getColumns().contains(column)) + .collect(Collectors.toList()); + } + + private void verifyRefreshJobCreated(JobID jobId, long startTime) throws Exception { + assertThat(clusterClient.listJobs().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS)) + .filteredOn(job -> job.getJobId().equals(jobId)) + .singleElement() + .satisfies(job -> assertThat(job.getStartTime()).isGreaterThan(startTime)); + assertThat( + clusterClient + .getJobDetails(jobId) + .get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS) + .getJobType()) + .isEqualTo(JobType.BATCH); + awaitJobStatus(jobId, JobStatus.FINISHED); + } + + private Set listJobIds() throws Exception { + return clusterClient.listJobs().get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS).stream() + .map(JobStatusMessage::getJobId) + .collect(Collectors.toSet()); + } + + private JobStatus getJobStatus(JobID jobId) throws Exception { + return clusterClient.getJobStatus(jobId).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } + + private void awaitJobStatus(JobID jobId, JobStatus expectedStatus) throws Exception { + CommonTestUtils.waitUtil( + () -> { + try { + return getJobStatus(jobId) == expectedStatus; + } catch (Exception e) { + return false; + } + }, + TIMEOUT, + PAUSE, + String.format("Job %s did not reach %s in time.", jobId, expectedStatus)); + } + + private long getCheckpointInterval(JobID jobId) throws Exception { + return RestMapperUtils.getStrictObjectMapper() + .valueToTree(sendJobRequest(CheckpointConfigHeaders.getInstance(), jobId)) + .get(CheckpointConfigInfo.FIELD_NAME_CHECKPOINT_INTERVAL) + .asLong(); + } + + private Optional getJobRestoreSavepointPath(JobID jobId) throws Exception { + return Optional.ofNullable( + sendJobRequest(CheckpointingStatisticsHeaders.getInstance(), jobId) + .getLatestCheckpoints() + .getRestoredCheckpointStatistics()) + .map(RestoredCheckpointStatistics::getExternalPath); + } + + private

P sendJobRequest( + MessageHeaders headers, JobID jobId) throws Exception { + final M parameters = headers.getUnresolvedMessageParameters(); + parameters.jobPathParameter.resolve(jobId); + return clusterClient + .sendRequest(headers, parameters, EmptyRequestBody.getInstance()) + .get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } +} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala index 6126e67a946bd9..b6a505aa1008f1 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala @@ -44,6 +44,7 @@ import org.apache.flink.table.connector.ChangelogMode import org.apache.flink.table.connector.source.{DynamicTableSource, InputFormatProvider, ScanTableSource} import org.apache.flink.table.data.{DecimalDataUtils, RowData} import org.apache.flink.table.delegation.{Executor, ExecutorFactory} +import org.apache.flink.table.delegation.materializedtable.MaterializedTableExecutorFactory import org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_TYPE import org.apache.flink.table.descriptors.DescriptorProperties import org.apache.flink.table.expressions.Expression @@ -1714,7 +1715,8 @@ class TestingTableEnvironment private ( executor: Executor, functionCatalog: FunctionCatalog, planner: PlannerBase, - isStreamingMode: Boolean) + isStreamingMode: Boolean, + materializedTableExecutorFactory: MaterializedTableExecutorFactory) extends TableEnvironmentImpl( catalogManager, moduleManager, @@ -1723,7 +1725,9 @@ class TestingTableEnvironment private ( executor, functionCatalog, planner, - isStreamingMode) { + isStreamingMode, + materializedTableExecutorFactory, + null) { def getResourceManager: ResourceManager = resourceManager @@ -1840,6 +1844,11 @@ object TestingTableEnvironment { functionCatalog) .asInstanceOf[PlannerBase] + val materializedTableExecutorFactory = FactoryUtil.discoverFactory( + userClassLoader, + classOf[MaterializedTableExecutorFactory], + MaterializedTableExecutorFactory.DEFAULT_IDENTIFIER) + new TestingTableEnvironment( catalogMgr, moduleManager, @@ -1848,7 +1857,8 @@ object TestingTableEnvironment { executor, functionCatalog, planner, - settings.isStreamingMode) + settings.isStreamingMode, + materializedTableExecutorFactory) } }