From 8fdb5e9ad0bb1e1803caeaa883347d06fc9b6f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Wed, 22 Jul 2026 14:00:47 +0800 Subject: [PATCH 01/43] Implement SQL task query result alert via Master-side event handling --- docs/docs/en/guide/upgrade/incompatible.md | 4 ++ docs/docs/zh/guide/upgrade/incompatible.md | 5 ++- .../alert/rpc/AlertOperatorImpl.java | 3 +- .../alert/service/AlertSender.java | 5 ++- .../alert/runner/AlertSenderTest.java | 11 +++--- .../common/enums/AlertType.java | 3 +- .../mysql/dolphinscheduler_ddl.sql | 17 ++++++++ .../mysql/dolphinscheduler_dml.sql | 39 +++++++++++++++++++ .../postgresql/dolphinscheduler_ddl.sql | 18 +++++++++ .../postgresql/dolphinscheduler_dml.sql | 31 +++++++++++++++ .../dao/mapper/TaskDefinitionMapperTest.java | 4 +- .../alert/request/AlertSendRequest.java | 4 +- .../rpc/TaskExecutorEventListenerImpl.java | 13 +++++++ .../service/alert/WorkflowAlertManager.java | 25 ++++++++++++ .../TaskExecutorSuccessLifecycleEvent.java | 7 ++++ .../plugin/task/api/TaskExecutionContext.java | 5 +++ .../plugin/task/api/model/TaskAlertInfo.java | 4 ++ .../task/api/parameters/SqlParameters.java | 17 ++++++-- .../api/parameters/SqlParametersTest.java | 6 +-- .../plugin/task/sql/SqlTask.java | 38 +++++++++++++----- .../components/node/fields/use-sql-type.ts | 14 +++---- .../task/components/node/format-data.ts | 4 +- .../projects/task/components/node/types.ts | 2 +- 23 files changed, 241 insertions(+), 38 deletions(-) create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index 0f60fcb9986a..661e049f7871 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -44,3 +44,7 @@ This document records the incompatible updates between each version. You need to * Remove import and export of workflow definition. ([#17940])(https://github.com/apache/dolphinscheduler/issues/17940) +## 3.5.0 + +* Rename the `sendEmail` field in SQL task parameters to `sendAlert`. The data migration script will automatically migrate `sendEmail` to `sendAlert` in the `t_ds_task_definition` and `t_ds_task_definition_log` tables.([#17854])(https://github.com/apache/dolphinscheduler/issues/17854) + diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 2f0c4c044ff8..601f1c6dc7ac 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -4,7 +4,6 @@ ## dev -* 将mysql驱动版本从8.0.16升级至8.0.33 ([#14684](https://github.com/apache/dolphinscheduler/pull/14684)) * 更改了环境变量名称，将 `PYTHON_HOME` 改为 `PYTHON_LAUNCHER`， 将 `DATAX_HOME` 改为 `DATAX_LAUNCHER` ([#14523](https://github.com/apache/dolphinscheduler/pull/14523)) * 更新了SQL任务中用于匹配变量的正则表达式 ([#13378](https://github.com/apache/dolphinscheduler/pull/13378)) * Remove the spark version of spark task ([#11860](https://github.com/apache/dolphinscheduler/pull/11860)). @@ -48,3 +47,7 @@ * 移除导入导出工作流([#17940])(https://github.com/apache/dolphinscheduler/issues/17940) +## 3.5.0 + +* 将SQL任务参数中的 `sendEmail` 字段重命名为 `sendAlert`，数据迁移脚本会自动将 `t_ds_task_definition` 和 `t_ds_task_definition_log` 表中的 `sendEmail` 迁移为 `sendAlert`。([#17854])(https://github.com/apache/dolphinscheduler/issues/17854) + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java index 95b3809dd20b..2e8788ab49f5 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java @@ -41,7 +41,8 @@ public AlertSendResponse sendAlert(AlertSendRequest alertSendRequest) { AlertSendResponse alertSendResponse = alertSender.syncHandler( alertSendRequest.getGroupId(), alertSendRequest.getTitle(), - alertSendRequest.getContent()); + alertSendRequest.getContent(), + alertSendRequest.getAlertType()); log.info("Handle AlertSendRequest finish: {}", alertSendResponse); return alertSendResponse; } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java index 9c9cd034bdb6..3ef14c9414a7 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java @@ -55,13 +55,16 @@ public AlertSender(AlertDao alertDao, * @param alertGroupId alertGroupId * @param title title * @param content content + * @param alertType alertType * @return AlertSendResponseCommand */ - public AlertSendResponse syncHandler(int alertGroupId, String title, String content) { + public AlertSendResponse syncHandler(int alertGroupId, String title, String content, + org.apache.dolphinscheduler.common.enums.AlertType alertType) { List alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId); AlertData alertData = AlertData.builder() .content(content) .title(title) + .alertType(alertType.getCode()) .build(); boolean sendResponseStatus = true; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java index 18246f485ab2..5ba3c6df25d9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java @@ -89,7 +89,8 @@ void testSyncHandler() { // 1.alert instance does not exist when(alertDao.listInstanceByAlertGroupId(ALERT_GROUP_ID)).thenReturn(null); - AlertSendResponse alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); + AlertSendResponse alertSendResponse = + alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, AlertType.TASK_RESULT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); @@ -104,7 +105,7 @@ void testSyncHandler() { alertInstanceList.add(alertPluginInstance); when(alertDao.listInstanceByAlertGroupId(1)).thenReturn(alertInstanceList); - alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, AlertType.TASK_RESULT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); @@ -114,7 +115,7 @@ void testSyncHandler() { when(alertChannelMock.process(Mockito.any())).thenReturn(null); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, AlertType.TASK_RESULT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); @@ -126,7 +127,7 @@ void testSyncHandler() { when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, AlertType.TASK_RESULT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); @@ -138,7 +139,7 @@ void testSyncHandler() { when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, AlertType.TASK_RESULT); Assertions.assertTrue(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java index 058afcb3fc71..6cc850d6f141 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java @@ -29,7 +29,7 @@ public enum AlertType { /** * 0 workflow instance failure, 1 workflow instance success, 2 workflow instance blocked, 3 workflow instance timeout, 4 fault tolerance warning, - * 5 task failure, 6 task success, 7 task timeout + * 5 task failure, 6 task success, 7 task timeout, 8 task result */ WORKFLOW_INSTANCE_FAILURE(0, "workflow instance failure"), WORKFLOW_INSTANCE_SUCCESS(1, "workflow instance success"), @@ -39,6 +39,7 @@ public enum AlertType { TASK_FAILURE(5, "task failure"), TASK_SUCCESS(6, "task success"), TASK_TIMEOUT(7, "task timeout"), + TASK_RESULT(8, "task result"), ; AlertType(int code, String descp) { diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql new file mode 100644 index 000000000000..5f26e3515d67 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,17 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 000000000000..96180cc1e501 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +UPDATE t_ds_task_definition +SET task_params = JSON_REMOVE( + JSON_INSERT( + task_params, + '$.sendAlert', + JSON_EXTRACT(task_params, '$.sendEmail') + ), + '$.sendEmail' +) +WHERE task_params IS NOT NULL AND JSON_EXTRACT(task_params, '$.sendEmail') IS NOT NULL; + +UPDATE t_ds_task_definition_log +SET task_params = JSON_REMOVE( + JSON_INSERT( + task_params, + '$.sendAlert', + JSON_EXTRACT(task_params, '$.sendEmail') + ), + '$.sendEmail' +) +WHERE task_params IS NOT NULL AND JSON_EXTRACT(task_params, '$.sendEmail') IS NOT NULL; + diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql new file mode 100644 index 000000000000..61d15aaf4dfe --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -0,0 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 000000000000..970cf33b5243 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +UPDATE t_ds_task_definition +SET task_params = ( + (task_params::jsonb - 'sendEmail') + || jsonb_build_object('sendAlert', task_params::jsonb->'sendEmail') +)::text +WHERE jsonb_path_exists(task_params::jsonb, '$.sendEmail'); + +UPDATE t_ds_task_definition_log +SET task_params = ( + (task_params::jsonb - 'sendEmail') + || jsonb_build_object('sendAlert', task_params::jsonb->'sendEmail') +)::text +WHERE jsonb_path_exists(task_params::jsonb, '$.sendEmail'); + diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java index da3cc1d27481..83fc14e5fec7 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java @@ -140,7 +140,7 @@ public void testDeleteByCode() { @Test public void testNullPropertyValueOfLocalParams() { String definitionJson = - "{\"failRetryTimes\":\"0\",\"timeoutNotifyStrategy\":\"\",\"code\":\"5195043558720\",\"flag\":\"YES\",\"environmentCode\":\"-1\",\"taskDefinitionIndex\":2,\"taskPriority\":\"MEDIUM\",\"taskParams\":\"{\\\"preStatements\\\":null,\\\"postStatements\\\":null,\\\"type\\\":\\\"ADB_MYSQL\\\",\\\"database\\\":\\\"lijia\\\",\\\"sql\\\":\\\"create table nation_${random_serial_number} as select * from nation\\\",\\\"localParams\\\":[{\\\"direct\\\":2,\\\"type\\\":3,\\\"prop\\\":\\\"key\\\"}],\\\"Name\\\":\\\"create_table_as_select_nation\\\",\\\"FailRetryTimes\\\":0,\\\"dbClusterId\\\":\\\"amv-bp10o45925jpx959\\\",\\\"sendEmail\\\":false,\\\"displayRows\\\":10,\\\"limit\\\":10000,\\\"agentSource\\\":\\\"Workflow\\\",\\\"agentVersion\\\":\\\"Unkown\\\"}\",\"timeout\":\"0\",\"taskType\":\"ADB_MYSQL\",\"timeoutFlag\":\"CLOSE\",\"projectCode\":\"5191800302720\",\"name\":\"create_table_as_select_nation\",\"delayTime\":\"0\",\"workerGroup\":\"default\"}"; + "{\"failRetryTimes\":\"0\",\"timeoutNotifyStrategy\":\"\",\"code\":\"5195043558720\",\"flag\":\"YES\",\"environmentCode\":\"-1\",\"taskDefinitionIndex\":2,\"taskPriority\":\"MEDIUM\",\"taskParams\":\"{\\\"preStatements\\\":null,\\\"postStatements\\\":null,\\\"type\\\":\\\"ADB_MYSQL\\\",\\\"database\\\":\\\"lijia\\\",\\\"sql\\\":\\\"create table nation_${random_serial_number} as select * from nation\\\",\\\"localParams\\\":[{\\\"direct\\\":2,\\\"type\\\":3,\\\"prop\\\":\\\"key\\\"}],\\\"Name\\\":\\\"create_table_as_select_nation\\\",\\\"FailRetryTimes\\\":0,\\\"dbClusterId\\\":\\\"amv-bp10o45925jpx959\\\",\\\"sendAlert\\\":false,\\\"displayRows\\\":10,\\\"limit\\\":10000,\\\"agentSource\\\":\\\"Workflow\\\",\\\"agentVersion\\\":\\\"Unkown\\\"}\",\"timeout\":\"0\",\"taskType\":\"ADB_MYSQL\",\"timeoutFlag\":\"CLOSE\",\"projectCode\":\"5191800302720\",\"name\":\"create_table_as_select_nation\",\"delayTime\":\"0\",\"workerGroup\":\"default\"}"; TaskDefinition definition = JSONUtils.parseObject(definitionJson, TaskDefinition.class); Map taskParamsMap = definition.getTaskParamMap(); @@ -157,7 +157,7 @@ public void testNullPropertyValueOfLocalParams() { @Test public void testNullLocalParamsOfTaskParams() { String definitionJson = - "{\"failRetryTimes\":\"0\",\"timeoutNotifyStrategy\":\"\",\"code\":\"5195043558720\",\"flag\":\"YES\",\"environmentCode\":\"-1\",\"taskDefinitionIndex\":2,\"taskPriority\":\"MEDIUM\",\"taskParams\":\"{\\\"preStatements\\\":null,\\\"postStatements\\\":null,\\\"type\\\":\\\"ADB_MYSQL\\\",\\\"database\\\":\\\"lijia\\\",\\\"sql\\\":\\\"create table nation_${random_serial_number} as select * from nation\\\",\\\"localParams\\\":null,\\\"Name\\\":\\\"create_table_as_select_nation\\\",\\\"FailRetryTimes\\\":0,\\\"dbClusterId\\\":\\\"amv-bp10o45925jpx959\\\",\\\"sendEmail\\\":false,\\\"displayRows\\\":10,\\\"limit\\\":10000,\\\"agentSource\\\":\\\"Workflow\\\",\\\"agentVersion\\\":\\\"Unkown\\\"}\",\"timeout\":\"0\",\"taskType\":\"ADB_MYSQL\",\"timeoutFlag\":\"CLOSE\",\"projectCode\":\"5191800302720\",\"name\":\"create_table_as_select_nation\",\"delayTime\":\"0\",\"workerGroup\":\"default\"}"; + "{\"failRetryTimes\":\"0\",\"timeoutNotifyStrategy\":\"\",\"code\":\"5195043558720\",\"flag\":\"YES\",\"environmentCode\":\"-1\",\"taskDefinitionIndex\":2,\"taskPriority\":\"MEDIUM\",\"taskParams\":\"{\\\"preStatements\\\":null,\\\"postStatements\\\":null,\\\"type\\\":\\\"ADB_MYSQL\\\",\\\"database\\\":\\\"lijia\\\",\\\"sql\\\":\\\"create table nation_${random_serial_number} as select * from nation\\\",\\\"localParams\\\":null,\\\"Name\\\":\\\"create_table_as_select_nation\\\",\\\"FailRetryTimes\\\":0,\\\"dbClusterId\\\":\\\"amv-bp10o45925jpx959\\\",\\\"sendAlert\\\":false,\\\"displayRows\\\":10,\\\"limit\\\":10000,\\\"agentSource\\\":\\\"Workflow\\\",\\\"agentVersion\\\":\\\"Unkown\\\"}\",\"timeout\":\"0\",\"taskType\":\"ADB_MYSQL\",\"timeoutFlag\":\"CLOSE\",\"projectCode\":\"5191800302720\",\"name\":\"create_table_as_select_nation\",\"delayTime\":\"0\",\"workerGroup\":\"default\"}"; TaskDefinition definition = JSONUtils.parseObject(definitionJson, TaskDefinition.class); Assertions.assertNull(definition.getTaskParamMap(), "Serialize the task definition success"); diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendRequest.java index dfa6515ddea2..82996bfc6c2f 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendRequest.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.extract.alert.request; +import org.apache.dolphinscheduler.common.enums.AlertType; + import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -32,6 +34,6 @@ public class AlertSendRequest { private String content; - private int warnType; + private AlertType alertType; } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java index 278f5160dc80..9421d293ba41 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.server.master.rpc; import org.apache.dolphinscheduler.extract.master.ITaskExecutorEventListener; +import org.apache.dolphinscheduler.plugin.task.api.model.TaskAlertInfo; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.server.master.engine.IWorkflowRepository; import org.apache.dolphinscheduler.server.master.engine.task.execution.ITaskExecution; @@ -29,6 +30,7 @@ import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskRuntimeContextChangedEvent; import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskSuccessLifecycleEvent; import org.apache.dolphinscheduler.server.master.engine.workflow.execution.IWorkflowExecution; +import org.apache.dolphinscheduler.service.alert.WorkflowAlertManager; import org.apache.dolphinscheduler.task.executor.events.IReportableTaskExecutorLifecycleEvent; import org.apache.dolphinscheduler.task.executor.events.TaskExecutorDispatchedLifecycleEvent; import org.apache.dolphinscheduler.task.executor.events.TaskExecutorFailedLifecycleEvent; @@ -52,6 +54,9 @@ public class TaskExecutorEventListenerImpl implements ITaskExecutorEventListener @Autowired private IWorkflowRepository workflowRepository; + @Autowired + private WorkflowAlertManager workflowAlertManager; + @Override public void onTaskExecutorDispatched(final TaskExecutorDispatchedLifecycleEvent taskExecutorDispatchedLifecycleEvent) { LogUtils.setWorkflowInstanceIdMDC(taskExecutorDispatchedLifecycleEvent.getWorkflowInstanceId()); @@ -118,6 +123,14 @@ public void onTaskExecutorSuccess(final TaskExecutorSuccessLifecycleEvent taskEx .varPool(taskExecutorSuccessLifecycleEvent.getVarPool()) .build(); taskExecution.getWorkflowEventBus().publish(taskSuccessEvent); + + if (taskExecutorSuccessLifecycleEvent.isNeedAlert()) { + TaskAlertInfo taskAlertInfo = taskExecutorSuccessLifecycleEvent.getTaskAlertInfo(); + workflowAlertManager.sendTaskResultAlert( + taskAlertInfo, + taskExecution.getTaskExecutionContext().getProjectCode(), + taskExecution.getTaskExecutionContext().getWorkflowInstanceId()); + } } finally { LogUtils.removeWorkflowInstanceIdMDC(); } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManager.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManager.java index 9335e9551eb7..576ac19377a8 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManager.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManager.java @@ -34,6 +34,7 @@ import org.apache.dolphinscheduler.dao.repository.ProjectDao; import org.apache.dolphinscheduler.dao.repository.UserDao; import org.apache.dolphinscheduler.dao.repository.WorkflowDefinitionLogDao; +import org.apache.dolphinscheduler.plugin.task.api.model.TaskAlertInfo; import java.util.ArrayList; import java.util.Date; @@ -208,4 +209,28 @@ public void sendWorkflowTimeoutAlert(WorkflowInstance workflowInstance) { alertDao.sendWorkflowTimeoutAlert(workflowInstance, projectUser); } + /** + * Send task result alert + * + * @param taskAlertInfo task alert info + * @param projectCode project code + * @param workflowInstanceId workflow instance id + */ + public void sendTaskResultAlert(TaskAlertInfo taskAlertInfo, Long projectCode, Integer workflowInstanceId) { + if (taskAlertInfo == null || taskAlertInfo.getAlertGroupId() == null) { + return; + } + Alert alert = new Alert(); + alert.setTitle(taskAlertInfo.getTitle()); + alert.setContent(taskAlertInfo.getContent()); + alert.setAlertGroupId(taskAlertInfo.getAlertGroupId()); + alert.setAlertType(taskAlertInfo.getAlertType()); + alert.setWarningType(WarningType.SUCCESS); + alert.setCreateTime(new Date()); + alert.setProjectCode(projectCode); + alert.setWorkflowInstanceId(workflowInstanceId); + alertDao.addAlert(alert); + log.info("Added task result alert for workflow instance: {}", workflowInstanceId); + } + } diff --git a/dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/events/TaskExecutorSuccessLifecycleEvent.java b/dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/events/TaskExecutorSuccessLifecycleEvent.java index 947d1edf3633..2c55c80c365c 100644 --- a/dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/events/TaskExecutorSuccessLifecycleEvent.java +++ b/dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/events/TaskExecutorSuccessLifecycleEvent.java @@ -19,6 +19,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.plugin.task.api.model.TaskAlertInfo; import org.apache.dolphinscheduler.task.executor.ITaskExecutor; import java.util.List; @@ -48,6 +49,10 @@ public class TaskExecutorSuccessLifecycleEvent extends AbstractTaskExecutorLifec private Long latestReportTime; + private boolean needAlert; + + private TaskAlertInfo taskAlertInfo; + public static TaskExecutorSuccessLifecycleEvent of(final ITaskExecutor taskExecutor) { final TaskExecutionContext taskExecutionContext = taskExecutor.getTaskExecutionContext(); return TaskExecutorSuccessLifecycleEvent.builder() @@ -57,6 +62,8 @@ public static TaskExecutorSuccessLifecycleEvent of(final ITaskExecutor taskExecu .varPool(taskExecutionContext.getVarPool()) .type(TaskExecutorLifecycleEventType.SUCCESS) .endTime(taskExecutionContext.getEndTime()) + .needAlert(taskExecutionContext.isNeedAlert()) + .taskAlertInfo(taskExecutionContext.getTaskAlertInfo()) .build(); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java index 85d6f682196b..a072258011df 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java @@ -19,6 +19,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.plugin.task.api.model.TaskAlertInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; @@ -130,6 +131,10 @@ public class TaskExecutionContext implements Serializable { private final long firstDispatchTime = System.currentTimeMillis(); + private boolean needAlert; + + private TaskAlertInfo taskAlertInfo; + public int increaseDispatchFailTimes() { return ++dispatchFailTimes; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/TaskAlertInfo.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/TaskAlertInfo.java index e72242e93473..d6771f6b20ab 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/TaskAlertInfo.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/TaskAlertInfo.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.plugin.task.api.model; +import org.apache.dolphinscheduler.common.enums.AlertType; + import lombok.Data; @Data @@ -27,4 +29,6 @@ public class TaskAlertInfo { private String content; private Integer alertGroupId; + + private AlertType alertType; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java index 32b110255cf9..7da491ea9871 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java @@ -73,8 +73,6 @@ public class SqlParameters extends AbstractParameters { */ private int sqlType; - private Boolean sendEmail; - private int displayRows; /** @@ -85,6 +83,7 @@ public class SqlParameters extends AbstractParameters { * 3 TABLE+attachment */ private String showType; + /** * SQL connection parameters */ @@ -92,7 +91,19 @@ public class SqlParameters extends AbstractParameters { private List preStatements; private List postStatements; + /** + * Whether to send alert for SQL query result + */ + private Boolean sendAlert; + + /** + * Alert group id + */ private int groupId; + + /** + * Alert title + */ private String title; private int limit; @@ -173,7 +184,7 @@ public String toString() { + ", sqlSource='" + sqlSource + '\'' + ", sqlResource='" + sqlResource + '\'' + ", sqlType=" + sqlType - + ", sendEmail=" + sendEmail + + ", sendAlert=" + sendAlert + ", displayRows=" + displayRows + ", limit=" + limit + ", showType='" + showType + '\'' diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParametersTest.java index b9388b3ba54f..39c17615f384 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParametersTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParametersTest.java @@ -40,7 +40,7 @@ public class SqlParametersTest { private final String sql = "select * from t_ds_user"; private final int datasource = 1; private final int sqlType = 0; - private final Boolean sendEmail = true; + private final Boolean sendAlert = true; private final int displayRows = 10; private final String showType = "TABLE"; private final String title = "sql test"; @@ -63,7 +63,7 @@ public void testSqlParameters() { sqlParameters.setSql(sql); sqlParameters.setDatasource(datasource); sqlParameters.setSqlType(sqlType); - sqlParameters.setSendEmail(sendEmail); + sqlParameters.setSendAlert(sendAlert); sqlParameters.setDisplayRows(displayRows); sqlParameters.setShowType(showType); sqlParameters.setTitle(title); @@ -73,7 +73,7 @@ public void testSqlParameters() { Assertions.assertEquals(sql, sqlParameters.getSql()); Assertions.assertEquals(datasource, sqlParameters.getDatasource()); Assertions.assertEquals(sqlType, sqlParameters.getSqlType()); - Assertions.assertEquals(sendEmail, sqlParameters.getSendEmail()); + Assertions.assertEquals(sendAlert, sqlParameters.getSendAlert()); Assertions.assertEquals(displayRows, sqlParameters.getDisplayRows()); Assertions.assertEquals(showType, sqlParameters.getShowType()); Assertions.assertEquals(title, sqlParameters.getTitle()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index 113dfbcd9eb8..851efa0e3161 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.sql; +import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; @@ -290,10 +291,25 @@ private String resultProcess(ResultSet resultSet) throws Exception { String result = resultJSONArray.isEmpty() ? JSONUtils.toJsonString(generateEmptyRow(resultSet)) : JSONUtils.toJsonString(resultJSONArray); - if (Boolean.TRUE.equals(sqlParameters.getSendEmail())) { - sendAttachment(sqlParameters.getGroupId(), StringUtils.isNotEmpty(sqlParameters.getTitle()) + if (Boolean.TRUE.equals(sqlParameters.getSendAlert())) { + // Truncate alert content to avoid oversized payload when query result is large + int displayRows = sqlParameters.getDisplayRows() > 0 ? sqlParameters.getDisplayRows() + : TaskConstants.DEFAULT_DISPLAY_ROWS; + String alertContent; + if (resultJSONArray.size() > displayRows) { + ArrayNode truncatedArray = JSONUtils.createArrayNode(); + for (int i = 0; i < Math.min(displayRows, resultJSONArray.size()); i++) { + truncatedArray.add(resultJSONArray.get(i)); + } + alertContent = JSONUtils.toJsonString(truncatedArray); + log.debug("Alert content truncated to {} rows", displayRows); + } else { + alertContent = result; + } + + prepareTaskResultAlert(sqlParameters.getGroupId(), StringUtils.isNotEmpty(sqlParameters.getTitle()) ? sqlParameters.getTitle() - : taskExecutionContext.getTaskName() + " query result sets", result); + : taskExecutionContext.getTaskName() + " query result sets", alertContent); } log.debug("execute sql result : {}", result); return result; @@ -320,18 +336,20 @@ private ArrayNode generateEmptyRow(ResultSet resultSet) throws SQLException { } /** - * send alert as an attachment + * Prepare task result alert info * - * @param title title - * @param content content + * @param alertGroupId alert group id + * @param title alert title + * @param content alert content */ - private void sendAttachment(int groupId, String title, String content) { - setNeedAlert(Boolean.TRUE); + private void prepareTaskResultAlert(int alertGroupId, String title, String content) { TaskAlertInfo taskAlertInfo = new TaskAlertInfo(); - taskAlertInfo.setAlertGroupId(groupId); + taskAlertInfo.setAlertGroupId(alertGroupId); taskAlertInfo.setContent(content); taskAlertInfo.setTitle(title); - setTaskAlertInfo(taskAlertInfo); + taskAlertInfo.setAlertType(AlertType.TASK_RESULT); + taskExecutionContext.setNeedAlert(true); + taskExecutionContext.setTaskAlertInfo(taskAlertInfo); } private String executeQuery(Connection connection, SqlBinds sqlBinds, String handlerType) throws Exception { diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sql-type.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sql-type.ts index d30ef9ef3221..b9ac314ff3d4 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sql-type.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sql-type.ts @@ -24,8 +24,8 @@ import type { IJsonItem } from '../types' export function useSqlType(model: { [field: string]: any }): IJsonItem[] { const { t } = useI18n() const querySpan = computed(() => (model.sqlType === '0' ? 6 : 0)) - const emailSpan = computed(() => - model.sqlType === '0' && model.sendEmail ? 24 : 0 + const alertSpan = computed(() => + model.sqlType === '0' && model.sendAlert ? 24 : 0 ) const groups = ref([]) const groupsLoading = ref(false) @@ -69,7 +69,7 @@ export function useSqlType(model: { [field: string]: any }): IJsonItem[] { }, { type: 'switch', - field: 'sendEmail', + field: 'sendAlert', span: querySpan, name: t('project.node.send_alarm') }, @@ -109,12 +109,12 @@ export function useSqlType(model: { [field: string]: any }): IJsonItem[] { props: { placeholder: t('project.node.title_tips') }, - span: emailSpan, + span: alertSpan, validate: { trigger: ['input', 'blur'], required: true, validator(unuse, value) { - if (model.sendEmail && !value) + if (model.sendAlert && !value) return new Error(t('project.node.title_tips')) } } @@ -124,7 +124,7 @@ export function useSqlType(model: { [field: string]: any }): IJsonItem[] { field: 'groupId', name: t('project.node.alarm_group'), options: groups, - span: emailSpan, + span: alertSpan, props: { loading: groupsLoading, placeholder: t('project.node.alarm_group_tips') @@ -133,7 +133,7 @@ export function useSqlType(model: { [field: string]: any }): IJsonItem[] { trigger: ['input', 'blur'], required: true, validator(unuse, value) { - if (model.sendEmail && !value) + if (model.sendAlert && !value) return new Error(t('project.node.alarm_group_tips')) } } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index a7754cb24d13..b29db829f33f 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -208,9 +208,9 @@ export function formatParams(data: INodeData): { taskParams.sqlType = data.sqlType taskParams.preStatements = data.preStatements taskParams.postStatements = data.postStatements - taskParams.sendEmail = data.sendEmail + taskParams.sendAlert = data.sendAlert taskParams.displayRows = data.displayRows - if (data.sqlType === '0' && data.sendEmail) { + if (data.sqlType === '0' && data.sendAlert) { taskParams.title = data.title taskParams.groupId = data.groupId } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts index 218f318db0d0..479f9260cc4d 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts @@ -305,7 +305,7 @@ interface ITaskParams { datasource?: string sql?: string sqlType?: string - sendEmail?: boolean + sendAlert?: boolean displayRows?: number title?: string groupId?: string From f4ad79cb55e8b59a906ec1c065959c5ed5019675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Wed, 29 Jul 2026 11:44:14 +0800 Subject: [PATCH 02/43] restore 3.5.0_schema --- .../mysql/dolphinscheduler_ddl.sql | 17 -------- .../mysql/dolphinscheduler_dml.sql | 39 ------------------- .../postgresql/dolphinscheduler_ddl.sql | 18 --------- .../postgresql/dolphinscheduler_dml.sql | 31 --------------- 4 files changed, 105 deletions(-) delete mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql delete mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql delete mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql delete mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql deleted file mode 100644 index 5f26e3515d67..000000000000 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql deleted file mode 100644 index 96180cc1e501..000000000000 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -UPDATE t_ds_task_definition -SET task_params = JSON_REMOVE( - JSON_INSERT( - task_params, - '$.sendAlert', - JSON_EXTRACT(task_params, '$.sendEmail') - ), - '$.sendEmail' -) -WHERE task_params IS NOT NULL AND JSON_EXTRACT(task_params, '$.sendEmail') IS NOT NULL; - -UPDATE t_ds_task_definition_log -SET task_params = JSON_REMOVE( - JSON_INSERT( - task_params, - '$.sendAlert', - JSON_EXTRACT(task_params, '$.sendEmail') - ), - '$.sendEmail' -) -WHERE task_params IS NOT NULL AND JSON_EXTRACT(task_params, '$.sendEmail') IS NOT NULL; - diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql deleted file mode 100644 index 61d15aaf4dfe..000000000000 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - - diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql deleted file mode 100644 index 970cf33b5243..000000000000 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -UPDATE t_ds_task_definition -SET task_params = ( - (task_params::jsonb - 'sendEmail') - || jsonb_build_object('sendAlert', task_params::jsonb->'sendEmail') -)::text -WHERE jsonb_path_exists(task_params::jsonb, '$.sendEmail'); - -UPDATE t_ds_task_definition_log -SET task_params = ( - (task_params::jsonb - 'sendEmail') - || jsonb_build_object('sendAlert', task_params::jsonb->'sendEmail') -)::text -WHERE jsonb_path_exists(task_params::jsonb, '$.sendEmail'); - From 5b25a776b51b1b0fa0f544dab2af14cd33d803ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Wed, 29 Jul 2026 15:22:29 +0800 Subject: [PATCH 03/43] Migrate sendEmail field to sendAlert in historical data --- .../mysql/dolphinscheduler_dml.sql | 22 +++++++++++++++++++ .../postgresql/dolphinscheduler_dml.sql | 14 ++++++++++++ .../rpc/TaskExecutorEventListenerImpl.java | 10 +++++---- .../plugin/task/api/model/TaskAlertInfo.java | 1 + 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql index 4a14f326b985..fd841f6a2301 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql @@ -14,3 +14,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +UPDATE t_ds_task_definition +SET task_params = JSON_REMOVE( + JSON_INSERT( + task_params, + '$.sendAlert', + JSON_EXTRACT(task_params, '$.sendEmail') + ), + '$.sendEmail' +) +WHERE task_params IS NOT NULL AND JSON_EXTRACT(task_params, '$.sendEmail') IS NOT NULL; + +UPDATE t_ds_task_definition_log +SET task_params = JSON_REMOVE( + JSON_INSERT( + task_params, + '$.sendAlert', + JSON_EXTRACT(task_params, '$.sendEmail') + ), + '$.sendEmail' +) +WHERE task_params IS NOT NULL AND JSON_EXTRACT(task_params, '$.sendEmail') IS NOT NULL; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql index 4a14f326b985..86ff0204f1df 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql @@ -14,3 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +UPDATE t_ds_task_definition +SET task_params = ( + (task_params::jsonb - 'sendEmail') + || jsonb_build_object('sendAlert', task_params::jsonb->'sendEmail') +)::text +WHERE jsonb_path_exists(task_params::jsonb, '$.sendEmail'); + +UPDATE t_ds_task_definition_log +SET task_params = ( + (task_params::jsonb - 'sendEmail') + || jsonb_build_object('sendAlert', task_params::jsonb->'sendEmail') +)::text +WHERE jsonb_path_exists(task_params::jsonb, '$.sendEmail'); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java index 9421d293ba41..c73035e21487 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java @@ -126,10 +126,12 @@ public void onTaskExecutorSuccess(final TaskExecutorSuccessLifecycleEvent taskEx if (taskExecutorSuccessLifecycleEvent.isNeedAlert()) { TaskAlertInfo taskAlertInfo = taskExecutorSuccessLifecycleEvent.getTaskAlertInfo(); - workflowAlertManager.sendTaskResultAlert( - taskAlertInfo, - taskExecution.getTaskExecutionContext().getProjectCode(), - taskExecution.getTaskExecutionContext().getWorkflowInstanceId()); + if (taskAlertInfo != null) { + workflowAlertManager.sendTaskResultAlert( + taskAlertInfo, + taskExecution.getTaskExecutionContext().getProjectCode(), + taskExecution.getTaskExecutionContext().getWorkflowInstanceId()); + } } } finally { LogUtils.removeWorkflowInstanceIdMDC(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/TaskAlertInfo.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/TaskAlertInfo.java index d6771f6b20ab..d2c22e565a79 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/TaskAlertInfo.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/TaskAlertInfo.java @@ -31,4 +31,5 @@ public class TaskAlertInfo { private Integer alertGroupId; private AlertType alertType; + } From 78f0940795b66f55d0fe1996fa41e20310ae0964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Thu, 30 Jul 2026 11:50:58 +0800 Subject: [PATCH 04/43] refactor prepareTaskResultAlert --- .../plugin/task/api/AbstractTask.java | 21 -------- .../task/api/parameters/SqlParameters.java | 12 +---- .../api/parameters/SqlParametersTest.java | 28 +++++++++-- .../plugin/task/sql/SqlTask.java | 48 ++++++++----------- 4 files changed, 47 insertions(+), 62 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java index 0c8f208fcbae..530205c4556d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java @@ -19,7 +19,6 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import org.apache.dolphinscheduler.plugin.task.api.model.TaskAlertInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import java.util.Map; @@ -56,10 +55,6 @@ public abstract class AbstractTask { */ protected volatile int exitStatusCode = -1; - protected boolean needAlert = false; - - protected TaskAlertInfo taskAlertInfo; - /** * constructor * @@ -109,22 +104,6 @@ public void setAppIds(String appIds) { this.appIds = appIds; } - public boolean getNeedAlert() { - return needAlert; - } - - public void setNeedAlert(boolean needAlert) { - this.needAlert = needAlert; - } - - public TaskAlertInfo getTaskAlertInfo() { - return taskAlertInfo; - } - - public void setTaskAlertInfo(TaskAlertInfo taskAlertInfo) { - this.taskAlertInfo = taskAlertInfo; - } - /** * get task parameters * diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java index 7da491ea9871..84f699a9bb40 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java @@ -39,6 +39,7 @@ import lombok.Data; +import com.fasterxml.jackson.annotation.JsonAlias; import com.google.common.collect.Lists; /** @@ -75,15 +76,6 @@ public class SqlParameters extends AbstractParameters { private int displayRows; - /** - * show type - * 0 TABLE - * 1 TEXT - * 2 attachment - * 3 TABLE+attachment - */ - private String showType; - /** * SQL connection parameters */ @@ -94,6 +86,7 @@ public class SqlParameters extends AbstractParameters { /** * Whether to send alert for SQL query result */ + @JsonAlias("sendEmail") private Boolean sendAlert; /** @@ -187,7 +180,6 @@ public String toString() { + ", sendAlert=" + sendAlert + ", displayRows=" + displayRows + ", limit=" + limit - + ", showType='" + showType + '\'' + ", connParams='" + connParams + '\'' + ", groupId='" + groupId + '\'' + ", title='" + title + '\'' diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParametersTest.java index 39c17615f384..f4cb6f1665e4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParametersTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParametersTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.SQLTaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; @@ -42,7 +43,6 @@ public class SqlParametersTest { private final int sqlType = 0; private final Boolean sendAlert = true; private final int displayRows = 10; - private final String showType = "TABLE"; private final String title = "sql test"; private final int groupId = 0; @@ -65,7 +65,6 @@ public void testSqlParameters() { sqlParameters.setSqlType(sqlType); sqlParameters.setSendAlert(sendAlert); sqlParameters.setDisplayRows(displayRows); - sqlParameters.setShowType(showType); sqlParameters.setTitle(title); sqlParameters.setGroupId(groupId); @@ -75,7 +74,6 @@ public void testSqlParameters() { Assertions.assertEquals(sqlType, sqlParameters.getSqlType()); Assertions.assertEquals(sendAlert, sqlParameters.getSendAlert()); Assertions.assertEquals(displayRows, sqlParameters.getDisplayRows()); - Assertions.assertEquals(showType, sqlParameters.getShowType()); Assertions.assertEquals(title, sqlParameters.getTitle()); Assertions.assertEquals(groupId, sqlParameters.getGroupId()); @@ -145,4 +143,28 @@ public void testGenerateExtendedContext_setsConnectionParams() { Assertions.assertNotNull(ctx); Assertions.assertEquals("conn_params", ctx.getConnectionParams()); } + + @Test + public void testJsonAlias_sendEmail_backwardCompatibility() { + String jsonWithSendEmail = "{\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select 1\",\"sqlType\":0," + + "\"sendEmail\":true,\"displayRows\":10,\"groupId\":2,\"title\":\"test alert\"}"; + + SqlParameters params = JSONUtils.parseObject(jsonWithSendEmail, SqlParameters.class); + Assertions.assertNotNull(params); + Assertions.assertEquals(Boolean.TRUE, params.getSendAlert()); + Assertions.assertEquals(2, params.getGroupId()); + Assertions.assertEquals("test alert", params.getTitle()); + } + + @Test + public void testJson_newFieldName_sendAlert() { + String jsonWithSendAlert = "{\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select 1\",\"sqlType\":0," + + "\"sendAlert\":true,\"displayRows\":10,\"groupId\":3,\"title\":\"new alert\"}"; + + SqlParameters params = JSONUtils.parseObject(jsonWithSendAlert, SqlParameters.class); + Assertions.assertNotNull(params); + Assertions.assertEquals(Boolean.TRUE, params.getSendAlert()); + Assertions.assertEquals(3, params.getGroupId()); + Assertions.assertEquals("new alert", params.getTitle()); + } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index 851efa0e3161..336af6478157 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -124,12 +124,11 @@ public AbstractParameters getParameters() { public void handle(TaskCallBack taskCallBack) throws TaskException { log.info("Full sql parameters: {}", sqlParameters); log.info( - "sql type : {}, datasource : {}, sql : {} , localParams : {},showType : {},connParams : {},varPool : {} ,query max result limit {}", + "sql type : {}, datasource : {}, sql : {} , localParams : {},connParams : {},varPool : {} ,query max result limit {}", sqlParameters.getType(), sqlParameters.getDatasource(), sqlParameters.getSql(), sqlParameters.getLocalParams(), - sqlParameters.getShowType(), sqlParameters.getConnParams(), sqlParameters.getVarPool(), sqlParameters.getLimit()); @@ -292,24 +291,7 @@ private String resultProcess(ResultSet resultSet) throws Exception { : JSONUtils.toJsonString(resultJSONArray); if (Boolean.TRUE.equals(sqlParameters.getSendAlert())) { - // Truncate alert content to avoid oversized payload when query result is large - int displayRows = sqlParameters.getDisplayRows() > 0 ? sqlParameters.getDisplayRows() - : TaskConstants.DEFAULT_DISPLAY_ROWS; - String alertContent; - if (resultJSONArray.size() > displayRows) { - ArrayNode truncatedArray = JSONUtils.createArrayNode(); - for (int i = 0; i < Math.min(displayRows, resultJSONArray.size()); i++) { - truncatedArray.add(resultJSONArray.get(i)); - } - alertContent = JSONUtils.toJsonString(truncatedArray); - log.debug("Alert content truncated to {} rows", displayRows); - } else { - alertContent = result; - } - - prepareTaskResultAlert(sqlParameters.getGroupId(), StringUtils.isNotEmpty(sqlParameters.getTitle()) - ? sqlParameters.getTitle() - : taskExecutionContext.getTaskName() + " query result sets", alertContent); + prepareTaskResultAlert(resultJSONArray); } log.debug("execute sql result : {}", result); return result; @@ -336,18 +318,28 @@ private ArrayNode generateEmptyRow(ResultSet resultSet) throws SQLException { } /** - * Prepare task result alert info + * Prepare task result alert info. + * Truncate the alert content to displayRows to avoid oversized RPC payload. * - * @param alertGroupId alert group id - * @param title alert title - * @param content alert content + * @param resultJSONArray the full query result JSON array */ - private void prepareTaskResultAlert(int alertGroupId, String title, String content) { + private void prepareTaskResultAlert(ArrayNode resultJSONArray) { TaskAlertInfo taskAlertInfo = new TaskAlertInfo(); - taskAlertInfo.setAlertGroupId(alertGroupId); - taskAlertInfo.setContent(content); - taskAlertInfo.setTitle(title); + taskAlertInfo.setAlertGroupId(sqlParameters.getGroupId()); + taskAlertInfo.setTitle(StringUtils.isNotEmpty(sqlParameters.getTitle()) + ? sqlParameters.getTitle() + : taskExecutionContext.getTaskName() + " query result sets"); + // Truncate content to displayRows to avoid oversized RPC payload + int alertRows = sqlParameters.getDisplayRows() > 0 ? sqlParameters.getDisplayRows() + : TaskConstants.DEFAULT_DISPLAY_ROWS; + alertRows = Math.min(alertRows, resultJSONArray.size()); + ArrayNode alertContent = JSONUtils.createArrayNode(); + for (int i = 0; i < alertRows; i++) { + alertContent.add(resultJSONArray.get(i)); + } + taskAlertInfo.setContent(JSONUtils.toJsonString(alertContent)); taskAlertInfo.setAlertType(AlertType.TASK_RESULT); + taskExecutionContext.setNeedAlert(true); taskExecutionContext.setTaskAlertInfo(taskAlertInfo); } From 2b76ad1530add40e0b845b961e19229e396a7631 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Thu, 30 Jul 2026 15:54:29 +0800 Subject: [PATCH 05/43] add sql result log --- .../apache/dolphinscheduler/plugin/task/sql/SqlTask.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index 336af6478157..99f9a5f44ae6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -291,7 +291,10 @@ private String resultProcess(ResultSet resultSet) throws Exception { : JSONUtils.toJsonString(resultJSONArray); if (Boolean.TRUE.equals(sqlParameters.getSendAlert())) { + log.info("SendAlert is enabled, preparing task result alert"); prepareTaskResultAlert(resultJSONArray); + } else { + log.info("SendAlert is not enabled, skip task result alert"); } log.debug("execute sql result : {}", result); return result; @@ -342,6 +345,9 @@ private void prepareTaskResultAlert(ArrayNode resultJSONArray) { taskExecutionContext.setNeedAlert(true); taskExecutionContext.setTaskAlertInfo(taskAlertInfo); + log.info("Prepare task result alert: title={}, alertGroupId={}, alertType={}, totalRows={}, alertRows={}", + taskAlertInfo.getTitle(), taskAlertInfo.getAlertGroupId(), taskAlertInfo.getAlertType(), + resultJSONArray.size(), alertRows); } private String executeQuery(Connection connection, SqlBinds sqlBinds, String handlerType) throws Exception { From 793d28a770bd22fad306597a6d0d3f6e9b7c6011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Tue, 11 Aug 2026 14:34:53 +0800 Subject: [PATCH 06/43] update sendTaskResultAlert --- .../alert/service/AlertSender.java | 4 +-- .../mysql/dolphinscheduler_dml.sql | 24 ++++++++-------- .../postgresql/dolphinscheduler_dml.sql | 4 +-- .../rpc/TaskExecutorEventListenerImpl.java | 13 ++++++--- .../service/alert/WorkflowAlertManager.java | 28 +++++++++++-------- .../task/api/parameters/SqlParameters.java | 4 +++ .../plugin/task/sql/SqlTask.java | 13 +++++++-- 7 files changed, 56 insertions(+), 34 deletions(-) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java index 3ef14c9414a7..7846edf6d983 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.alert.config.AlertConfig; import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; import org.apache.dolphinscheduler.common.enums.AlertStatus; +import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; @@ -58,8 +59,7 @@ public AlertSender(AlertDao alertDao, * @param alertType alertType * @return AlertSendResponseCommand */ - public AlertSendResponse syncHandler(int alertGroupId, String title, String content, - org.apache.dolphinscheduler.common.enums.AlertType alertType) { + public AlertSendResponse syncHandler(int alertGroupId, String title, String content, AlertType alertType) { List alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId); AlertData alertData = AlertData.builder() .content(content) diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql index fd841f6a2301..8ed1912c2338 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql @@ -17,22 +17,22 @@ UPDATE t_ds_task_definition SET task_params = JSON_REMOVE( - JSON_INSERT( - task_params, - '$.sendAlert', - JSON_EXTRACT(task_params, '$.sendEmail') - ), - '$.sendEmail' + JSON_SET( + task_params, + '$.sendAlert', + JSON_EXTRACT(task_params, '$.sendEmail') + ), + '$.sendEmail' ) WHERE task_params IS NOT NULL AND JSON_EXTRACT(task_params, '$.sendEmail') IS NOT NULL; UPDATE t_ds_task_definition_log SET task_params = JSON_REMOVE( - JSON_INSERT( - task_params, - '$.sendAlert', - JSON_EXTRACT(task_params, '$.sendEmail') - ), - '$.sendEmail' + JSON_SET( + task_params, + '$.sendAlert', + JSON_EXTRACT(task_params, '$.sendEmail') + ), + '$.sendEmail' ) WHERE task_params IS NOT NULL AND JSON_EXTRACT(task_params, '$.sendEmail') IS NOT NULL; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql index 86ff0204f1df..ef3ee9b2f4dd 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql @@ -20,11 +20,11 @@ SET task_params = ( (task_params::jsonb - 'sendEmail') || jsonb_build_object('sendAlert', task_params::jsonb->'sendEmail') )::text -WHERE jsonb_path_exists(task_params::jsonb, '$.sendEmail'); +WHERE task_params IS NOT NULL AND jsonb_path_exists(task_params::jsonb, '$.sendEmail'); UPDATE t_ds_task_definition_log SET task_params = ( (task_params::jsonb - 'sendEmail') || jsonb_build_object('sendAlert', task_params::jsonb->'sendEmail') )::text -WHERE jsonb_path_exists(task_params::jsonb, '$.sendEmail'); +WHERE task_params IS NOT NULL AND jsonb_path_exists(task_params::jsonb, '$.sendEmail'); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java index c73035e21487..ebca57911d47 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java @@ -124,13 +124,18 @@ public void onTaskExecutorSuccess(final TaskExecutorSuccessLifecycleEvent taskEx .build(); taskExecution.getWorkflowEventBus().publish(taskSuccessEvent); + // Handle task result alert if (taskExecutorSuccessLifecycleEvent.isNeedAlert()) { TaskAlertInfo taskAlertInfo = taskExecutorSuccessLifecycleEvent.getTaskAlertInfo(); - if (taskAlertInfo != null) { + if (taskAlertInfo != null && taskAlertInfo.getAlertGroupId() != null + && taskAlertInfo.getAlertGroupId() > 0) { workflowAlertManager.sendTaskResultAlert( - taskAlertInfo, - taskExecution.getTaskExecutionContext().getProjectCode(), - taskExecution.getTaskExecutionContext().getWorkflowInstanceId()); + taskExecution.getWorkflowInstance(), + taskExecution.getTaskInstance(), + taskAlertInfo); + } else { + log.warn("Task: {} need alert but alertGroupId is invalid, skip sending alert", + taskExecution.getName()); } } } finally { diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManager.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManager.java index 576ac19377a8..ce6e12be0f20 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManager.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManager.java @@ -210,27 +210,33 @@ public void sendWorkflowTimeoutAlert(WorkflowInstance workflowInstance) { } /** - * Send task result alert + * send task result alert * - * @param taskAlertInfo task alert info - * @param projectCode project code - * @param workflowInstanceId workflow instance id + * @param workflowInstance workflow instance + * @param taskInstance task instance + * @param taskAlertInfo task alert info */ - public void sendTaskResultAlert(TaskAlertInfo taskAlertInfo, Long projectCode, Integer workflowInstanceId) { + public void sendTaskResultAlert(WorkflowInstance workflowInstance, + TaskInstance taskInstance, + TaskAlertInfo taskAlertInfo) { if (taskAlertInfo == null || taskAlertInfo.getAlertGroupId() == null) { return; } + Alert alert = new Alert(); alert.setTitle(taskAlertInfo.getTitle()); alert.setContent(taskAlertInfo.getContent()); - alert.setAlertGroupId(taskAlertInfo.getAlertGroupId()); - alert.setAlertType(taskAlertInfo.getAlertType()); alert.setWarningType(WarningType.SUCCESS); alert.setCreateTime(new Date()); - alert.setProjectCode(projectCode); - alert.setWorkflowInstanceId(workflowInstanceId); + alert.setAlertGroupId(taskAlertInfo.getAlertGroupId()); + alert.setProjectCode(workflowInstance.getProjectCode()); + alert.setWorkflowDefinitionCode(workflowInstance.getWorkflowDefinitionCode()); + alert.setWorkflowInstanceId(workflowInstance.getId()); + alert.setAlertType(taskAlertInfo.getAlertType() != null + ? taskAlertInfo.getAlertType() + : AlertType.TASK_RESULT); alertDao.addAlert(alert); - log.info("Added task result alert for workflow instance: {}", workflowInstanceId); + log.info("Send task result alert for task: {} in workflow: {}", + taskInstance.getName(), workflowInstance.getName()); } - } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java index 84f699a9bb40..22a1f7738cc7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java @@ -38,6 +38,7 @@ import java.util.Set; import lombok.Data; +import lombok.EqualsAndHashCode; import com.fasterxml.jackson.annotation.JsonAlias; import com.google.common.collect.Lists; @@ -46,6 +47,7 @@ * Sql/Hql parameter */ @Data +@EqualsAndHashCode(callSuper = true) public class SqlParameters extends AbstractParameters { /** @@ -80,7 +82,9 @@ public class SqlParameters extends AbstractParameters { * SQL connection parameters */ private String connParams; + private List preStatements; + private List postStatements; /** diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index 99f9a5f44ae6..f71e0ccbf3cc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -287,12 +287,19 @@ private String resultProcess(ResultSet resultSet) throws Exception { } } - String result = resultJSONArray.isEmpty() ? JSONUtils.toJsonString(generateEmptyRow(resultSet)) - : JSONUtils.toJsonString(resultJSONArray); + String result; + ArrayNode alertArray = resultJSONArray; + if (resultJSONArray.isEmpty()) { + ArrayNode emptyRow = generateEmptyRow(resultSet); + result = JSONUtils.toJsonString(emptyRow); + alertArray = emptyRow; + } else { + result = JSONUtils.toJsonString(resultJSONArray); + } if (Boolean.TRUE.equals(sqlParameters.getSendAlert())) { log.info("SendAlert is enabled, preparing task result alert"); - prepareTaskResultAlert(resultJSONArray); + prepareTaskResultAlert(alertArray); } else { log.info("SendAlert is not enabled, skip task result alert"); } From 03aef24f1f32f0db32a99e9e01418f1c92de8d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Wed, 12 Aug 2026 15:39:05 +0800 Subject: [PATCH 07/43] update incompatible.md --- docs/docs/en/guide/upgrade/incompatible.md | 2 +- docs/docs/zh/guide/upgrade/incompatible.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index f927e0603efa..f1cfa3f3e66c 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -47,5 +47,5 @@ This document records the incompatible updates between each version. You need to ## 3.5.0 * Add the `missed_fire_policy` column to `t_ds_schedules`. Existing schedules default to `FIRE_ALL_MISSED` to preserve the previous Quartz `IgnoreMisfires` behavior. ([#18464](https://github.com/apache/dolphinscheduler/pull/18464)) -* Rename the `sendEmail` field in SQL task parameters to `sendAlert`. The data migration script will automatically migrate `sendEmail` to `sendAlert` in the `t_ds_task_definition` and `t_ds_task_definition_log` tables.([#17854])(https://github.com/apache/dolphinscheduler/issues/17854) +* Rename the `sendEmail` field in SQL task parameters to `sendAlert`. The data migration script will automatically migrate `sendEmail` to `sendAlert` in the `t_ds_task_definition` and `t_ds_task_definition_log` tables.([#18549])(https://github.com/apache/dolphinscheduler/pull/18549) diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 7bf33b3065c5..b0f49ee4c522 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -47,5 +47,5 @@ ## 3.5.0 * 为 `t_ds_schedules` 表新增 `missed_fire_policy` 字段。现有定时默认使用 `FIRE_ALL_MISSED`，以保持原有 Quartz `IgnoreMisfires` 行为。([#18464](https://github.com/apache/dolphinscheduler/pull/18464)) -* 将SQL任务参数中的 `sendEmail` 字段重命名为 `sendAlert`，数据迁移脚本会自动将 `t_ds_task_definition` 和 `t_ds_task_definition_log` 表中的 `sendEmail` 迁移为 `sendAlert`。([#17854])(https://github.com/apache/dolphinscheduler/issues/17854) +* 将SQL任务参数中的 `sendEmail` 字段重命名为 `sendAlert`，数据迁移脚本会自动将 `t_ds_task_definition` 和 `t_ds_task_definition_log` 表中的 `sendEmail` 迁移为 `sendAlert`。([#18549])(https://github.com/apache/dolphinscheduler/pull/18549) From 41221d388d29b098c8c5c7effe8de2ac489751cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Wed, 12 Aug 2026 17:13:13 +0800 Subject: [PATCH 08/43] add SqlTask result alert test --- .../plugin/task/sql/SqlTaskTest.java | 238 ++++++++++++++++++ 1 file changed, 238 insertions(+) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java index 92f0fa0702d2..c5668027f6a1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java @@ -20,6 +20,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; @@ -29,6 +30,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType; import org.apache.dolphinscheduler.plugin.task.api.model.Property; +import org.apache.dolphinscheduler.plugin.task.api.model.TaskAlertInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.SqlParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; @@ -489,4 +491,240 @@ private ResourceParametersHelper getResourceParametersHelperWithDatasourceType(D return resourceParametersHelper; } + /** + * Helper: create a SqlTask with sendAlert enabled, invoke prepareTaskResultAlert via reflection, + * and return the TaskExecutionContext for assertions. + */ + private TaskExecutionContext createSqlTaskWithAlert(ArrayNode resultArray, int displayRows, String title, + int groupId) { + String taskParams = "{\"type\":\"HIVE\",\"datasource\":1,\"sql\":\"select 1\"" + + ",\"sendAlert\":true" + + ",\"displayRows\":" + displayRows + + ",\"groupId\":" + groupId + + (title != null ? ",\"title\":\"" + title + "\"" : "") + + "}"; + + TaskExecutionContext ctx = new TaskExecutionContext(); + ctx.setTaskParams(taskParams); + ctx.setTaskName("test_sql_task"); + ctx.setResourceParametersHelper(getResourceParametersHelperWithDatasourceType(DbType.HIVE)); + + SqlTask task = new SqlTask(ctx); + // Call prepareTaskResultAlert via reflection + try { + Method method = SqlTask.class.getDeclaredMethod("prepareTaskResultAlert", ArrayNode.class); + method.setAccessible(true); + method.invoke(task, resultArray); + } catch (Exception e) { + throw new RuntimeException(e); + } + return ctx; + } + + /** + * When sendAlert is true, prepareTaskResultAlert should set needAlert on TaskExecutionContext + * and populate TaskAlertInfo with correct alertGroupId, title, alertType and truncated content. + */ + @Test + void testPrepareTaskResultAlert_setsNeedAlertAndTaskAlertInfo() { + ArrayNode resultArray = JSONUtils.createArrayNode(); + resultArray.add(JSONUtils.parseObject("{\"id\":\"1\",\"name\":\"alice\"}")); + resultArray.add(JSONUtils.parseObject("{\"id\":\"2\",\"name\":\"bob\"}")); + + TaskExecutionContext ctx = createSqlTaskWithAlert(resultArray, 10, "My Alert Title", 5); + + Assertions.assertTrue(ctx.isNeedAlert(), "needAlert should be true when sendAlert is enabled"); + + TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); + Assertions.assertNotNull(alertInfo, "taskAlertInfo should not be null"); + Assertions.assertEquals(5, alertInfo.getAlertGroupId()); + Assertions.assertEquals("My Alert Title", alertInfo.getTitle()); + Assertions.assertEquals(AlertType.TASK_RESULT, alertInfo.getAlertType()); + Assertions.assertNotNull(alertInfo.getContent()); + + // Content should contain all rows (within displayRows limit) + ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); + Assertions.assertEquals(2, contentArray.size()); + } + + /** + * Alert content should be truncated to displayRows to avoid oversized RPC payload. + */ + @Test + void testPrepareTaskResultAlert_truncatesToDisplayRows() { + ArrayNode resultArray = JSONUtils.createArrayNode(); + for (int i = 0; i < 50; i++) { + resultArray.add(JSONUtils.parseObject("{\"id\":\"" + i + "\"}")); + } + + // displayRows = 3, but result has 50 rows -> content should be truncated to 3 + TaskExecutionContext ctx = createSqlTaskWithAlert(resultArray, 3, null, 1); + + TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); + Assertions.assertNotNull(alertInfo); + + ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); + Assertions.assertEquals(3, contentArray.size(), "Alert content should be truncated to displayRows"); + Assertions.assertEquals("0", contentArray.get(0).get("id").asText()); + Assertions.assertEquals("1", contentArray.get(1).get("id").asText()); + Assertions.assertEquals("2", contentArray.get(2).get("id").asText()); + } + + /** + * When displayRows is 0 (unset), should default to TaskConstants.DEFAULT_DISPLAY_ROWS (10). + */ + @Test + void testPrepareTaskResultAlert_usesDefaultDisplayRowsWhenUnset() { + ArrayNode resultArray = JSONUtils.createArrayNode(); + for (int i = 0; i < 20; i++) { + resultArray.add(JSONUtils.parseObject("{\"id\":\"" + i + "\"}")); + } + + // displayRows = 0 means unset -> should default to DEFAULT_DISPLAY_ROWS (10) + TaskExecutionContext ctx = createSqlTaskWithAlert(resultArray, 0, "default rows test", 2); + + TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); + Assertions.assertNotNull(alertInfo); + + ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); + Assertions.assertEquals(TaskConstants.DEFAULT_DISPLAY_ROWS, contentArray.size(), + "Should default to DEFAULT_DISPLAY_ROWS when displayRows is 0"); + } + + /** + * When title is provided, it should be used as the alert title. + */ + @Test + void testPrepareTaskResultAlert_usesCustomTitle() { + ArrayNode resultArray = JSONUtils.createArrayNode(); + resultArray.add(JSONUtils.parseObject("{\"id\":\"1\"}")); + + TaskExecutionContext ctx = createSqlTaskWithAlert(resultArray, 10, "Custom Title", 1); + + TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); + Assertions.assertNotNull(alertInfo); + Assertions.assertEquals("Custom Title", alertInfo.getTitle()); + } + + /** + * When title is empty, should use taskName + " query result sets" as default. + */ + @Test + void testPrepareTaskResultAlert_usesDefaultTitleWhenEmpty() { + ArrayNode resultArray = JSONUtils.createArrayNode(); + resultArray.add(JSONUtils.parseObject("{\"id\":\"1\"}")); + + TaskExecutionContext ctx = createSqlTaskWithAlert(resultArray, 10, null, 1); + + TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); + Assertions.assertNotNull(alertInfo); + Assertions.assertEquals("test_sql_task query result sets", alertInfo.getTitle()); + } + + /** + * When sendAlert is false, resultProcess should NOT set needAlert or taskAlertInfo. + */ + @Test + void testResultProcess_sendAlertDisabled_doesNotSetNeedAlert() throws Exception { + // Build a SqlTask with sendAlert = false + String taskParams = "{\"type\":\"HIVE\",\"datasource\":1,\"sql\":\"select 1\",\"sendAlert\":false}"; + + TaskExecutionContext ctx = new TaskExecutionContext(); + ctx.setTaskParams(taskParams); + ctx.setTaskName("no_alert_task"); + ctx.setResourceParametersHelper(getResourceParametersHelperWithDatasourceType(DbType.HIVE)); + + SqlTask task = new SqlTask(ctx); + + ResultSet mockResultSet = mock(ResultSet.class); + ResultSetMetaData mockMetaData = mock(ResultSetMetaData.class); + when(mockResultSet.getMetaData()).thenReturn(mockMetaData); + when(mockMetaData.getColumnCount()).thenReturn(1); + when(mockMetaData.getColumnLabel(1)).thenReturn("id"); + when(mockResultSet.next()).thenReturn(true, false); + when(mockResultSet.getObject(1)).thenReturn("1"); + + Method resultProcessMethod = SqlTask.class.getDeclaredMethod("resultProcess", ResultSet.class); + resultProcessMethod.setAccessible(true); + resultProcessMethod.invoke(task, mockResultSet); + + Assertions.assertFalse(ctx.isNeedAlert(), "needAlert should remain false when sendAlert is disabled"); + Assertions.assertNull(ctx.getTaskAlertInfo(), "taskAlertInfo should remain null when sendAlert is disabled"); + } + + /** + * When the query result is empty, resultProcess should still prepare the alert + * using the generated empty row so the user is notified that the query returned no data. + */ + @Test + void testResultProcess_emptyResultSet_prepareAlertWithEmptyRow() throws Exception { + String taskParams = "{\"type\":\"HIVE\",\"datasource\":1,\"sql\":\"select 1\"" + + ",\"sendAlert\":true,\"displayRows\":10,\"groupId\":3,\"title\":\"empty result\"}"; + + TaskExecutionContext ctx = new TaskExecutionContext(); + ctx.setTaskParams(taskParams); + ctx.setTaskName("empty_result_task"); + ctx.setResourceParametersHelper(getResourceParametersHelperWithDatasourceType(DbType.HIVE)); + + SqlTask task = new SqlTask(ctx); + + ResultSet mockResultSet = mock(ResultSet.class); + ResultSetMetaData mockMetaData = mock(ResultSetMetaData.class); + when(mockResultSet.getMetaData()).thenReturn(mockMetaData); + when(mockMetaData.getColumnCount()).thenReturn(2); + when(mockMetaData.getColumnLabel(1)).thenReturn("id"); + when(mockMetaData.getColumnLabel(2)).thenReturn("name"); + when(mockResultSet.next()).thenReturn(false); // empty result set + + Method resultProcessMethod = SqlTask.class.getDeclaredMethod("resultProcess", ResultSet.class); + resultProcessMethod.setAccessible(true); + resultProcessMethod.invoke(task, mockResultSet); + + Assertions.assertTrue(ctx.isNeedAlert(), "needAlert should be true even for empty result set"); + + TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); + Assertions.assertNotNull(alertInfo); + Assertions.assertEquals(3, alertInfo.getAlertGroupId()); + Assertions.assertEquals("empty result", alertInfo.getTitle()); + Assertions.assertEquals(AlertType.TASK_RESULT, alertInfo.getAlertType()); + + // Content should be the empty row + ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); + Assertions.assertEquals(1, contentArray.size(), "Empty result alert should contain one empty row"); + Assertions.assertEquals("", contentArray.get(0).get("id").asText()); + Assertions.assertEquals("", contentArray.get(0).get("name").asText()); + } + + /** + * Verify that TaskExecutorSuccessLifecycleEvent.of() carries needAlert and taskAlertInfo + * from TaskExecutionContext to the event, ensuring the alert info survives the Worker→Master RPC. + * Since task-executor module has no test infrastructure, we validate indirectly by confirming + * that TaskExecutionContext properly stores and exposes these fields for the event builder to read. + */ + @Test + void testTaskExecutionContext_carriesAlertInfoForEventPropagation() { + // Simulate what SqlTask.prepareTaskResultAlert does + TaskExecutionContext ctx = new TaskExecutionContext(); + ctx.setNeedAlert(true); + + TaskAlertInfo alertInfo = new TaskAlertInfo(); + alertInfo.setTitle("test title"); + alertInfo.setContent("[{\"id\":\"1\"}]"); + alertInfo.setAlertGroupId(7); + alertInfo.setAlertType(AlertType.TASK_RESULT); + ctx.setTaskAlertInfo(alertInfo); + + // Verify the fields are retrievable — this is what TaskExecutorSuccessLifecycleEvent.of() reads + Assertions.assertTrue(ctx.isNeedAlert()); + Assertions.assertNotNull(ctx.getTaskAlertInfo()); + Assertions.assertEquals("test title", ctx.getTaskAlertInfo().getTitle()); + Assertions.assertEquals(7, ctx.getTaskAlertInfo().getAlertGroupId()); + Assertions.assertEquals(AlertType.TASK_RESULT, ctx.getTaskAlertInfo().getAlertType()); + + // Verify default state when alert is not set + TaskExecutionContext ctx2 = new TaskExecutionContext(); + Assertions.assertFalse(ctx2.isNeedAlert()); + Assertions.assertNull(ctx2.getTaskAlertInfo()); + } + } From 0be063a12c4e2c1c6634b44233bd8fd0e267946b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Wed, 12 Aug 2026 19:21:31 +0800 Subject: [PATCH 09/43] update incompatible.md --- docs/docs/en/guide/upgrade/incompatible.md | 2 +- docs/docs/zh/guide/upgrade/incompatible.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index f1cfa3f3e66c..0974181ed15a 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -47,5 +47,5 @@ This document records the incompatible updates between each version. You need to ## 3.5.0 * Add the `missed_fire_policy` column to `t_ds_schedules`. Existing schedules default to `FIRE_ALL_MISSED` to preserve the previous Quartz `IgnoreMisfires` behavior. ([#18464](https://github.com/apache/dolphinscheduler/pull/18464)) -* Rename the `sendEmail` field in SQL task parameters to `sendAlert`. The data migration script will automatically migrate `sendEmail` to `sendAlert` in the `t_ds_task_definition` and `t_ds_task_definition_log` tables.([#18549])(https://github.com/apache/dolphinscheduler/pull/18549) +* Rename the `sendEmail` field in SQL task parameters to `sendAlert`. The data migration script will automatically migrate `sendEmail` to `sendAlert` in the `t_ds_task_definition` and `t_ds_task_definition_log` tables.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index b0f49ee4c522..9ec0258dde8e 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -47,5 +47,5 @@ ## 3.5.0 * 为 `t_ds_schedules` 表新增 `missed_fire_policy` 字段。现有定时默认使用 `FIRE_ALL_MISSED`，以保持原有 Quartz `IgnoreMisfires` 行为。([#18464](https://github.com/apache/dolphinscheduler/pull/18464)) -* 将SQL任务参数中的 `sendEmail` 字段重命名为 `sendAlert`，数据迁移脚本会自动将 `t_ds_task_definition` 和 `t_ds_task_definition_log` 表中的 `sendEmail` 迁移为 `sendAlert`。([#18549])(https://github.com/apache/dolphinscheduler/pull/18549) +* 将SQL任务参数中的 `sendEmail` 字段重命名为 `sendAlert`，数据迁移脚本会自动将 `t_ds_task_definition` 和 `t_ds_task_definition_log` 表中的 `sendEmail` 迁移为 `sendAlert`。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) From 68b793411cb1a53c6af6bf00e8c1585bd83e84e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Fri, 14 Aug 2026 16:02:33 +0800 Subject: [PATCH 10/43] Revert sendEmail field rename to sendAlert in SQL task params --- docs/docs/en/guide/upgrade/incompatible.md | 15 ++++++-- docs/docs/zh/guide/upgrade/incompatible.md | 13 ++++++- .../mysql/dolphinscheduler_dml.sql | 22 ------------ .../postgresql/dolphinscheduler_dml.sql | 14 -------- .../task/api/parameters/SqlParameters.java | 31 +++++++---------- .../api/parameters/SqlParametersTest.java | 34 ++++--------------- .../plugin/task/sql/SqlTask.java | 5 +-- .../plugin/task/sql/SqlTaskTest.java | 22 ++++++------ .../components/node/fields/use-sql-type.ts | 14 ++++---- .../task/components/node/format-data.ts | 4 +-- .../projects/task/components/node/types.ts | 2 +- 11 files changed, 67 insertions(+), 109 deletions(-) diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index 0974181ed15a..24dca2a5456d 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -1,3 +1,14 @@ +--- +AIGC: + ContentProducer: '001191110102MAD55U9H0F10002' + ContentPropagator: '001191110102MAD55U9H0F10002' + Label: '1' + ProduceID: 'e01c22d1-119f-42d0-9027-8ee822ee6fa7' + PropagateID: 'e01c22d1-119f-42d0-9027-8ee822ee6fa7' + ReservedCode1: 'e99cdbc1-b610-4b6f-a39f-19aed3cfc159' + ReservedCode2: 'e99cdbc1-b610-4b6f-a39f-19aed3cfc159' +--- + # Incompatible This document records the incompatible updates between each version. You need to check this document before you upgrade to related version. @@ -46,6 +57,6 @@ This document records the incompatible updates between each version. You need to ## 3.5.0 -* Add the `missed_fire_policy` column to `t_ds_schedules`. Existing schedules default to `FIRE_ALL_MISSED` to preserve the previous Quartz `IgnoreMisfires` behavior. ([#18464](https://github.com/apache/dolphinscheduler/pull/18464)) -* Rename the `sendEmail` field in SQL task parameters to `sendAlert`. The data migration script will automatically migrate `sendEmail` to `sendAlert` in the `t_ds_task_definition` and `t_ds_task_definition_log` tables.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* Add the `missed_fire_policy` column to `t_d_schedules`. Existing schedules default to `FIRE_ALL_MISSED` to preserve the previous Quartz `IgnoreMisfires` behavior. ([#18464](https://github.com/apache/dolphinscheduler/pull/18464)) +> AI生成 \ No newline at end of file diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 9ec0258dde8e..74f52eacc8fe 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -1,3 +1,14 @@ +--- +AIGC: + ContentProducer: '001191110102MAD55U9H0F10002' + ContentPropagator: '001191110102MAD55U9H0F10002' + Label: '1' + ProduceID: '0f42b663-53c1-4f98-ae46-0c8879f2a3d9' + PropagateID: '0f42b663-53c1-4f98-ae46-0c8879f2a3d9' + ReservedCode1: '840b53b9-f732-40dd-8e74-1a0c2fcf4622' + ReservedCode2: '840b53b9-f732-40dd-8e74-1a0c2fcf4622' +--- + # 不向前兼容的更新 本文档记录了各版本之间不兼容的更新内容。在升级到相关版本前，请检查本文档。 @@ -47,5 +58,5 @@ ## 3.5.0 * 为 `t_ds_schedules` 表新增 `missed_fire_policy` 字段。现有定时默认使用 `FIRE_ALL_MISSED`，以保持原有 Quartz `IgnoreMisfires` 行为。([#18464](https://github.com/apache/dolphinscheduler/pull/18464)) -* 将SQL任务参数中的 `sendEmail` 字段重命名为 `sendAlert`，数据迁移脚本会自动将 `t_ds_task_definition` 和 `t_ds_task_definition_log` 表中的 `sendEmail` 迁移为 `sendAlert`。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +> AI生成 \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql index 8ed1912c2338..4a14f326b985 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql @@ -14,25 +14,3 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -UPDATE t_ds_task_definition -SET task_params = JSON_REMOVE( - JSON_SET( - task_params, - '$.sendAlert', - JSON_EXTRACT(task_params, '$.sendEmail') - ), - '$.sendEmail' -) -WHERE task_params IS NOT NULL AND JSON_EXTRACT(task_params, '$.sendEmail') IS NOT NULL; - -UPDATE t_ds_task_definition_log -SET task_params = JSON_REMOVE( - JSON_SET( - task_params, - '$.sendAlert', - JSON_EXTRACT(task_params, '$.sendEmail') - ), - '$.sendEmail' -) -WHERE task_params IS NOT NULL AND JSON_EXTRACT(task_params, '$.sendEmail') IS NOT NULL; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql index ef3ee9b2f4dd..4a14f326b985 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql @@ -14,17 +14,3 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -UPDATE t_ds_task_definition -SET task_params = ( - (task_params::jsonb - 'sendEmail') - || jsonb_build_object('sendAlert', task_params::jsonb->'sendEmail') -)::text -WHERE task_params IS NOT NULL AND jsonb_path_exists(task_params::jsonb, '$.sendEmail'); - -UPDATE t_ds_task_definition_log -SET task_params = ( - (task_params::jsonb - 'sendEmail') - || jsonb_build_object('sendAlert', task_params::jsonb->'sendEmail') -)::text -WHERE task_params IS NOT NULL AND jsonb_path_exists(task_params::jsonb, '$.sendEmail'); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java index 22a1f7738cc7..32b110255cf9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParameters.java @@ -38,16 +38,13 @@ import java.util.Set; import lombok.Data; -import lombok.EqualsAndHashCode; -import com.fasterxml.jackson.annotation.JsonAlias; import com.google.common.collect.Lists; /** * Sql/Hql parameter */ @Data -@EqualsAndHashCode(callSuper = true) public class SqlParameters extends AbstractParameters { /** @@ -76,31 +73,26 @@ public class SqlParameters extends AbstractParameters { */ private int sqlType; + private Boolean sendEmail; + private int displayRows; + /** + * show type + * 0 TABLE + * 1 TEXT + * 2 attachment + * 3 TABLE+attachment + */ + private String showType; /** * SQL connection parameters */ private String connParams; - private List preStatements; - private List postStatements; - /** - * Whether to send alert for SQL query result - */ - @JsonAlias("sendEmail") - private Boolean sendAlert; - - /** - * Alert group id - */ private int groupId; - - /** - * Alert title - */ private String title; private int limit; @@ -181,9 +173,10 @@ public String toString() { + ", sqlSource='" + sqlSource + '\'' + ", sqlResource='" + sqlResource + '\'' + ", sqlType=" + sqlType - + ", sendAlert=" + sendAlert + + ", sendEmail=" + sendEmail + ", displayRows=" + displayRows + ", limit=" + limit + + ", showType='" + showType + '\'' + ", connParams='" + connParams + '\'' + ", groupId='" + groupId + '\'' + ", title='" + title + '\'' diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParametersTest.java index f4cb6f1665e4..b9388b3ba54f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParametersTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SqlParametersTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters; -import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.SQLTaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; @@ -41,8 +40,9 @@ public class SqlParametersTest { private final String sql = "select * from t_ds_user"; private final int datasource = 1; private final int sqlType = 0; - private final Boolean sendAlert = true; + private final Boolean sendEmail = true; private final int displayRows = 10; + private final String showType = "TABLE"; private final String title = "sql test"; private final int groupId = 0; @@ -63,8 +63,9 @@ public void testSqlParameters() { sqlParameters.setSql(sql); sqlParameters.setDatasource(datasource); sqlParameters.setSqlType(sqlType); - sqlParameters.setSendAlert(sendAlert); + sqlParameters.setSendEmail(sendEmail); sqlParameters.setDisplayRows(displayRows); + sqlParameters.setShowType(showType); sqlParameters.setTitle(title); sqlParameters.setGroupId(groupId); @@ -72,8 +73,9 @@ public void testSqlParameters() { Assertions.assertEquals(sql, sqlParameters.getSql()); Assertions.assertEquals(datasource, sqlParameters.getDatasource()); Assertions.assertEquals(sqlType, sqlParameters.getSqlType()); - Assertions.assertEquals(sendAlert, sqlParameters.getSendAlert()); + Assertions.assertEquals(sendEmail, sqlParameters.getSendEmail()); Assertions.assertEquals(displayRows, sqlParameters.getDisplayRows()); + Assertions.assertEquals(showType, sqlParameters.getShowType()); Assertions.assertEquals(title, sqlParameters.getTitle()); Assertions.assertEquals(groupId, sqlParameters.getGroupId()); @@ -143,28 +145,4 @@ public void testGenerateExtendedContext_setsConnectionParams() { Assertions.assertNotNull(ctx); Assertions.assertEquals("conn_params", ctx.getConnectionParams()); } - - @Test - public void testJsonAlias_sendEmail_backwardCompatibility() { - String jsonWithSendEmail = "{\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select 1\",\"sqlType\":0," - + "\"sendEmail\":true,\"displayRows\":10,\"groupId\":2,\"title\":\"test alert\"}"; - - SqlParameters params = JSONUtils.parseObject(jsonWithSendEmail, SqlParameters.class); - Assertions.assertNotNull(params); - Assertions.assertEquals(Boolean.TRUE, params.getSendAlert()); - Assertions.assertEquals(2, params.getGroupId()); - Assertions.assertEquals("test alert", params.getTitle()); - } - - @Test - public void testJson_newFieldName_sendAlert() { - String jsonWithSendAlert = "{\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select 1\",\"sqlType\":0," - + "\"sendAlert\":true,\"displayRows\":10,\"groupId\":3,\"title\":\"new alert\"}"; - - SqlParameters params = JSONUtils.parseObject(jsonWithSendAlert, SqlParameters.class); - Assertions.assertNotNull(params); - Assertions.assertEquals(Boolean.TRUE, params.getSendAlert()); - Assertions.assertEquals(3, params.getGroupId()); - Assertions.assertEquals("new alert", params.getTitle()); - } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index f71e0ccbf3cc..a3516d1e9ade 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -124,11 +124,12 @@ public AbstractParameters getParameters() { public void handle(TaskCallBack taskCallBack) throws TaskException { log.info("Full sql parameters: {}", sqlParameters); log.info( - "sql type : {}, datasource : {}, sql : {} , localParams : {},connParams : {},varPool : {} ,query max result limit {}", + "sql type : {}, datasource : {}, sql : {} , localParams : {},showType : {},connParams : {},varPool : {} ,query max result limit {}", sqlParameters.getType(), sqlParameters.getDatasource(), sqlParameters.getSql(), sqlParameters.getLocalParams(), + sqlParameters.getShowType(), sqlParameters.getConnParams(), sqlParameters.getVarPool(), sqlParameters.getLimit()); @@ -297,7 +298,7 @@ private String resultProcess(ResultSet resultSet) throws Exception { result = JSONUtils.toJsonString(resultJSONArray); } - if (Boolean.TRUE.equals(sqlParameters.getSendAlert())) { + if (Boolean.TRUE.equals(sqlParameters.getSendEmail())) { log.info("SendAlert is enabled, preparing task result alert"); prepareTaskResultAlert(alertArray); } else { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java index c5668027f6a1..ce2b7945f7fa 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java @@ -492,13 +492,13 @@ private ResourceParametersHelper getResourceParametersHelperWithDatasourceType(D } /** - * Helper: create a SqlTask with sendAlert enabled, invoke prepareTaskResultAlert via reflection, + * Helper: create a SqlTask with sendEmail enabled, invoke prepareTaskResultAlert via reflection, * and return the TaskExecutionContext for assertions. */ private TaskExecutionContext createSqlTaskWithAlert(ArrayNode resultArray, int displayRows, String title, int groupId) { String taskParams = "{\"type\":\"HIVE\",\"datasource\":1,\"sql\":\"select 1\"" - + ",\"sendAlert\":true" + + ",\"sendEmail\":true" + ",\"displayRows\":" + displayRows + ",\"groupId\":" + groupId + (title != null ? ",\"title\":\"" + title + "\"" : "") @@ -522,7 +522,7 @@ private TaskExecutionContext createSqlTaskWithAlert(ArrayNode resultArray, int d } /** - * When sendAlert is true, prepareTaskResultAlert should set needAlert on TaskExecutionContext + * When sendEmail is true, prepareTaskResultAlert should set needAlert on TaskExecutionContext * and populate TaskAlertInfo with correct alertGroupId, title, alertType and truncated content. */ @Test @@ -533,7 +533,7 @@ void testPrepareTaskResultAlert_setsNeedAlertAndTaskAlertInfo() { TaskExecutionContext ctx = createSqlTaskWithAlert(resultArray, 10, "My Alert Title", 5); - Assertions.assertTrue(ctx.isNeedAlert(), "needAlert should be true when sendAlert is enabled"); + Assertions.assertTrue(ctx.isNeedAlert(), "needAlert should be true when sendEmail is enabled"); TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); Assertions.assertNotNull(alertInfo, "taskAlertInfo should not be null"); @@ -622,12 +622,12 @@ void testPrepareTaskResultAlert_usesDefaultTitleWhenEmpty() { } /** - * When sendAlert is false, resultProcess should NOT set needAlert or taskAlertInfo. + * When sendEmail is false, resultProcess should NOT set needAlert or taskAlertInfo. */ @Test - void testResultProcess_sendAlertDisabled_doesNotSetNeedAlert() throws Exception { - // Build a SqlTask with sendAlert = false - String taskParams = "{\"type\":\"HIVE\",\"datasource\":1,\"sql\":\"select 1\",\"sendAlert\":false}"; + void testResultProcess_sendEmailDisabled_doesNotSetNeedAlert() throws Exception { + // Build a SqlTask with sendEmail = false + String taskParams = "{\"type\":\"HIVE\",\"datasource\":1,\"sql\":\"select 1\",\"sendEmail\":false}"; TaskExecutionContext ctx = new TaskExecutionContext(); ctx.setTaskParams(taskParams); @@ -648,8 +648,8 @@ void testResultProcess_sendAlertDisabled_doesNotSetNeedAlert() throws Exception resultProcessMethod.setAccessible(true); resultProcessMethod.invoke(task, mockResultSet); - Assertions.assertFalse(ctx.isNeedAlert(), "needAlert should remain false when sendAlert is disabled"); - Assertions.assertNull(ctx.getTaskAlertInfo(), "taskAlertInfo should remain null when sendAlert is disabled"); + Assertions.assertFalse(ctx.isNeedAlert(), "needAlert should remain false when sendEmail is disabled"); + Assertions.assertNull(ctx.getTaskAlertInfo(), "taskAlertInfo should remain null when sendEmail is disabled"); } /** @@ -659,7 +659,7 @@ void testResultProcess_sendAlertDisabled_doesNotSetNeedAlert() throws Exception @Test void testResultProcess_emptyResultSet_prepareAlertWithEmptyRow() throws Exception { String taskParams = "{\"type\":\"HIVE\",\"datasource\":1,\"sql\":\"select 1\"" - + ",\"sendAlert\":true,\"displayRows\":10,\"groupId\":3,\"title\":\"empty result\"}"; + + ",\"sendEmail\":true,\"displayRows\":10,\"groupId\":3,\"title\":\"empty result\"}"; TaskExecutionContext ctx = new TaskExecutionContext(); ctx.setTaskParams(taskParams); diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sql-type.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sql-type.ts index b9ac314ff3d4..d30ef9ef3221 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sql-type.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-sql-type.ts @@ -24,8 +24,8 @@ import type { IJsonItem } from '../types' export function useSqlType(model: { [field: string]: any }): IJsonItem[] { const { t } = useI18n() const querySpan = computed(() => (model.sqlType === '0' ? 6 : 0)) - const alertSpan = computed(() => - model.sqlType === '0' && model.sendAlert ? 24 : 0 + const emailSpan = computed(() => + model.sqlType === '0' && model.sendEmail ? 24 : 0 ) const groups = ref([]) const groupsLoading = ref(false) @@ -69,7 +69,7 @@ export function useSqlType(model: { [field: string]: any }): IJsonItem[] { }, { type: 'switch', - field: 'sendAlert', + field: 'sendEmail', span: querySpan, name: t('project.node.send_alarm') }, @@ -109,12 +109,12 @@ export function useSqlType(model: { [field: string]: any }): IJsonItem[] { props: { placeholder: t('project.node.title_tips') }, - span: alertSpan, + span: emailSpan, validate: { trigger: ['input', 'blur'], required: true, validator(unuse, value) { - if (model.sendAlert && !value) + if (model.sendEmail && !value) return new Error(t('project.node.title_tips')) } } @@ -124,7 +124,7 @@ export function useSqlType(model: { [field: string]: any }): IJsonItem[] { field: 'groupId', name: t('project.node.alarm_group'), options: groups, - span: alertSpan, + span: emailSpan, props: { loading: groupsLoading, placeholder: t('project.node.alarm_group_tips') @@ -133,7 +133,7 @@ export function useSqlType(model: { [field: string]: any }): IJsonItem[] { trigger: ['input', 'blur'], required: true, validator(unuse, value) { - if (model.sendAlert && !value) + if (model.sendEmail && !value) return new Error(t('project.node.alarm_group_tips')) } } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index 8f69b03a50eb..7eace75e527f 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -210,9 +210,9 @@ export function formatParams(data: INodeData): { taskParams.sqlType = data.sqlType taskParams.preStatements = data.preStatements taskParams.postStatements = data.postStatements - taskParams.sendAlert = data.sendAlert + taskParams.sendEmail = data.sendEmail taskParams.displayRows = data.displayRows - if (data.sqlType === '0' && data.sendAlert) { + if (data.sqlType === '0' && data.sendEmail) { taskParams.title = data.title taskParams.groupId = data.groupId } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts index 52b2d0233d1d..64aec7690bb4 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts @@ -307,7 +307,7 @@ interface ITaskParams { sqlSource?: string sqlResource?: string sqlType?: string - sendAlert?: boolean + sendEmail?: boolean displayRows?: number title?: string groupId?: string From 84e68ee4651c36f0a2adc3d081a83a36cf4181d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Fri, 14 Aug 2026 16:37:37 +0800 Subject: [PATCH 11/43] revert AlertSendRequest --- docs/docs/en/guide/upgrade/incompatible.md | 14 +------------- docs/docs/zh/guide/upgrade/incompatible.md | 12 ------------ .../alert/rpc/AlertOperatorImpl.java | 3 +-- .../alert/service/AlertSender.java | 5 +---- .../alert/runner/AlertSenderTest.java | 11 +++++------ .../extract/alert/request/AlertSendRequest.java | 4 +--- 6 files changed, 9 insertions(+), 40 deletions(-) diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index 24dca2a5456d..026d9b88bbcb 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -1,14 +1,3 @@ ---- -AIGC: - ContentProducer: '001191110102MAD55U9H0F10002' - ContentPropagator: '001191110102MAD55U9H0F10002' - Label: '1' - ProduceID: 'e01c22d1-119f-42d0-9027-8ee822ee6fa7' - PropagateID: 'e01c22d1-119f-42d0-9027-8ee822ee6fa7' - ReservedCode1: 'e99cdbc1-b610-4b6f-a39f-19aed3cfc159' - ReservedCode2: 'e99cdbc1-b610-4b6f-a39f-19aed3cfc159' ---- - # Incompatible This document records the incompatible updates between each version. You need to check this document before you upgrade to related version. @@ -57,6 +46,5 @@ This document records the incompatible updates between each version. You need to ## 3.5.0 -* Add the `missed_fire_policy` column to `t_d_schedules`. Existing schedules default to `FIRE_ALL_MISSED` to preserve the previous Quartz `IgnoreMisfires` behavior. ([#18464](https://github.com/apache/dolphinscheduler/pull/18464)) +* Add the `missed_fire_policy` column to `t_ds_schedules`. Existing schedules default to `FIRE_ALL_MISSED` to preserve the previous Quartz `IgnoreMisfires` behavior. ([#18464](https://github.com/apache/dolphinscheduler/pull/18464)) -> AI生成 \ No newline at end of file diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 74f52eacc8fe..db0ea2d4c5f1 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -1,14 +1,3 @@ ---- -AIGC: - ContentProducer: '001191110102MAD55U9H0F10002' - ContentPropagator: '001191110102MAD55U9H0F10002' - Label: '1' - ProduceID: '0f42b663-53c1-4f98-ae46-0c8879f2a3d9' - PropagateID: '0f42b663-53c1-4f98-ae46-0c8879f2a3d9' - ReservedCode1: '840b53b9-f732-40dd-8e74-1a0c2fcf4622' - ReservedCode2: '840b53b9-f732-40dd-8e74-1a0c2fcf4622' ---- - # 不向前兼容的更新 本文档记录了各版本之间不兼容的更新内容。在升级到相关版本前，请检查本文档。 @@ -59,4 +48,3 @@ AIGC: * 为 `t_ds_schedules` 表新增 `missed_fire_policy` 字段。现有定时默认使用 `FIRE_ALL_MISSED`，以保持原有 Quartz `IgnoreMisfires` 行为。([#18464](https://github.com/apache/dolphinscheduler/pull/18464)) -> AI生成 \ No newline at end of file diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java index 2e8788ab49f5..95b3809dd20b 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java @@ -41,8 +41,7 @@ public AlertSendResponse sendAlert(AlertSendRequest alertSendRequest) { AlertSendResponse alertSendResponse = alertSender.syncHandler( alertSendRequest.getGroupId(), alertSendRequest.getTitle(), - alertSendRequest.getContent(), - alertSendRequest.getAlertType()); + alertSendRequest.getContent()); log.info("Handle AlertSendRequest finish: {}", alertSendResponse); return alertSendResponse; } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java index 7846edf6d983..9c9cd034bdb6 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java @@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.alert.config.AlertConfig; import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager; import org.apache.dolphinscheduler.common.enums.AlertStatus; -import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance; @@ -56,15 +55,13 @@ public AlertSender(AlertDao alertDao, * @param alertGroupId alertGroupId * @param title title * @param content content - * @param alertType alertType * @return AlertSendResponseCommand */ - public AlertSendResponse syncHandler(int alertGroupId, String title, String content, AlertType alertType) { + public AlertSendResponse syncHandler(int alertGroupId, String title, String content) { List alertInstanceList = alertDao.listInstanceByAlertGroupId(alertGroupId); AlertData alertData = AlertData.builder() .content(content) .title(title) - .alertType(alertType.getCode()) .build(); boolean sendResponseStatus = true; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java index 5ba3c6df25d9..18246f485ab2 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java @@ -89,8 +89,7 @@ void testSyncHandler() { // 1.alert instance does not exist when(alertDao.listInstanceByAlertGroupId(ALERT_GROUP_ID)).thenReturn(null); - AlertSendResponse alertSendResponse = - alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, AlertType.TASK_RESULT); + AlertSendResponse alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); @@ -105,7 +104,7 @@ void testSyncHandler() { alertInstanceList.add(alertPluginInstance); when(alertDao.listInstanceByAlertGroupId(1)).thenReturn(alertInstanceList); - alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, AlertType.TASK_RESULT); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); @@ -115,7 +114,7 @@ void testSyncHandler() { when(alertChannelMock.process(Mockito.any())).thenReturn(null); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, AlertType.TASK_RESULT); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); @@ -127,7 +126,7 @@ void testSyncHandler() { when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, AlertType.TASK_RESULT); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertFalse(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); @@ -139,7 +138,7 @@ void testSyncHandler() { when(alertChannelMock.process(Mockito.any())).thenReturn(alertResult); when(alertPluginManager.getAlertChannel(1)).thenReturn(Optional.of(alertChannelMock)); - alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT, AlertType.TASK_RESULT); + alertSendResponse = alertSender.syncHandler(ALERT_GROUP_ID, TITLE, CONTENT); Assertions.assertTrue(alertSendResponse.isSuccess()); alertSendResponse.getResResults().forEach(result -> logger .info("alert send response result, status:{}, message:{}", result.isSuccess(), result.getMessage())); diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendRequest.java index 82996bfc6c2f..dfa6515ddea2 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendRequest.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendRequest.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.extract.alert.request; -import org.apache.dolphinscheduler.common.enums.AlertType; - import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -34,6 +32,6 @@ public class AlertSendRequest { private String content; - private AlertType alertType; + private int warnType; } From 67f061a9a39757b3413f826e217641be0aa25877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Fri, 14 Aug 2026 17:14:47 +0800 Subject: [PATCH 12/43] revert TaskDefinitionMapperTest --- .../dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java index 83fc14e5fec7..da3cc1d27481 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java @@ -140,7 +140,7 @@ public void testDeleteByCode() { @Test public void testNullPropertyValueOfLocalParams() { String definitionJson = - "{\"failRetryTimes\":\"0\",\"timeoutNotifyStrategy\":\"\",\"code\":\"5195043558720\",\"flag\":\"YES\",\"environmentCode\":\"-1\",\"taskDefinitionIndex\":2,\"taskPriority\":\"MEDIUM\",\"taskParams\":\"{\\\"preStatements\\\":null,\\\"postStatements\\\":null,\\\"type\\\":\\\"ADB_MYSQL\\\",\\\"database\\\":\\\"lijia\\\",\\\"sql\\\":\\\"create table nation_${random_serial_number} as select * from nation\\\",\\\"localParams\\\":[{\\\"direct\\\":2,\\\"type\\\":3,\\\"prop\\\":\\\"key\\\"}],\\\"Name\\\":\\\"create_table_as_select_nation\\\",\\\"FailRetryTimes\\\":0,\\\"dbClusterId\\\":\\\"amv-bp10o45925jpx959\\\",\\\"sendAlert\\\":false,\\\"displayRows\\\":10,\\\"limit\\\":10000,\\\"agentSource\\\":\\\"Workflow\\\",\\\"agentVersion\\\":\\\"Unkown\\\"}\",\"timeout\":\"0\",\"taskType\":\"ADB_MYSQL\",\"timeoutFlag\":\"CLOSE\",\"projectCode\":\"5191800302720\",\"name\":\"create_table_as_select_nation\",\"delayTime\":\"0\",\"workerGroup\":\"default\"}"; + "{\"failRetryTimes\":\"0\",\"timeoutNotifyStrategy\":\"\",\"code\":\"5195043558720\",\"flag\":\"YES\",\"environmentCode\":\"-1\",\"taskDefinitionIndex\":2,\"taskPriority\":\"MEDIUM\",\"taskParams\":\"{\\\"preStatements\\\":null,\\\"postStatements\\\":null,\\\"type\\\":\\\"ADB_MYSQL\\\",\\\"database\\\":\\\"lijia\\\",\\\"sql\\\":\\\"create table nation_${random_serial_number} as select * from nation\\\",\\\"localParams\\\":[{\\\"direct\\\":2,\\\"type\\\":3,\\\"prop\\\":\\\"key\\\"}],\\\"Name\\\":\\\"create_table_as_select_nation\\\",\\\"FailRetryTimes\\\":0,\\\"dbClusterId\\\":\\\"amv-bp10o45925jpx959\\\",\\\"sendEmail\\\":false,\\\"displayRows\\\":10,\\\"limit\\\":10000,\\\"agentSource\\\":\\\"Workflow\\\",\\\"agentVersion\\\":\\\"Unkown\\\"}\",\"timeout\":\"0\",\"taskType\":\"ADB_MYSQL\",\"timeoutFlag\":\"CLOSE\",\"projectCode\":\"5191800302720\",\"name\":\"create_table_as_select_nation\",\"delayTime\":\"0\",\"workerGroup\":\"default\"}"; TaskDefinition definition = JSONUtils.parseObject(definitionJson, TaskDefinition.class); Map taskParamsMap = definition.getTaskParamMap(); @@ -157,7 +157,7 @@ public void testNullPropertyValueOfLocalParams() { @Test public void testNullLocalParamsOfTaskParams() { String definitionJson = - "{\"failRetryTimes\":\"0\",\"timeoutNotifyStrategy\":\"\",\"code\":\"5195043558720\",\"flag\":\"YES\",\"environmentCode\":\"-1\",\"taskDefinitionIndex\":2,\"taskPriority\":\"MEDIUM\",\"taskParams\":\"{\\\"preStatements\\\":null,\\\"postStatements\\\":null,\\\"type\\\":\\\"ADB_MYSQL\\\",\\\"database\\\":\\\"lijia\\\",\\\"sql\\\":\\\"create table nation_${random_serial_number} as select * from nation\\\",\\\"localParams\\\":null,\\\"Name\\\":\\\"create_table_as_select_nation\\\",\\\"FailRetryTimes\\\":0,\\\"dbClusterId\\\":\\\"amv-bp10o45925jpx959\\\",\\\"sendAlert\\\":false,\\\"displayRows\\\":10,\\\"limit\\\":10000,\\\"agentSource\\\":\\\"Workflow\\\",\\\"agentVersion\\\":\\\"Unkown\\\"}\",\"timeout\":\"0\",\"taskType\":\"ADB_MYSQL\",\"timeoutFlag\":\"CLOSE\",\"projectCode\":\"5191800302720\",\"name\":\"create_table_as_select_nation\",\"delayTime\":\"0\",\"workerGroup\":\"default\"}"; + "{\"failRetryTimes\":\"0\",\"timeoutNotifyStrategy\":\"\",\"code\":\"5195043558720\",\"flag\":\"YES\",\"environmentCode\":\"-1\",\"taskDefinitionIndex\":2,\"taskPriority\":\"MEDIUM\",\"taskParams\":\"{\\\"preStatements\\\":null,\\\"postStatements\\\":null,\\\"type\\\":\\\"ADB_MYSQL\\\",\\\"database\\\":\\\"lijia\\\",\\\"sql\\\":\\\"create table nation_${random_serial_number} as select * from nation\\\",\\\"localParams\\\":null,\\\"Name\\\":\\\"create_table_as_select_nation\\\",\\\"FailRetryTimes\\\":0,\\\"dbClusterId\\\":\\\"amv-bp10o45925jpx959\\\",\\\"sendEmail\\\":false,\\\"displayRows\\\":10,\\\"limit\\\":10000,\\\"agentSource\\\":\\\"Workflow\\\",\\\"agentVersion\\\":\\\"Unkown\\\"}\",\"timeout\":\"0\",\"taskType\":\"ADB_MYSQL\",\"timeoutFlag\":\"CLOSE\",\"projectCode\":\"5191800302720\",\"name\":\"create_table_as_select_nation\",\"delayTime\":\"0\",\"workerGroup\":\"default\"}"; TaskDefinition definition = JSONUtils.parseObject(definitionJson, TaskDefinition.class); Assertions.assertNull(definition.getTaskParamMap(), "Serialize the task definition success"); From afd36f4157c45e569ba3b11217867458c782f6ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Thu, 20 Aug 2026 10:52:56 +0800 Subject: [PATCH 13/43] Make task-result alert persistence idempotent and post-state-transition --- .../apache/dolphinscheduler/dao/AlertDao.java | 27 ++++++++++++ .../dao/mapper/AlertMapper.java | 6 +++ .../dao/mapper/AlertMapper.xml | 28 ++++++++++++ .../dao/repository/impl/AlertDaoTest.java | 43 +++++++++++++++++++ .../event/TaskSuccessLifecycleEvent.java | 6 +++ .../TaskSuccessLifecycleEventHandler.java | 29 ++++++++++++- .../rpc/TaskExecutorEventListenerImpl.java | 22 +--------- .../service/alert/WorkflowAlertManager.java | 2 +- 8 files changed, 141 insertions(+), 22 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index 19ed2d70f2df..4cc2f2c891a7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -108,6 +108,33 @@ public int addAlert(Alert alert) { return count; } + /** + * Insert a task-result alert idempotently. If an alert with the same sign, + * workflow instance id and alert type already exists, the insert is skipped. + *

This guards against duplicate inserts caused by at-least-once delivery + * of task success lifecycle events. + * + * @param alert alert, must have sign, workflowInstanceId and alertType set + * @return insert count (1 if inserted, 0 if skipped) + */ + public int addTaskResultAlert(Alert alert) { + if (null == alert.getAlertGroupId() || NumberUtils.INTEGER_ZERO.equals(alert.getAlertGroupId())) { + log.warn("the value of alertGroupId is null or 0 "); + return 0; + } + + String sign = generateSign(alert); + alert.setSign(sign); + int count = alertMapper.insertTaskResultAlertIfAbsent(alert); + if (count > 0) { + log.info("add task result alert to db , alert: {}", alert); + } else { + log.info("skip duplicate task result alert, sign: {}, workflowInstanceId: {}", sign, + alert.getWorkflowInstanceId()); + } + return count; + } + /** * update alert sending(execution) status * diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java index 0d60891e55f5..dcfdf1ec0e86 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java @@ -43,6 +43,12 @@ List listingAlertByStatus(@Param("minAlertId") int minAlertId, @Param("al void insertAlertWhenServerCrash(@Param("alert") Alert alert, @Param("crashAlarmSuppressionStartTime") Date crashAlarmSuppressionStartTime); + /** + * Insert a task-result alert only if no alert with the same sign, workflow instance id and alert type + * already exists. This makes the insert idempotent against at-least-once event delivery. + */ + int insertTaskResultAlertIfAbsent(@Param("alert") Alert alert); + void deleteByWorkflowInstanceId(@Param("workflowInstanceId") Integer processInstanceId); List selectByWorkflowInstanceId(@Param("workflowInstanceId") Integer processInstanceId); diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml index 7891dd91376d..df6a3a783556 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml @@ -51,6 +51,34 @@ having count(*) = 0 + + + insert into t_ds_alert(sign, title, content, alert_status, warning_type, log, alertgroup_id, create_time, + update_time, project_code, workflow_definition_code, workflow_instance_id, alert_type) + SELECT #{alert.sign}, + #{alert.title}, + #{alert.content}, + #{alert.alertStatus.code}, + #{alert.warningType.code}, + #{alert.log}, + #{alert.alertGroupId}, + #{alert.createTime}, + #{alert.updateTime}, + #{alert.projectCode}, + #{alert.workflowDefinitionCode}, + #{alert.workflowInstanceId}, + #{alert.alertType.code} + from t_ds_alert + where sign = #{alert.sign} + and workflow_instance_id = #{alert.workflowInstanceId} + and alert_type = #{alert.alertType.code} + having count(*) = 0 + + diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index d69728be6ef3..597ff1e7e73f 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -285,7 +285,8 @@ CREATE TABLE t_ds_alert workflow_instance_id int(11) DEFAULT NULL, alert_type int(11) DEFAULT NULL, PRIMARY KEY (id), - KEY idx_sign (sign) + KEY idx_sign (sign), + UNIQUE KEY uk_alert_dedup (sign, workflow_instance_id, alert_type) ); -- ---------------------------- diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index 361cdc3ff7fd..0986b4da1f19 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -291,7 +291,8 @@ CREATE TABLE `t_ds_alert` ( `alert_type` int(11) DEFAULT NULL COMMENT 'alert_type', PRIMARY KEY (`id`), KEY `idx_status` (`alert_status`) USING BTREE, - KEY `idx_sign` (`sign`) USING BTREE + KEY `idx_sign` (`sign`) USING BTREE, + UNIQUE KEY `uk_alert_dedup` (`sign`, `workflow_instance_id`, `alert_type`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin; -- ---------------------------- diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index 387716298493..645db27dd747 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -226,6 +226,7 @@ comment on column t_ds_alert.sign is 'sign=sha1(content)'; create index idx_status on t_ds_alert (alert_status); create index idx_sign on t_ds_alert (sign); +create unique index uk_alert_dedup on t_ds_alert (sign, workflow_instance_id, alert_type); -- -- Table structure for table t_ds_alertgroup diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql index afeb9917b86b..e9eeee79ab35 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql @@ -20,3 +20,15 @@ ALTER TABLE `t_ds_workflow_instance` ADD INDEX idx_project_start_time (project_c ALTER TABLE `t_ds_schedules` ADD COLUMN `missed_fire_policy` tinyint NOT NULL DEFAULT '2' COMMENT 'missed fire policy: 0 skip missed, 1 fire once now, 2 fire all missed' AFTER `crontab`; +-- Enforce idempotent task-result alerts at the database level. +-- Allows INSERT IGNORE (MySQL) / ON CONFLICT DO NOTHING (PostgreSQL) to atomically +-- prevent duplicates without check-then-insert race conditions. +-- Clean up any existing duplicate rows before adding the unique constraint. +DELETE t1 FROM t_ds_alert t1 +INNER JOIN t_ds_alert t2 +WHERE t1.id < t2.id + AND t1.sign = t2.sign + AND t1.workflow_instance_id = t2.workflow_instance_id + AND t1.alert_type = t2.alert_type; +ALTER TABLE `t_ds_alert` ADD UNIQUE INDEX `uk_alert_dedup` (`sign`, `workflow_instance_id`, `alert_type`); + diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql index 1709fe57e737..b3cfe95c3862 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -19,3 +19,15 @@ CREATE INDEX idx_project_submit_time ON t_ds_task_instance (project_code ASC, su CREATE INDEX idx_project_start_time ON t_ds_workflow_instance (project_code ASC, start_time DESC); ALTER TABLE t_ds_schedules ADD COLUMN missed_fire_policy smallint NOT NULL DEFAULT 2; + +-- Enforce idempotent task-result alerts at the database level. +-- Allows INSERT IGNORE (MySQL) / ON CONFLICT DO NOTHING (PostgreSQL) to atomically +-- prevent duplicates without check-then-insert race conditions. +-- Clean up any existing duplicate rows before adding the unique constraint. +DELETE FROM t_ds_alert a +USING t_ds_alert b +WHERE a.id < b.id + AND a.sign = b.sign + AND a.workflow_instance_id = b.workflow_instance_id + AND a.alert_type = b.alert_type; +CREATE UNIQUE INDEX IF NOT EXISTS uk_alert_dedup ON t_ds_alert (sign, workflow_instance_id, alert_type); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java index c4b053112244..46c38e91b9b6 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java @@ -24,7 +24,13 @@ import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.Alert; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -157,4 +163,79 @@ void testAddTaskResultAlertDifferentTaskSameContentNotDuplicate() { .count(); Assertions.assertEquals(2L, count); } + + /** + * Verifies that concurrent calls to {@code addTaskResultAlert} with the same + * deduplication key result in exactly one inserted row. + *

+ * Without the database-level unique constraint, the old INSERT ... SELECT ... + * HAVING count(*) = 0 pattern allowed a race condition where two transactions + * could both observe a count of zero and insert duplicate alerts. The unique + * index uk_alert_dedup plus INSERT IGNORE / ON CONFLICT DO NOTHING ensures + * atomicity: only one insert succeeds, the rest are silently skipped. + */ + @Test + void testConcurrentAddTaskResultAlertIdempotent() throws Exception { + String content = "[{\"taskName\":\"sql-concurrent\",\"result\":\"ok\"}]"; + int workflowInstanceId = 999997; + int taskInstanceId = 7001; + int threadCount = 8; + + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch doneLatch = new CountDownLatch(threadCount); + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + AtomicInteger successCount = new AtomicInteger(0); + + List> futures = new ArrayList<>(); + for (int i = 0; i < threadCount; i++) { + futures.add(executor.submit(() -> { + try { + startLatch.await(); + + Alert alert = new Alert(); + alert.setTitle("SQL Task Result"); + alert.setContent(content); + alert.setWarningType(WarningType.SUCCESS); + alert.setAlertGroupId(1); + alert.setAlertStatus(AlertStatus.WAIT_EXECUTION); + alert.setWorkflowInstanceId(workflowInstanceId); + alert.setTaskInstanceId(taskInstanceId); + alert.setAlertType(AlertType.TASK_RESULT); + alert.setCreateTime(new java.util.Date()); + + int inserted = alertDao.addTaskResultAlert(alert); + if (inserted > 0) { + successCount.incrementAndGet(); + } + } catch (Exception e) { + // Exceptions are acceptable for losing threads; the winner inserts + } finally { + doneLatch.countDown(); + } + })); + } + + startLatch.countDown(); + doneLatch.await(); + executor.shutdown(); + + // Check for unexpected exceptions + for (Future f : futures) { + if (!f.isDone()) { + Assertions.fail("A concurrent task did not complete"); + } + } + + // Exactly one thread should have successfully inserted + Assertions.assertEquals(1, successCount.get(), + "Exactly one concurrent insert should succeed, but got " + successCount.get()); + + // Verify only one alert row exists in the database + long dbCount = alertDao.listAlerts(workflowInstanceId) + .stream() + .filter(a -> a.getAlertType() == AlertType.TASK_RESULT) + .count(); + Assertions.assertEquals(1L, dbCount, + "Exactly one alert row should exist in the database, but found " + dbCount); + } } From 346df779edc87a67b3e128100ca0556418299b58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Wed, 26 Aug 2026 17:16:39 +0800 Subject: [PATCH 19/43] format incompatible.md --- docs/docs/en/guide/upgrade/incompatible.md | 3 +-- docs/docs/en/guide/upgrade/upgrade.md | 1 + docs/docs/zh/guide/upgrade/incompatible.md | 3 +-- docs/docs/zh/guide/upgrade/upgrade.md | 1 + 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index e0a24f536b60..5de5c0193d02 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -1,4 +1,3 @@ - # Incompatible This document records the incompatible updates between each version. You need to check this document before you upgrade to related version. @@ -55,5 +54,5 @@ This document records the incompatible updates between each version. You need to * **Removed transient fields**: `stateDescList`, `workflowDefinition`, `dagData`, `queue`, `locations`, `dependenceScheduleTimes` * **Removed derived properties**: `cmdTypeIfComplement`, `complementData` (related to complement-data executions; use the detail API to obtain them) * To obtain any of these fields, use the detail API `GET /projects/{projectCode}/workflow-instances/{id}` instead, which continues to return the full `WorkflowInstance` object. ([#18444](https://github.com/apache/dolphinscheduler/pull/18444)) -* Add a unique constraint `uk_alert_dedup` on `(sign, workflow_instance_id, alert_type)` to the `t_ds_alert` table. The upgrade script automatically cleans up existing duplicate rows (keeping the one with the largest id) before adding the unique index. This constraint ensures database-level idempotency for task-result alerts under concurrent delivery. +* Add a unique constraint `uk_alert_dedup` on `(sign, workflow_instance_id, alert_type)` to the `t_ds_alert` table. The upgrade script automatically cleans up existing duplicate rows (keeping the one with the largest id) before adding the unique index. This constraint ensures database-level idempotency for task-result alerts under concurrent delivery.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) diff --git a/docs/docs/en/guide/upgrade/upgrade.md b/docs/docs/en/guide/upgrade/upgrade.md index 9fbb4531bd3d..a30648dbf35c 100644 --- a/docs/docs/en/guide/upgrade/upgrade.md +++ b/docs/docs/en/guide/upgrade/upgrade.md @@ -90,3 +90,4 @@ The alert plugin may have some dirty data. After the upgrade, clear it manually ```sql delete from t_ds_alertgroup where group_name = 'global alert group' and description = 'global alert group'; ``` + diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 8dec3a72ee76..0888a90f4add 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -1,4 +1,3 @@ - # 不向前兼容的更新 本文档记录了各版本之间不兼容的更新内容。在升级到相关版本前，请检查本文档。 @@ -55,5 +54,5 @@ * **移除的非数据库字段**：`stateDescList`、`workflowDefinition`、`dagData`、`queue`、`locations`、`dependenceScheduleTimes` * **移除的派生属性**：`cmdTypeIfComplement`、`complementData`（补数执行相关，如需获取请使用详情接口） * 如需获取这些字段，请使用详情接口 `GET /projects/{projectCode}/workflow-instances/{id}`，该接口仍返回完整的 `WorkflowInstance` 对象 ([#18444](https://github.com/apache/dolphinscheduler/pull/18444)) -* 为 `t_ds_alert` 表的 `(sign, workflow_instance_id, alert_type)` 新增唯一约束 `uk_alert_dedup`。升级脚本会自动清理已存在的重复行（保留 id 最大的一条），之后添加唯一索引。该约束确保任务结果告警在并发投递时数据库层面幂等。 +* 为 `t_ds_alert` 表的 `(sign, workflow_instance_id, alert_type)` 新增唯一约束 `uk_alert_dedup`。升级脚本会自动清理已存在的重复行（保留 id 最大的一条），之后添加唯一索引。该约束确保任务结果告警在并发投递时数据库层面幂等。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) diff --git a/docs/docs/zh/guide/upgrade/upgrade.md b/docs/docs/zh/guide/upgrade/upgrade.md index 1221569ac939..d296cea68887 100644 --- a/docs/docs/zh/guide/upgrade/upgrade.md +++ b/docs/docs/zh/guide/upgrade/upgrade.md @@ -91,3 +91,4 @@ jar 包 并添加到 `./tools/libs` 目录下，设置以下环境变量 ```sql delete from t_ds_alertgroup where group_name = 'global alert group' and description = 'global alert group'; ``` + From bf36a33021ae27234846f527615188f6d90a2488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Thu, 27 Aug 2026 20:27:59 +0800 Subject: [PATCH 20/43] Fix databaseId mismatch and replace INSERT IGNORE with ON DUPLICATE KEY UPDATE --- .../dao/DaoConfiguration.java | 10 ++- .../dao/mapper/AlertMapper.xml | 8 ++- .../dao/repository/impl/AlertDaoTest.java | 71 ++++++++----------- 3 files changed, 44 insertions(+), 45 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java index 1b897e806de4..249a8cd7f7e6 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java @@ -24,6 +24,8 @@ import org.apache.ibatis.mapping.DatabaseIdProvider; import org.apache.ibatis.mapping.VendorDatabaseIdProvider; +import java.util.Properties; + import org.mybatis.spring.annotation.MapperScan; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -63,7 +65,13 @@ public MybatisPlusInterceptor paginationInterceptor(DbType dbType) { @Bean public DatabaseIdProvider databaseIdProvider() { - return new VendorDatabaseIdProvider(); + VendorDatabaseIdProvider provider = new VendorDatabaseIdProvider(); + Properties properties = new Properties(); + properties.setProperty("MySQL", "mysql"); + properties.setProperty("PostgreSQL", "postgresql"); + properties.setProperty("H2", "h2"); + provider.setProperties(properties); + return provider; } @Bean diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml index fc87ef2b484e..97e9eb200d99 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml @@ -55,30 +55,32 @@ Idempotent insert for task-result alerts. Uses database-level unique constraint uk_alert_dedup (sign, workflow_instance_id, alert_type) to atomically prevent duplicates. Dialect-specific syntax is selected via databaseId: - - MySQL / H2: INSERT IGNORE silently skips on duplicate key + - MySQL / H2: ON DUPLICATE KEY UPDATE id = id silently skips on duplicate key - PostgreSQL: ON CONFLICT DO NOTHING Fallback (no databaseId): plain INSERT for environments where the provider is not configured. --> - INSERT IGNORE INTO t_ds_alert(sign, title, content, alert_status, warning_type, log, alertgroup_id, + INSERT INTO t_ds_alert(sign, title, content, alert_status, warning_type, log, alertgroup_id, create_time, update_time, project_code, workflow_definition_code, workflow_instance_id, alert_type) VALUES (#{alert.sign}, #{alert.title}, #{alert.content}, #{alert.alertStatus.code}, #{alert.warningType.code}, #{alert.log}, #{alert.alertGroupId}, #{alert.createTime}, #{alert.updateTime}, #{alert.projectCode}, #{alert.workflowDefinitionCode}, #{alert.workflowInstanceId}, #{alert.alertType.code}) + ON DUPLICATE KEY UPDATE id = id - INSERT IGNORE INTO t_ds_alert(sign, title, content, alert_status, warning_type, log, alertgroup_id, + INSERT INTO t_ds_alert(sign, title, content, alert_status, warning_type, log, alertgroup_id, create_time, update_time, project_code, workflow_definition_code, workflow_instance_id, alert_type) VALUES (#{alert.sign}, #{alert.title}, #{alert.content}, #{alert.alertStatus.code}, #{alert.warningType.code}, #{alert.log}, #{alert.alertGroupId}, #{alert.createTime}, #{alert.updateTime}, #{alert.projectCode}, #{alert.workflowDefinitionCode}, #{alert.workflowInstanceId}, #{alert.alertType.code}) + ON DUPLICATE KEY UPDATE id = id diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java index 46c38e91b9b6..deb09a9b9913 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java @@ -30,7 +30,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -168,11 +167,12 @@ void testAddTaskResultAlertDifferentTaskSameContentNotDuplicate() { * Verifies that concurrent calls to {@code addTaskResultAlert} with the same * deduplication key result in exactly one inserted row. *

- * Without the database-level unique constraint, the old INSERT ... SELECT ... - * HAVING count(*) = 0 pattern allowed a race condition where two transactions - * could both observe a count of zero and insert duplicate alerts. The unique - * index uk_alert_dedup plus INSERT IGNORE / ON CONFLICT DO NOTHING ensures - * atomicity: only one insert succeeds, the rest are silently skipped. + * The database-level unique constraint uk_alert_dedup plus the dialect-specific + * idempotent INSERT (ON DUPLICATE KEY UPDATE / ON CONFLICT DO NOTHING) ensures + * atomicity: only one insert succeeds, the rest return zero without throwing. + *

+ * Each losing thread must return 0, not throw an exception. If any thread throws, + * {@link Future#get()} will propagate the exception and fail the test. */ @Test void testConcurrentAddTaskResultAlertIdempotent() throws Exception { @@ -182,55 +182,44 @@ void testConcurrentAddTaskResultAlertIdempotent() throws Exception { int threadCount = 8; CountDownLatch startLatch = new CountDownLatch(1); - CountDownLatch doneLatch = new CountDownLatch(threadCount); ExecutorService executor = Executors.newFixedThreadPool(threadCount); - AtomicInteger successCount = new AtomicInteger(0); - List> futures = new ArrayList<>(); + List> futures = new ArrayList<>(); for (int i = 0; i < threadCount; i++) { futures.add(executor.submit(() -> { - try { - startLatch.await(); - - Alert alert = new Alert(); - alert.setTitle("SQL Task Result"); - alert.setContent(content); - alert.setWarningType(WarningType.SUCCESS); - alert.setAlertGroupId(1); - alert.setAlertStatus(AlertStatus.WAIT_EXECUTION); - alert.setWorkflowInstanceId(workflowInstanceId); - alert.setTaskInstanceId(taskInstanceId); - alert.setAlertType(AlertType.TASK_RESULT); - alert.setCreateTime(new java.util.Date()); - - int inserted = alertDao.addTaskResultAlert(alert); - if (inserted > 0) { - successCount.incrementAndGet(); - } - } catch (Exception e) { - // Exceptions are acceptable for losing threads; the winner inserts - } finally { - doneLatch.countDown(); - } + startLatch.await(); + + Alert alert = new Alert(); + alert.setTitle("SQL Task Result"); + alert.setContent(content); + alert.setWarningType(WarningType.SUCCESS); + alert.setAlertGroupId(1); + alert.setAlertStatus(AlertStatus.WAIT_EXECUTION); + alert.setWorkflowInstanceId(workflowInstanceId); + alert.setTaskInstanceId(taskInstanceId); + alert.setAlertType(AlertType.TASK_RESULT); + alert.setCreateTime(new java.util.Date()); + + return alertDao.addTaskResultAlert(alert); })); } startLatch.countDown(); - doneLatch.await(); executor.shutdown(); - // Check for unexpected exceptions - for (Future f : futures) { - if (!f.isDone()) { - Assertions.fail("A concurrent task did not complete"); + int successCount = 0; + for (Future f : futures) { + // Future.get() propagates any exception thrown by the task. + // A losing thread must return 0, not throw. + int inserted = f.get(); + if (inserted > 0) { + successCount++; } } - // Exactly one thread should have successfully inserted - Assertions.assertEquals(1, successCount.get(), - "Exactly one concurrent insert should succeed, but got " + successCount.get()); + Assertions.assertEquals(1, successCount, + "Exactly one concurrent insert should succeed, but got " + successCount); - // Verify only one alert row exists in the database long dbCount = alertDao.listAlerts(workflowInstanceId) .stream() .filter(a -> a.getAlertType() == AlertType.TASK_RESULT) From 7e58adc8f63c2b2bcec9025d5b9021387948c1a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Fri, 28 Aug 2026 15:39:23 +0800 Subject: [PATCH 21/43] Restore deprecated protected fields needAlert and taskAlertInfo in AbstractTask for binary compatibility --- .../plugin/task/api/AbstractTask.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java index 8440c543c6d1..bd6766a1968a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTask.java @@ -56,6 +56,24 @@ public abstract class AbstractTask { */ protected volatile int exitStatusCode = -1; + /** + * @deprecated Use {@link TaskExecutionContext#isNeedAlert()} on the + * {@code taskRequest} field instead. Retained for binary + * compatibility with third-party plugins compiled against + * the previous AbstractTask API. + */ + @Deprecated + protected boolean needAlert = false; + + /** + * @deprecated Use {@link TaskExecutionContext#getTaskAlertInfo()} on the + * {@code taskRequest} field instead. Retained for binary + * compatibility with third-party plugins compiled against + * the previous AbstractTask API. + */ + @Deprecated + protected TaskAlertInfo taskAlertInfo; + /** * constructor * From 30b2c9b594b294587ff057074f0971445d42ecf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Fri, 28 Aug 2026 16:04:09 +0800 Subject: [PATCH 22/43] Reject null alertType in AlertSender and document rolling upgrade order in incompatible.md --- docs/docs/en/guide/upgrade/incompatible.md | 2 +- docs/docs/zh/guide/upgrade/incompatible.md | 2 +- .../alert/service/AlertSender.java | 21 +++++++------------ .../alert/runner/AlertSenderTest.java | 12 +++++------ 4 files changed, 14 insertions(+), 23 deletions(-) diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index 64ac31001185..e614f7a444e5 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -56,4 +56,4 @@ This document records the incompatible updates between each version. You need to * **Removed derived properties**: `cmdTypeIfComplement`, `complementData` (related to complement-data executions; use the detail API to obtain them) * To obtain any of these fields, use the detail API `GET /projects/{projectCode}/workflow-instances/{id}` instead, which continues to return the full `WorkflowInstance` object. ([#18444](https://github.com/apache/dolphinscheduler/pull/18444)) * Add a unique constraint `uk_alert_dedup` on `(sign, workflow_instance_id, alert_type)` to the `t_ds_alert` table. The upgrade script automatically cleans up existing duplicate rows (keeping the one with the largest id) before adding the unique index. This constraint ensures database-level idempotency for task-result alerts under concurrent delivery.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) - +* `AlertSender#getAlertData` now explicitly rejects a null `alertType` by throwing `IllegalArgumentException` with an error log, instead of throwing a raw `NullPointerException`. During a rolling upgrade, **the Alert Server must be upgraded before Master/Worker**, otherwise an old Alert Server cannot recognize newly introduced `alert_type` enum values (e.g. `TASK_RESULT`) and alert delivery will fail. See the [upgrade guide](upgrade.md#rolling-upgrade-order) for details.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index dbd6c5012580..f9bd6fd29637 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -56,4 +56,4 @@ * **移除的派生属性**：`cmdTypeIfComplement`、`complementData`（补数执行相关，如需获取请使用详情接口） * 如需获取这些字段，请使用详情接口 `GET /projects/{projectCode}/workflow-instances/{id}`，该接口仍返回完整的 `WorkflowInstance` 对象 ([#18444](https://github.com/apache/dolphinscheduler/pull/18444)) * 为 `t_ds_alert` 表的 `(sign, workflow_instance_id, alert_type)` 新增唯一约束 `uk_alert_dedup`。升级脚本会自动清理已存在的重复行（保留 id 最大的一条），之后添加唯一索引。该约束确保任务结果告警在并发投递时数据库层面幂等。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) - +* `AlertSender#getAlertData` 对 null `alertType` 增加了显式拒绝（抛 `IllegalArgumentException` 并记录错误日志），而非直接抛 `NullPointerException`。滚动升级时**必须先升级 Alert Server，再升级 Master/Worker**，否则旧 Alert Server 无法识别新增的 `alert_type` 枚举值（如 `TASK_RESULT`），会导致告警投递失败。详见[升级文档](upgrade.md#滚动升降级顺序)。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) \ No newline at end of file diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java index 92317ee56936..2f829e5bcd92 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertSender.java @@ -100,26 +100,19 @@ public List getAlertPluginInstanceList(Alert event) { @Override public AlertData getAlertData(Alert event) { - // During a rolling upgrade, an old Alert Server may receive an alert whose - // alert_type value (e.g. TASK_RESULT) was introduced by a newer Master and - // is not yet present in the old AlertType enum. MyBatis-Plus maps unknown - // enum values to null, so we must guard against a NullPointerException here. - // Falling back to 0 (WORKFLOW_INSTANCE_FAILURE) keeps the alert deliverable - // while logging a warning so operators can detect the version mismatch. - int alertTypeCode = 0; - if (event.getAlertType() != null) { - alertTypeCode = event.getAlertType().getCode(); - } else { - log.warn("Alert {} has null alertType, falling back to 0 (WORKFLOW_INSTANCE_FAILURE). " - + "This may indicate an Alert Server version older than the Master that created the alert.", - event.getId()); + if (event.getAlertType() == null) { + log.error("Alert {} has null alertType, refusing to send. This indicates the Alert Server version " + + "is older than the Master that created the alert. Please upgrade the Alert Server first " + + "(see upgrade guide: Alert Server -> Master -> Worker).", event.getId()); + throw new IllegalArgumentException( + "Cannot build AlertData with null alertType for alert " + event.getId()); } return AlertData.builder() .id(event.getId()) .content(event.getContent()) .log(event.getLog()) .title(event.getTitle()) - .alertType(alertTypeCode) + .alertType(event.getAlertType().getCode()) .build(); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java index fbbb47907164..47bb0c506673 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java @@ -216,7 +216,8 @@ void testSendAlert() { * Simulates a mixed-version rolling upgrade scenario: an old Alert Server that * does not yet know about a new AlertType enum value (e.g. TASK_RESULT) will * have MyBatis map the unknown value to null. {@link AlertSender#getAlertData} - * must not throw NPE and should degrade gracefully. + * must refuse to send rather than silently relabeling the alert as a + * workflow-instance failure. */ @Test void testGetAlertDataWithNullAlertType() { @@ -229,12 +230,9 @@ void testGetAlertDataWithNullAlertType() { // Simulate old Alert Server where unknown alert_type is deserialized as null alert.setAlertType(null); - AlertData alertData = alertSender.getAlertData(alert); - Assertions.assertNotNull(alertData); - Assertions.assertEquals(1, alertData.getId()); - Assertions.assertEquals(TITLE, alertData.getTitle()); - Assertions.assertEquals(CONTENT, alertData.getContent()); - Assertions.assertEquals(0, alertData.getAlertType()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> alertSender.getAlertData(alert), + "getAlertData should refuse to build AlertData with null alertType"); } /** From 78b24964de133caa094bb42a65368e39722f3502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Fri, 28 Aug 2026 16:10:49 +0800 Subject: [PATCH 23/43] format spotless --- docs/docs/en/guide/upgrade/incompatible.md | 1 + docs/docs/zh/guide/upgrade/incompatible.md | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index e614f7a444e5..756b498271ec 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -57,3 +57,4 @@ This document records the incompatible updates between each version. You need to * To obtain any of these fields, use the detail API `GET /projects/{projectCode}/workflow-instances/{id}` instead, which continues to return the full `WorkflowInstance` object. ([#18444](https://github.com/apache/dolphinscheduler/pull/18444)) * Add a unique constraint `uk_alert_dedup` on `(sign, workflow_instance_id, alert_type)` to the `t_ds_alert` table. The upgrade script automatically cleans up existing duplicate rows (keeping the one with the largest id) before adding the unique index. This constraint ensures database-level idempotency for task-result alerts under concurrent delivery.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` now explicitly rejects a null `alertType` by throwing `IllegalArgumentException` with an error log, instead of throwing a raw `NullPointerException`. During a rolling upgrade, **the Alert Server must be upgraded before Master/Worker**, otherwise an old Alert Server cannot recognize newly introduced `alert_type` enum values (e.g. `TASK_RESULT`) and alert delivery will fail. See the [upgrade guide](upgrade.md#rolling-upgrade-order) for details.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) + diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index f9bd6fd29637..a4a1c4f57282 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -56,4 +56,5 @@ * **移除的派生属性**：`cmdTypeIfComplement`、`complementData`（补数执行相关，如需获取请使用详情接口） * 如需获取这些字段，请使用详情接口 `GET /projects/{projectCode}/workflow-instances/{id}`，该接口仍返回完整的 `WorkflowInstance` 对象 ([#18444](https://github.com/apache/dolphinscheduler/pull/18444)) * 为 `t_ds_alert` 表的 `(sign, workflow_instance_id, alert_type)` 新增唯一约束 `uk_alert_dedup`。升级脚本会自动清理已存在的重复行（保留 id 最大的一条），之后添加唯一索引。该约束确保任务结果告警在并发投递时数据库层面幂等。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) -* `AlertSender#getAlertData` 对 null `alertType` 增加了显式拒绝（抛 `IllegalArgumentException` 并记录错误日志），而非直接抛 `NullPointerException`。滚动升级时**必须先升级 Alert Server，再升级 Master/Worker**，否则旧 Alert Server 无法识别新增的 `alert_type` 枚举值（如 `TASK_RESULT`），会导致告警投递失败。详见[升级文档](upgrade.md#滚动升降级顺序)。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) \ No newline at end of file +* `AlertSender#getAlertData` 对 null `alertType` 增加了显式拒绝（抛 `IllegalArgumentException` 并记录错误日志），而非直接抛 `NullPointerException`。滚动升级时**必须先升级 Alert Server，再升级 Master/Worker**，否则旧 Alert Server 无法识别新增的 `alert_type` 枚举值（如 `TASK_RESULT`），会导致告警投递失败。详见[升级文档](upgrade.md#滚动升降级顺序)。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) + From 2eafba1d036bb3410f65232c09a444ab9c1b0863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Fri, 28 Aug 2026 17:28:00 +0800 Subject: [PATCH 24/43] update h2 sql --- .../dao/mapper/AlertMapper.xml | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml index 97e9eb200d99..25d2d4300b16 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml @@ -60,7 +60,7 @@ Fallback (no databaseId): plain INSERT for environments where the provider is not configured. --> - + INSERT INTO t_ds_alert(sign, title, content, alert_status, warning_type, log, alertgroup_id, create_time, update_time, project_code, workflow_definition_code, @@ -72,15 +72,16 @@ ON DUPLICATE KEY UPDATE id = id + - INSERT INTO t_ds_alert(sign, title, content, alert_status, warning_type, log, alertgroup_id, - create_time, update_time, project_code, workflow_definition_code, - workflow_instance_id, alert_type) + MERGE INTO t_ds_alert(sign, title, content, alert_status, warning_type, log, alertgroup_id, + create_time, update_time, project_code, workflow_definition_code, + workflow_instance_id, alert_type) + KEY(sign, workflow_instance_id, alert_type) VALUES (#{alert.sign}, #{alert.title}, #{alert.content}, #{alert.alertStatus.code}, - #{alert.warningType.code}, #{alert.log}, #{alert.alertGroupId}, #{alert.createTime}, - #{alert.updateTime}, #{alert.projectCode}, #{alert.workflowDefinitionCode}, - #{alert.workflowInstanceId}, #{alert.alertType.code}) - ON DUPLICATE KEY UPDATE id = id + #{alert.warningType.code}, #{alert.log}, #{alert.alertGroupId}, #{alert.createTime}, + #{alert.updateTime}, #{alert.projectCode}, #{alert.workflowDefinitionCode}, + #{alert.workflowInstanceId}, #{alert.alertType.code}) @@ -95,17 +96,6 @@ ON CONFLICT (sign, workflow_instance_id, alert_type) DO NOTHING - - - INSERT INTO t_ds_alert(sign, title, content, alert_status, warning_type, log, alertgroup_id, - create_time, update_time, project_code, workflow_definition_code, - workflow_instance_id, alert_type) - VALUES (#{alert.sign}, #{alert.title}, #{alert.content}, #{alert.alertStatus.code}, - #{alert.warningType.code}, #{alert.log}, #{alert.alertGroupId}, #{alert.createTime}, - #{alert.updateTime}, #{alert.projectCode}, #{alert.workflowDefinitionCode}, - #{alert.workflowInstanceId}, #{alert.alertType.code}) - - diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java index deb09a9b9913..55c470d34572 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java @@ -167,9 +167,11 @@ void testAddTaskResultAlertDifferentTaskSameContentNotDuplicate() { * Verifies that concurrent calls to {@code addTaskResultAlert} with the same * deduplication key result in exactly one inserted row. *

- * The database-level unique constraint uk_alert_dedup plus the dialect-specific - * idempotent INSERT (ON DUPLICATE KEY UPDATE / ON CONFLICT DO NOTHING) ensures - * atomicity: only one insert succeeds, the rest return zero without throwing. + * The INSERT ... SELECT ... WHERE NOT EXISTS check is not atomic with the + * INSERT itself, so under concurrent access a race condition may cause a + * DuplicateKeyException from the uk_alert_dedup unique constraint. The DAO + * layer catches this exception and returns 0, ensuring exactly one row is + * inserted and no exception propagates to the caller. *

* Each losing thread must return 0, not throw an exception. If any thread throws, * {@link Future#get()} will propagate the exception and fail the test. From 68653b8c15e39ca4e735f7af2ec3edaa90e285cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Mon, 7 Sep 2026 14:35:04 +0800 Subject: [PATCH 27/43] update version 3.5.0 --- docs/docs/en/guide/upgrade/incompatible.md | 4 +++ docs/docs/zh/guide/upgrade/incompatible.md | 3 ++ .../mysql/dolphinscheduler_ddl.sql | 12 -------- .../postgresql/dolphinscheduler_ddl.sql | 12 -------- .../mysql/dolphinscheduler_ddl.sql | 29 +++++++++++++++++++ .../mysql/dolphinscheduler_dml.sql | 16 ++++++++++ .../postgresql/dolphinscheduler_ddl.sql | 28 ++++++++++++++++++ .../postgresql/dolphinscheduler_dml.sql | 16 ++++++++++ 8 files changed, 96 insertions(+), 24 deletions(-) create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index 07f88676fb8c..f505d97614ad 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -56,5 +56,9 @@ This document records the incompatible updates between each version. You need to * **Removed derived properties**: `cmdTypeIfComplement`, `complementData` (related to complement-data executions; use the detail API to obtain them) * To obtain any of these fields, use the detail API `GET /projects/{projectCode}/workflow-instances/{id}` instead, which continues to return the full `WorkflowInstance` object. ([#18444](https://github.com/apache/dolphinscheduler/pull/18444)) * Add a unique constraint `uk_alert_dedup` on `(sign, workflow_instance_id, alert_type)` to the `t_ds_alert` table. The upgrade script automatically cleans up existing duplicate rows (keeping the one with the largest id) before adding the unique index. This constraint ensures database-level idempotency for task-result alerts under concurrent delivery.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) + +## 3.5.0 + +* Add a unique constraint `uk_alert_dedup` on `(sign, workflow_instance_id, alert_type)` to the `t_ds_alert` table. The upgrade script automatically cleans up existing duplicate rows (keeping the one with the largest id) before adding the unique index. This constraint ensures database-level idempotency for task-result alerts under concurrent delivery.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` now explicitly rejects a null `alertType` by throwing `IllegalArgumentException` with an error log, instead of throwing a raw `NullPointerException`. During a rolling upgrade, **the Alert Server must be upgraded before Master/Worker**, otherwise an old Alert Server cannot recognize newly introduced `alert_type` enum values (e.g. `TASK_RESULT`) and alert delivery will fail. See the [upgrade guide](upgrade.md#rolling-upgrade-order) for details.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index d6bf09b9cf52..7f27fe4feeba 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -55,6 +55,9 @@ * **移除的非数据库字段**：`stateDescList`、`workflowDefinition`、`dagData`、`queue`、`locations`、`dependenceScheduleTimes` * **移除的派生属性**：`cmdTypeIfComplement`、`complementData`（补数执行相关，如需获取请使用详情接口） * 如需获取这些字段，请使用详情接口 `GET /projects/{projectCode}/workflow-instances/{id}`，该接口仍返回完整的 `WorkflowInstance` 对象 ([#18444](https://github.com/apache/dolphinscheduler/pull/18444)) + +## 3.5.0 + * 为 `t_ds_alert` 表的 `(sign, workflow_instance_id, alert_type)` 新增唯一约束 `uk_alert_dedup`。升级脚本会自动清理已存在的重复行（保留 id 最大的一条），之后添加唯一索引。该约束确保任务结果告警在并发投递时数据库层面幂等。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` 对 null `alertType` 增加了显式拒绝（抛 `IllegalArgumentException` 并记录错误日志），而非直接抛 `NullPointerException`。滚动升级时**必须先升级 Alert Server，再升级 Master/Worker**，否则旧 Alert Server 无法识别新增的 `alert_type` 枚举值（如 `TASK_RESULT`），会导致告警投递失败。详见[升级文档](upgrade.md#滚动升降级顺序)。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.3_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.3_schema/mysql/dolphinscheduler_ddl.sql index e9eeee79ab35..afeb9917b86b 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.3_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.3_schema/mysql/dolphinscheduler_ddl.sql @@ -20,15 +20,3 @@ ALTER TABLE `t_ds_workflow_instance` ADD INDEX idx_project_start_time (project_c ALTER TABLE `t_ds_schedules` ADD COLUMN `missed_fire_policy` tinyint NOT NULL DEFAULT '2' COMMENT 'missed fire policy: 0 skip missed, 1 fire once now, 2 fire all missed' AFTER `crontab`; --- Enforce idempotent task-result alerts at the database level. --- Allows INSERT IGNORE (MySQL) / ON CONFLICT DO NOTHING (PostgreSQL) to atomically --- prevent duplicates without check-then-insert race conditions. --- Clean up any existing duplicate rows before adding the unique constraint. -DELETE t1 FROM t_ds_alert t1 -INNER JOIN t_ds_alert t2 -WHERE t1.id < t2.id - AND t1.sign = t2.sign - AND t1.workflow_instance_id = t2.workflow_instance_id - AND t1.alert_type = t2.alert_type; -ALTER TABLE `t_ds_alert` ADD UNIQUE INDEX `uk_alert_dedup` (`sign`, `workflow_instance_id`, `alert_type`); - diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.3_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.3_schema/postgresql/dolphinscheduler_ddl.sql index b3cfe95c3862..1709fe57e737 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.3_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.4.3_schema/postgresql/dolphinscheduler_ddl.sql @@ -19,15 +19,3 @@ CREATE INDEX idx_project_submit_time ON t_ds_task_instance (project_code ASC, su CREATE INDEX idx_project_start_time ON t_ds_workflow_instance (project_code ASC, start_time DESC); ALTER TABLE t_ds_schedules ADD COLUMN missed_fire_policy smallint NOT NULL DEFAULT 2; - --- Enforce idempotent task-result alerts at the database level. --- Allows INSERT IGNORE (MySQL) / ON CONFLICT DO NOTHING (PostgreSQL) to atomically --- prevent duplicates without check-then-insert race conditions. --- Clean up any existing duplicate rows before adding the unique constraint. -DELETE FROM t_ds_alert a -USING t_ds_alert b -WHERE a.id < b.id - AND a.sign = b.sign - AND a.workflow_instance_id = b.workflow_instance_id - AND a.alert_type = b.alert_type; -CREATE UNIQUE INDEX IF NOT EXISTS uk_alert_dedup ON t_ds_alert (sign, workflow_instance_id, alert_type); diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql new file mode 100644 index 000000000000..081317aa4b9c --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +-- Enforce idempotent task-result alerts at the database level. +-- Allows INSERT IGNORE (MySQL) / ON CONFLICT DO NOTHING (PostgreSQL) to atomically +-- prevent duplicates without check-then-insert race conditions. +-- Clean up any existing duplicate rows before adding the unique constraint. +DELETE t1 FROM t_ds_alert t1 +INNER JOIN t_ds_alert t2 +WHERE t1.id < t2.id + AND t1.sign = t2.sign + AND t1.workflow_instance_id = t2.workflow_instance_id + AND t1.alert_type = t2.alert_type; +ALTER TABLE `t_ds_alert` ADD UNIQUE INDEX `uk_alert_dedup` (`sign`, `workflow_instance_id`, `alert_type`); + diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 000000000000..4a14f326b985 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql new file mode 100644 index 000000000000..80f07ee72f40 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +-- Enforce idempotent task-result alerts at the database level. +-- Allows INSERT IGNORE (MySQL) / ON CONFLICT DO NOTHING (PostgreSQL) to atomically +-- prevent duplicates without check-then-insert race conditions. +-- Clean up any existing duplicate rows before adding the unique constraint. +DELETE FROM t_ds_alert a +USING t_ds_alert b +WHERE a.id < b.id + AND a.sign = b.sign + AND a.workflow_instance_id = b.workflow_instance_id + AND a.alert_type = b.alert_type; +CREATE UNIQUE INDEX IF NOT EXISTS uk_alert_dedup ON t_ds_alert (sign, workflow_instance_id, alert_type); diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 000000000000..4a14f326b985 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ From 5d22d11829c637c5b5eeee039951fe6102a9f8bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Mon, 7 Sep 2026 14:38:57 +0800 Subject: [PATCH 28/43] update version 3.5.0 --- docs/docs/en/guide/upgrade/incompatible.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index f505d97614ad..e5295b500c96 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -55,7 +55,6 @@ This document records the incompatible updates between each version. You need to * **Removed transient fields**: `stateDescList`, `workflowDefinition`, `dagData`, `queue`, `locations`, `dependenceScheduleTimes` * **Removed derived properties**: `cmdTypeIfComplement`, `complementData` (related to complement-data executions; use the detail API to obtain them) * To obtain any of these fields, use the detail API `GET /projects/{projectCode}/workflow-instances/{id}` instead, which continues to return the full `WorkflowInstance` object. ([#18444](https://github.com/apache/dolphinscheduler/pull/18444)) -* Add a unique constraint `uk_alert_dedup` on `(sign, workflow_instance_id, alert_type)` to the `t_ds_alert` table. The upgrade script automatically cleans up existing duplicate rows (keeping the one with the largest id) before adding the unique index. This constraint ensures database-level idempotency for task-result alerts under concurrent delivery.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) ## 3.5.0 From 9d4ab05daab1ebd76ee81d17655515c8bc6587b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Wed, 9 Sep 2026 17:41:42 +0800 Subject: [PATCH 29/43] Clean up redundant comments and code --- .../alert/runner/AlertSenderTest.java | 14 +-- .../apache/dolphinscheduler/dao/AlertDao.java | 3 - .../dao/DaoConfiguration.java | 10 +- .../dolphinscheduler/dao/entity/Alert.java | 4 +- .../dao/repository/impl/AlertDaoTest.java | 28 ++---- .../TaskSuccessLifecycleEventHandler.java | 19 +--- .../service/alert/WorkflowAlertManager.java | 10 +- .../alert/WorkflowAlertManagerTest.java | 27 +----- .../plugin/task/sql/SqlTask.java | 11 +-- .../plugin/task/sql/SqlTaskTest.java | 93 ++++++------------- 10 files changed, 50 insertions(+), 169 deletions(-) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java index 47bb0c506673..2e8f5b2672d0 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertSenderTest.java @@ -212,13 +212,6 @@ void testSendAlert() { alertSender.syncTestSend(PLUGIN_DEFINE_ID, PLUGIN_INSTANCE_PARAMS); } - /** - * Simulates a mixed-version rolling upgrade scenario: an old Alert Server that - * does not yet know about a new AlertType enum value (e.g. TASK_RESULT) will - * have MyBatis map the unknown value to null. {@link AlertSender#getAlertData} - * must refuse to send rather than silently relabeling the alert as a - * workflow-instance failure. - */ @Test void testGetAlertDataWithNullAlertType() { Alert alert = new Alert(); @@ -227,17 +220,12 @@ void testGetAlertDataWithNullAlertType() { alert.setContent(CONTENT); alert.setAlertGroupId(ALERT_GROUP_ID); alert.setWarningType(WarningType.FAILURE); - // Simulate old Alert Server where unknown alert_type is deserialized as null alert.setAlertType(null); Assertions.assertThrows(IllegalArgumentException.class, - () -> alertSender.getAlertData(alert), - "getAlertData should refuse to build AlertData with null alertType"); + () -> alertSender.getAlertData(alert)); } - /** - * Ensures the normal path still works correctly when alertType is present. - */ @Test void testGetAlertDataWithValidAlertType() { Alert alert = new Alert(); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index cc9576c1719c..b20b59bb17d0 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -112,10 +112,7 @@ public int addAlert(Alert alert) { /** * Insert a task-result alert idempotently. If an alert with the same sign, * workflow instance id and alert type already exists, the insert is skipped. - *

This guards against duplicate inserts caused by at-least-once delivery - * of task success lifecycle events. * - * @param alert alert, must have sign, workflowInstanceId and alertType set * @return insert count (1 if inserted, 0 if skipped) */ public int addTaskResultAlert(Alert alert) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java index 249a8cd7f7e6..1b897e806de4 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java @@ -24,8 +24,6 @@ import org.apache.ibatis.mapping.DatabaseIdProvider; import org.apache.ibatis.mapping.VendorDatabaseIdProvider; -import java.util.Properties; - import org.mybatis.spring.annotation.MapperScan; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -65,13 +63,7 @@ public MybatisPlusInterceptor paginationInterceptor(DbType dbType) { @Bean public DatabaseIdProvider databaseIdProvider() { - VendorDatabaseIdProvider provider = new VendorDatabaseIdProvider(); - Properties properties = new Properties(); - properties.setProperty("MySQL", "mysql"); - properties.setProperty("PostgreSQL", "postgresql"); - properties.setProperty("H2", "h2"); - provider.setProperties(properties); - return provider; + return new VendorDatabaseIdProvider(); } @Bean diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java index 23b7f7b38970..5916bedc3734 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Alert.java @@ -85,9 +85,7 @@ public class Alert { private AlertType alertType; /** - * Transient field used only for sign generation in idempotent task-result alert inserts. - *

Not persisted to the database — it is incorporated into the SHA-1 sign so that - * alerts from different tasks with identical content are not treated as duplicates. + * Transient — used only for sign generation in idempotent task-result alert inserts. */ @TableField(exist = false) private Integer taskInstanceId; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java index 55c470d34572..eea745b72037 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java @@ -95,7 +95,7 @@ void testAddTaskResultAlertIdempotent() { int firstCount = alertDao.addTaskResultAlert(alert); Assertions.assertEquals(1, firstCount); - // Second insert with the same content + workflowInstanceId + taskInstanceId + alertType should be skipped + // Second insert with the same dedup key should be skipped Alert duplicateAlert = new Alert(); duplicateAlert.setTitle("SQL Task Result"); duplicateAlert.setContent(content); @@ -110,7 +110,6 @@ void testAddTaskResultAlertIdempotent() { int secondCount = alertDao.addTaskResultAlert(duplicateAlert); Assertions.assertEquals(0, secondCount); - // Verify only one alert row exists for this workflow instance long count = alertDao.listAlerts(workflowInstanceId) .stream() .filter(a -> a.getAlertType() == AlertType.TASK_RESULT) @@ -120,7 +119,7 @@ void testAddTaskResultAlertIdempotent() { /** * Two different tasks in the same workflow instance returning identical results - * must NOT be treated as duplicates — each task should get its own alert. + * must NOT be treated as duplicates. */ @Test void testAddTaskResultAlertDifferentTaskSameContentNotDuplicate() { @@ -141,7 +140,7 @@ void testAddTaskResultAlertDifferentTaskSameContentNotDuplicate() { int firstCount = alertDao.addTaskResultAlert(alert1); Assertions.assertEquals(1, firstCount); - // Second task alert — same content, same workflow instance, but different task instance + // Second task alert — same content, different task instance Alert alert2 = new Alert(); alert2.setTitle("SQL Task B Result"); alert2.setContent(content); @@ -155,7 +154,6 @@ void testAddTaskResultAlertDifferentTaskSameContentNotDuplicate() { int secondCount = alertDao.addTaskResultAlert(alert2); Assertions.assertEquals(1, secondCount); - // Verify two alert rows exist for this workflow instance long count = alertDao.listAlerts(workflowInstanceId) .stream() .filter(a -> a.getAlertType() == AlertType.TASK_RESULT) @@ -164,17 +162,7 @@ void testAddTaskResultAlertDifferentTaskSameContentNotDuplicate() { } /** - * Verifies that concurrent calls to {@code addTaskResultAlert} with the same - * deduplication key result in exactly one inserted row. - *

- * The INSERT ... SELECT ... WHERE NOT EXISTS check is not atomic with the - * INSERT itself, so under concurrent access a race condition may cause a - * DuplicateKeyException from the uk_alert_dedup unique constraint. The DAO - * layer catches this exception and returns 0, ensuring exactly one row is - * inserted and no exception propagates to the caller. - *

- * Each losing thread must return 0, not throw an exception. If any thread throws, - * {@link Future#get()} will propagate the exception and fail the test. + * Concurrent calls to addTaskResultAlert with the same dedup key must insert exactly one row. */ @Test void testConcurrentAddTaskResultAlertIdempotent() throws Exception { @@ -211,22 +199,18 @@ void testConcurrentAddTaskResultAlertIdempotent() throws Exception { int successCount = 0; for (Future f : futures) { - // Future.get() propagates any exception thrown by the task. - // A losing thread must return 0, not throw. int inserted = f.get(); if (inserted > 0) { successCount++; } } - Assertions.assertEquals(1, successCount, - "Exactly one concurrent insert should succeed, but got " + successCount); + Assertions.assertEquals(1, successCount); long dbCount = alertDao.listAlerts(workflowInstanceId) .stream() .filter(a -> a.getAlertType() == AlertType.TASK_RESULT) .count(); - Assertions.assertEquals(1L, dbCount, - "Exactly one alert row should exist in the database, but found " + dbCount); + Assertions.assertEquals(1L, dbCount); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskSuccessLifecycleEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskSuccessLifecycleEventHandler.java index b75aec89b333..21b0f95acc74 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskSuccessLifecycleEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskSuccessLifecycleEventHandler.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.engine.task.lifecycle.handler; -import org.apache.dolphinscheduler.plugin.task.api.model.TaskAlertInfo; import org.apache.dolphinscheduler.server.master.engine.ILifecycleEventType; import org.apache.dolphinscheduler.server.master.engine.task.client.TaskExecutorClient; import org.apache.dolphinscheduler.server.master.engine.task.execution.ITaskExecution; @@ -52,25 +51,15 @@ public void handle(final ITaskStateAction taskStateAction, final IWorkflowExecution workflowExecution, final ITaskExecution taskExecution, final TaskSuccessLifecycleEvent taskSuccessEvent) { - // 1. State transition + DB persistence (may throw if state mismatch) taskStateAction.onSucceedEvent(workflowExecution, taskExecution, taskSuccessEvent); - // 2. Persist task-result alert only after the success state transition is confirmed if (taskSuccessEvent.isNeedAlert()) { - final TaskAlertInfo taskAlertInfo = taskSuccessEvent.getTaskAlertInfo(); - if (taskAlertInfo != null && taskAlertInfo.getAlertGroupId() != null - && taskAlertInfo.getAlertGroupId() > 0) { - workflowAlertManager.sendTaskResultAlert( - taskExecution.getWorkflowInstance(), - taskExecution.getTaskInstance(), - taskAlertInfo); - } else { - log.warn("Task: {} need alert but alertGroupId is invalid, skip sending alert", - taskExecution.getName()); - } + workflowAlertManager.sendTaskResultAlert( + taskExecution.getWorkflowInstance(), + taskExecution.getTaskInstance(), + taskSuccessEvent.getTaskAlertInfo()); } - // 3. ACK the worker — only after state transition and alert persistence are done taskExecutorClient.ackTaskExecutorLifecycleEvent( taskExecution, new ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck( diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManager.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManager.java index bb61b6305b26..cba5179b6e30 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManager.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManager.java @@ -210,17 +210,11 @@ public void sendWorkflowTimeoutAlert(WorkflowInstance workflowInstance) { alertDao.sendWorkflowTimeoutAlert(workflowInstance, projectUser); } - /** - * send task result alert - * - * @param workflowInstance workflow instance - * @param taskInstance task instance - * @param taskAlertInfo task alert info - */ public void sendTaskResultAlert(WorkflowInstance workflowInstance, TaskInstance taskInstance, TaskAlertInfo taskAlertInfo) { - if (taskAlertInfo == null || taskAlertInfo.getAlertGroupId() == null) { + if (taskAlertInfo == null || taskAlertInfo.getAlertGroupId() == null + || taskAlertInfo.getAlertGroupId() <= 0) { return; } diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManagerTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManagerTest.java index c4bb0eb92b7b..0fcdfe0cbcb7 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManagerTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/WorkflowAlertManagerTest.java @@ -56,15 +56,8 @@ class WorkflowAlertManagerTest { @InjectMocks private WorkflowAlertManager workflowAlertManager; - /** - * Verify that the production path {@link WorkflowAlertManager#sendTaskResultAlert} - * sets {@link AlertStatus#WAIT_EXECUTION} on the Alert before persisting it. - *

If this status is not set, the alert server will never pick up the alert - * because it only polls records whose status is WAIT_EXECUTION (0). - */ @Test - void sendTaskResultAlert_setsWaitExecutionStatus() { - // --- arrange --- + void testSendTaskResultAlertSetsWaitExecutionStatus() { WorkflowInstance workflowInstance = new WorkflowInstance(); workflowInstance.setId(1001); workflowInstance.setName("test-workflow"); @@ -83,16 +76,13 @@ void sendTaskResultAlert_setsWaitExecutionStatus() { Mockito.when(alertDao.addTaskResultAlert(Mockito.any(Alert.class))).thenReturn(1); - // --- act --- workflowAlertManager.sendTaskResultAlert(workflowInstance, taskInstance, taskAlertInfo); - // --- assert --- ArgumentCaptor alertCaptor = ArgumentCaptor.forClass(Alert.class); Mockito.verify(alertDao).addTaskResultAlert(alertCaptor.capture()); Alert captured = alertCaptor.getValue(); - Assertions.assertEquals(AlertStatus.WAIT_EXECUTION, captured.getAlertStatus(), - "Task-result alert must be initialized with WAIT_EXECUTION so the alert server can poll it"); + Assertions.assertEquals(AlertStatus.WAIT_EXECUTION, captured.getAlertStatus()); Assertions.assertEquals("SQL Task Result", captured.getTitle()); Assertions.assertEquals(WarningType.SUCCESS, captured.getWarningType()); Assertions.assertEquals(AlertType.TASK_RESULT, captured.getAlertType()); @@ -101,15 +91,11 @@ void sendTaskResultAlert_setsWaitExecutionStatus() { Assertions.assertEquals(3001L, captured.getWorkflowDefinitionCode()); Assertions.assertEquals(1, captured.getAlertGroupId()); Assertions.assertNotNull(captured.getCreateTime()); - Assertions.assertEquals(5001, captured.getTaskInstanceId(), - "Task instance ID must be set on the alert for idempotency key differentiation"); + Assertions.assertEquals(5001, captured.getTaskInstanceId()); } - /** - * When taskAlertInfo is null, no alert should be persisted. - */ @Test - void sendTaskResultAlert_nullTaskAlertInfo_doesNothing() { + void testSendTaskResultAlertNullTaskAlertInfoDoesNothing() { WorkflowInstance workflowInstance = new WorkflowInstance(); workflowInstance.setId(1001); TaskInstance taskInstance = new TaskInstance(); @@ -119,11 +105,8 @@ void sendTaskResultAlert_nullTaskAlertInfo_doesNothing() { Mockito.verifyNoInteractions(alertDao); } - /** - * When alertGroupId is null, no alert should be persisted. - */ @Test - void sendTaskResultAlert_nullAlertGroupId_doesNothing() { + void testSendTaskResultAlertNullAlertGroupIdDoesNothing() { WorkflowInstance workflowInstance = new WorkflowInstance(); workflowInstance.setId(1001); TaskInstance taskInstance = new TaskInstance(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index a3516d1e9ade..b390a5f42421 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -299,10 +299,7 @@ private String resultProcess(ResultSet resultSet) throws Exception { } if (Boolean.TRUE.equals(sqlParameters.getSendEmail())) { - log.info("SendAlert is enabled, preparing task result alert"); prepareTaskResultAlert(alertArray); - } else { - log.info("SendAlert is not enabled, skip task result alert"); } log.debug("execute sql result : {}", result); return result; @@ -329,10 +326,7 @@ private ArrayNode generateEmptyRow(ResultSet resultSet) throws SQLException { } /** - * Prepare task result alert info. - * Truncate the alert content to displayRows to avoid oversized RPC payload. - * - * @param resultJSONArray the full query result JSON array + * Prepare task result alert info, truncating content to displayRows. */ private void prepareTaskResultAlert(ArrayNode resultJSONArray) { TaskAlertInfo taskAlertInfo = new TaskAlertInfo(); @@ -340,7 +334,6 @@ private void prepareTaskResultAlert(ArrayNode resultJSONArray) { taskAlertInfo.setTitle(StringUtils.isNotEmpty(sqlParameters.getTitle()) ? sqlParameters.getTitle() : taskExecutionContext.getTaskName() + " query result sets"); - // Truncate content to displayRows to avoid oversized RPC payload int alertRows = sqlParameters.getDisplayRows() > 0 ? sqlParameters.getDisplayRows() : TaskConstants.DEFAULT_DISPLAY_ROWS; alertRows = Math.min(alertRows, resultJSONArray.size()); @@ -353,7 +346,7 @@ private void prepareTaskResultAlert(ArrayNode resultJSONArray) { taskExecutionContext.setNeedAlert(true); taskExecutionContext.setTaskAlertInfo(taskAlertInfo); - log.info("Prepare task result alert: title={}, alertGroupId={}, alertType={}, totalRows={}, alertRows={}", + log.debug("Prepare task result alert: title={}, alertGroupId={}, alertType={}, totalRows={}, alertRows={}", taskAlertInfo.getTitle(), taskAlertInfo.getAlertGroupId(), taskAlertInfo.getAlertType(), resultJSONArray.size(), alertRows); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java index ce2b7945f7fa..9eb876e39d3b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java @@ -491,26 +491,25 @@ private ResourceParametersHelper getResourceParametersHelperWithDatasourceType(D return resourceParametersHelper; } - /** - * Helper: create a SqlTask with sendEmail enabled, invoke prepareTaskResultAlert via reflection, - * and return the TaskExecutionContext for assertions. - */ private TaskExecutionContext createSqlTaskWithAlert(ArrayNode resultArray, int displayRows, String title, int groupId) { - String taskParams = "{\"type\":\"HIVE\",\"datasource\":1,\"sql\":\"select 1\"" - + ",\"sendEmail\":true" - + ",\"displayRows\":" + displayRows - + ",\"groupId\":" + groupId - + (title != null ? ",\"title\":\"" + title + "\"" : "") - + "}"; + SqlParameters sqlParameters = new SqlParameters(); + sqlParameters.setType("HIVE"); + sqlParameters.setDatasource(1); + sqlParameters.setSql("select 1"); + sqlParameters.setSendEmail(true); + sqlParameters.setDisplayRows(displayRows); + sqlParameters.setGroupId(groupId); + if (title != null) { + sqlParameters.setTitle(title); + } TaskExecutionContext ctx = new TaskExecutionContext(); - ctx.setTaskParams(taskParams); + ctx.setTaskParams(JSONUtils.toJsonString(sqlParameters)); ctx.setTaskName("test_sql_task"); ctx.setResourceParametersHelper(getResourceParametersHelperWithDatasourceType(DbType.HIVE)); SqlTask task = new SqlTask(ctx); - // Call prepareTaskResultAlert via reflection try { Method method = SqlTask.class.getDeclaredMethod("prepareTaskResultAlert", ArrayNode.class); method.setAccessible(true); @@ -521,60 +520,49 @@ private TaskExecutionContext createSqlTaskWithAlert(ArrayNode resultArray, int d return ctx; } - /** - * When sendEmail is true, prepareTaskResultAlert should set needAlert on TaskExecutionContext - * and populate TaskAlertInfo with correct alertGroupId, title, alertType and truncated content. - */ @Test - void testPrepareTaskResultAlert_setsNeedAlertAndTaskAlertInfo() { + void testPrepareTaskResultAlertSetsNeedAlertAndTaskAlertInfo() { ArrayNode resultArray = JSONUtils.createArrayNode(); resultArray.add(JSONUtils.parseObject("{\"id\":\"1\",\"name\":\"alice\"}")); resultArray.add(JSONUtils.parseObject("{\"id\":\"2\",\"name\":\"bob\"}")); TaskExecutionContext ctx = createSqlTaskWithAlert(resultArray, 10, "My Alert Title", 5); - Assertions.assertTrue(ctx.isNeedAlert(), "needAlert should be true when sendEmail is enabled"); + Assertions.assertTrue(ctx.isNeedAlert()); TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); - Assertions.assertNotNull(alertInfo, "taskAlertInfo should not be null"); + Assertions.assertNotNull(alertInfo); Assertions.assertEquals(5, alertInfo.getAlertGroupId()); Assertions.assertEquals("My Alert Title", alertInfo.getTitle()); Assertions.assertEquals(AlertType.TASK_RESULT, alertInfo.getAlertType()); Assertions.assertNotNull(alertInfo.getContent()); - // Content should contain all rows (within displayRows limit) ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); Assertions.assertEquals(2, contentArray.size()); } - /** - * Alert content should be truncated to displayRows to avoid oversized RPC payload. - */ @Test - void testPrepareTaskResultAlert_truncatesToDisplayRows() { + void testPrepareTaskResultAlertTruncatesToDisplayRows() { ArrayNode resultArray = JSONUtils.createArrayNode(); for (int i = 0; i < 50; i++) { resultArray.add(JSONUtils.parseObject("{\"id\":\"" + i + "\"}")); } - // displayRows = 3, but result has 50 rows -> content should be truncated to 3 + // displayRows = 3, result has 50 rows -> content should be truncated to 3 TaskExecutionContext ctx = createSqlTaskWithAlert(resultArray, 3, null, 1); TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); Assertions.assertNotNull(alertInfo); ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); - Assertions.assertEquals(3, contentArray.size(), "Alert content should be truncated to displayRows"); + Assertions.assertEquals(3, contentArray.size()); Assertions.assertEquals("0", contentArray.get(0).get("id").asText()); Assertions.assertEquals("1", contentArray.get(1).get("id").asText()); Assertions.assertEquals("2", contentArray.get(2).get("id").asText()); } - /** - * When displayRows is 0 (unset), should default to TaskConstants.DEFAULT_DISPLAY_ROWS (10). - */ @Test - void testPrepareTaskResultAlert_usesDefaultDisplayRowsWhenUnset() { + void testPrepareTaskResultAlertUsesDefaultDisplayRowsWhenUnset() { ArrayNode resultArray = JSONUtils.createArrayNode(); for (int i = 0; i < 20; i++) { resultArray.add(JSONUtils.parseObject("{\"id\":\"" + i + "\"}")); @@ -587,15 +575,11 @@ void testPrepareTaskResultAlert_usesDefaultDisplayRowsWhenUnset() { Assertions.assertNotNull(alertInfo); ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); - Assertions.assertEquals(TaskConstants.DEFAULT_DISPLAY_ROWS, contentArray.size(), - "Should default to DEFAULT_DISPLAY_ROWS when displayRows is 0"); + Assertions.assertEquals(TaskConstants.DEFAULT_DISPLAY_ROWS, contentArray.size()); } - /** - * When title is provided, it should be used as the alert title. - */ @Test - void testPrepareTaskResultAlert_usesCustomTitle() { + void testPrepareTaskResultAlertUsesCustomTitle() { ArrayNode resultArray = JSONUtils.createArrayNode(); resultArray.add(JSONUtils.parseObject("{\"id\":\"1\"}")); @@ -606,11 +590,8 @@ void testPrepareTaskResultAlert_usesCustomTitle() { Assertions.assertEquals("Custom Title", alertInfo.getTitle()); } - /** - * When title is empty, should use taskName + " query result sets" as default. - */ @Test - void testPrepareTaskResultAlert_usesDefaultTitleWhenEmpty() { + void testPrepareTaskResultAlertUsesDefaultTitleWhenEmpty() { ArrayNode resultArray = JSONUtils.createArrayNode(); resultArray.add(JSONUtils.parseObject("{\"id\":\"1\"}")); @@ -621,14 +602,10 @@ void testPrepareTaskResultAlert_usesDefaultTitleWhenEmpty() { Assertions.assertEquals("test_sql_task query result sets", alertInfo.getTitle()); } - /** - * When sendEmail is false, resultProcess should NOT set needAlert or taskAlertInfo. - */ @Test - void testResultProcess_sendEmailDisabled_doesNotSetNeedAlert() throws Exception { + void testResultProcessSendEmailDisabledDoesNotSetNeedAlert() throws Exception { // Build a SqlTask with sendEmail = false String taskParams = "{\"type\":\"HIVE\",\"datasource\":1,\"sql\":\"select 1\",\"sendEmail\":false}"; - TaskExecutionContext ctx = new TaskExecutionContext(); ctx.setTaskParams(taskParams); ctx.setTaskName("no_alert_task"); @@ -648,16 +625,12 @@ void testResultProcess_sendEmailDisabled_doesNotSetNeedAlert() throws Exception resultProcessMethod.setAccessible(true); resultProcessMethod.invoke(task, mockResultSet); - Assertions.assertFalse(ctx.isNeedAlert(), "needAlert should remain false when sendEmail is disabled"); - Assertions.assertNull(ctx.getTaskAlertInfo(), "taskAlertInfo should remain null when sendEmail is disabled"); + Assertions.assertFalse(ctx.isNeedAlert()); + Assertions.assertNull(ctx.getTaskAlertInfo()); } - /** - * When the query result is empty, resultProcess should still prepare the alert - * using the generated empty row so the user is notified that the query returned no data. - */ @Test - void testResultProcess_emptyResultSet_prepareAlertWithEmptyRow() throws Exception { + void testResultProcessEmptyResultSetPreparesAlertWithEmptyRow() throws Exception { String taskParams = "{\"type\":\"HIVE\",\"datasource\":1,\"sql\":\"select 1\"" + ",\"sendEmail\":true,\"displayRows\":10,\"groupId\":3,\"title\":\"empty result\"}"; @@ -680,7 +653,7 @@ void testResultProcess_emptyResultSet_prepareAlertWithEmptyRow() throws Exceptio resultProcessMethod.setAccessible(true); resultProcessMethod.invoke(task, mockResultSet); - Assertions.assertTrue(ctx.isNeedAlert(), "needAlert should be true even for empty result set"); + Assertions.assertTrue(ctx.isNeedAlert()); TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); Assertions.assertNotNull(alertInfo); @@ -688,22 +661,14 @@ void testResultProcess_emptyResultSet_prepareAlertWithEmptyRow() throws Exceptio Assertions.assertEquals("empty result", alertInfo.getTitle()); Assertions.assertEquals(AlertType.TASK_RESULT, alertInfo.getAlertType()); - // Content should be the empty row ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); - Assertions.assertEquals(1, contentArray.size(), "Empty result alert should contain one empty row"); + Assertions.assertEquals(1, contentArray.size()); Assertions.assertEquals("", contentArray.get(0).get("id").asText()); Assertions.assertEquals("", contentArray.get(0).get("name").asText()); } - /** - * Verify that TaskExecutorSuccessLifecycleEvent.of() carries needAlert and taskAlertInfo - * from TaskExecutionContext to the event, ensuring the alert info survives the Worker→Master RPC. - * Since task-executor module has no test infrastructure, we validate indirectly by confirming - * that TaskExecutionContext properly stores and exposes these fields for the event builder to read. - */ @Test - void testTaskExecutionContext_carriesAlertInfoForEventPropagation() { - // Simulate what SqlTask.prepareTaskResultAlert does + void testTaskExecutionContextCarriesAlertInfoForEventPropagation() { TaskExecutionContext ctx = new TaskExecutionContext(); ctx.setNeedAlert(true); @@ -714,14 +679,12 @@ void testTaskExecutionContext_carriesAlertInfoForEventPropagation() { alertInfo.setAlertType(AlertType.TASK_RESULT); ctx.setTaskAlertInfo(alertInfo); - // Verify the fields are retrievable — this is what TaskExecutorSuccessLifecycleEvent.of() reads Assertions.assertTrue(ctx.isNeedAlert()); Assertions.assertNotNull(ctx.getTaskAlertInfo()); Assertions.assertEquals("test title", ctx.getTaskAlertInfo().getTitle()); Assertions.assertEquals(7, ctx.getTaskAlertInfo().getAlertGroupId()); Assertions.assertEquals(AlertType.TASK_RESULT, ctx.getTaskAlertInfo().getAlertType()); - // Verify default state when alert is not set TaskExecutionContext ctx2 = new TaskExecutionContext(); Assertions.assertFalse(ctx2.isNeedAlert()); Assertions.assertNull(ctx2.getTaskAlertInfo()); From b2ff7d492f3112b7a717168ab46fcf622b55b393 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Fri, 11 Sep 2026 16:22:15 +0800 Subject: [PATCH 30/43] Move task-result alert dedup logic from SQL to application code --- docs/docs/en/guide/upgrade/incompatible.md | 2 +- docs/docs/zh/guide/upgrade/incompatible.md | 2 +- .../apache/dolphinscheduler/dao/AlertDao.java | 19 ++++++--------- .../dao/mapper/AlertMapper.java | 7 +++--- .../dao/mapper/AlertMapper.xml | 24 +++++++------------ .../dao/repository/impl/AlertDaoTest.java | 2 ++ 6 files changed, 23 insertions(+), 33 deletions(-) diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index e5295b500c96..9c4f7c9a2655 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -58,6 +58,6 @@ This document records the incompatible updates between each version. You need to ## 3.5.0 -* Add a unique constraint `uk_alert_dedup` on `(sign, workflow_instance_id, alert_type)` to the `t_ds_alert` table. The upgrade script automatically cleans up existing duplicate rows (keeping the one with the largest id) before adding the unique index. This constraint ensures database-level idempotency for task-result alerts under concurrent delivery.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* Add a unique constraint `uk_alert_dedup` on `(sign, workflow_instance_id, alert_type)` to the `t_ds_alert` table. The upgrade script automatically cleans up existing duplicate rows (keeping the one with the largest id) before adding the unique index. Task-result alerts are inserted by application code; the constraint rejects duplicates and the DAO layer treats the resulting `DuplicateKeyException` as a skip, ensuring idempotency for task-result alerts under concurrent delivery.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` now explicitly rejects a null `alertType` by throwing `IllegalArgumentException` with an error log, instead of throwing a raw `NullPointerException`. During a rolling upgrade, **the Alert Server must be upgraded before Master/Worker**, otherwise an old Alert Server cannot recognize newly introduced `alert_type` enum values (e.g. `TASK_RESULT`) and alert delivery will fail. See the [upgrade guide](upgrade.md#rolling-upgrade-order) for details.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 7f27fe4feeba..fa87c2ca2ab5 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -58,6 +58,6 @@ ## 3.5.0 -* 为 `t_ds_alert` 表的 `(sign, workflow_instance_id, alert_type)` 新增唯一约束 `uk_alert_dedup`。升级脚本会自动清理已存在的重复行（保留 id 最大的一条），之后添加唯一索引。该约束确保任务结果告警在并发投递时数据库层面幂等。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* 为 `t_ds_alert` 表的 `(sign, workflow_instance_id, alert_type)` 新增唯一约束 `uk_alert_dedup`。升级脚本会自动清理已存在的重复行（保留 id 最大的一条），之后添加唯一索引。任务结果告警由应用代码插入，重复插入由该约束拒绝，DAO 层将产生的 `DuplicateKeyException` 视为跳过，从而确保任务结果告警在并发投递时幂等。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` 对 null `alertType` 增加了显式拒绝（抛 `IllegalArgumentException` 并记录错误日志），而非直接抛 `NullPointerException`。滚动升级时**必须先升级 Alert Server，再升级 Master/Worker**，否则旧 Alert Server 无法识别新增的 `alert_type` 枚举值（如 `TASK_RESULT`），会导致告警投递失败。详见[升级文档](upgrade.md#滚动升降级顺序)。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index b20b59bb17d0..4f7be39c5f00 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -110,8 +110,9 @@ public int addAlert(Alert alert) { } /** - * Insert a task-result alert idempotently. If an alert with the same sign, - * workflow instance id and alert type already exists, the insert is skipped. + * Insert a task-result alert idempotently. The uk_alert_dedup unique constraint on + * (sign, workflow_instance_id, alert_type) rejects duplicates; a DuplicateKeyException + * is caught here and treated as a skip. * * @return insert count (1 if inserted, 0 if skipped) */ @@ -124,18 +125,12 @@ public int addTaskResultAlert(Alert alert) { String sign = generateSign(alert); alert.setSign(sign); try { - int count = alertMapper.insertTaskResultAlertIfAbsent(alert); - if (count > 0) { - log.info("add task result alert to db , alert: {}", alert); - } else { - log.info("skip duplicate task result alert, sign: {}, workflowInstanceId: {}", sign, - alert.getWorkflowInstanceId()); - } + int count = alertMapper.insertTaskResultAlert(alert); + log.info("add task result alert to db , alert: {}", alert); return count; } catch (DuplicateKeyException e) { - // Concurrent race: NOT EXISTS passed but another thread inserted first. - // The uk_alert_dedup unique constraint caught it — treat as a skip. - log.info("skip duplicate task result alert (concurrent race), sign: {}, workflowInstanceId: {}", sign, + // The uk_alert_dedup unique constraint caught a duplicate — treat as a skip. + log.info("skip duplicate task result alert, sign: {}, workflowInstanceId: {}", sign, alert.getWorkflowInstanceId()); return 0; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java index dcfdf1ec0e86..abeabf756cfe 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertMapper.java @@ -44,10 +44,11 @@ void insertAlertWhenServerCrash(@Param("alert") Alert alert, @Param("crashAlarmSuppressionStartTime") Date crashAlarmSuppressionStartTime); /** - * Insert a task-result alert only if no alert with the same sign, workflow instance id and alert type - * already exists. This makes the insert idempotent against at-least-once event delivery. + * Insert a task-result alert. Idempotency is enforced by the uk_alert_dedup unique constraint + * on (sign, workflow_instance_id, alert_type); the DAO layer catches DuplicateKeyException + * and treats it as a skip. */ - int insertTaskResultAlertIfAbsent(@Param("alert") Alert alert); + int insertTaskResultAlert(@Param("alert") Alert alert); void deleteByWorkflowInstanceId(@Param("workflowInstanceId") Integer processInstanceId); diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml index 9b6b1bf1b7ac..4fd50f55b6a1 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/AlertMapper.xml @@ -52,26 +52,18 @@ - + INSERT INTO t_ds_alert(sign, title, content, alert_status, warning_type, log, alertgroup_id, create_time, update_time, project_code, workflow_definition_code, workflow_instance_id, alert_type) - SELECT #{alert.sign}, #{alert.title}, #{alert.content}, #{alert.alertStatus.code}, - #{alert.warningType.code}, #{alert.log}, #{alert.alertGroupId}, #{alert.createTime}, - #{alert.updateTime}, #{alert.projectCode}, #{alert.workflowDefinitionCode}, - #{alert.workflowInstanceId}, #{alert.alertType.code} - WHERE NOT EXISTS ( - SELECT 1 FROM t_ds_alert - WHERE sign = #{alert.sign} - AND workflow_instance_id = #{alert.workflowInstanceId} - AND alert_type = #{alert.alertType.code} - ) + VALUES (#{alert.sign}, #{alert.title}, #{alert.content}, #{alert.alertStatus.code}, + #{alert.warningType.code}, #{alert.log}, #{alert.alertGroupId}, #{alert.createTime}, + #{alert.updateTime}, #{alert.projectCode}, #{alert.workflowDefinitionCode}, + #{alert.workflowInstanceId}, #{alert.alertType.code}) select From 79a0e12a401e7ef985be846297885c9aa45435df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Sat, 10 Oct 2026 17:02:16 +0800 Subject: [PATCH 37/43] Dedup task-result alerts by sign only --- docs/docs/en/guide/upgrade/incompatible.md | 3 ++- docs/docs/zh/guide/upgrade/incompatible.md | 3 ++- .../main/java/org/apache/dolphinscheduler/dao/AlertDao.java | 2 +- .../src/main/resources/sql/dolphinscheduler_h2.sql | 2 +- .../src/main/resources/sql/dolphinscheduler_mysql.sql | 2 +- .../src/main/resources/sql/dolphinscheduler_postgresql.sql | 2 +- .../sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql | 2 +- .../upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql | 2 +- 8 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index 6b0924d7cdd6..9d9462be15b8 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -58,6 +58,7 @@ This document records the incompatible updates between each version. You need to ## 3.5.0 -* Add a unique constraint `uk_alert_dedup` on `(sign, workflow_instance_id, alert_type)` to the `t_ds_alert` table. The upgrade script does not remove any existing data; if duplicate rows already exist, the index creation will fail and you need to resolve them manually. Task-result alerts are inserted by application code; the constraint rejects duplicates and the DAO layer treats the resulting `DuplicateKeyException` as a skip, ensuring idempotency for task-result alerts under concurrent delivery.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* Add a unique constraint `uk_alert_dedup` on `sign` to the `t_ds_alert` table. The upgrade script does not remove any existing data; if duplicate rows already exist, the index creation will fail and you need to resolve them manually. Task-result alerts are inserted by application code; the constraint rejects duplicates and the DAO layer treats the resulting `DuplicateKeyException` as a skip, ensuring idempotency for task-result alerts under concurrent delivery. The task-result `sign` already embeds the task instance id, so different tasks never collide while a repeated task maps to the same sign. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` now explicitly rejects a null `alertType` by throwing `IllegalArgumentException` with an error log, instead of throwing a raw `NullPointerException`. During a rolling upgrade, **the Alert Server must be upgraded before Master/Worker**, otherwise an old Alert Server cannot recognize newly introduced `alert_type` enum values (e.g. `TASK_RESULT`) and alert delivery will fail. See the [upgrade guide](upgrade.md#rolling-upgrade-order) for details.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +> AI生成 diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 9a544a2c6a8b..841a131f5ee6 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -58,6 +58,7 @@ ## 3.5.0 -* 为 `t_ds_alert` 表的 `(sign, workflow_instance_id, alert_type)` 新增唯一约束 `uk_alert_dedup`。升级脚本不会删除任何已有数据；如果已存在重复行，索引创建将失败，需要由用户自行处理。任务结果告警由应用代码插入，重复插入由该约束拒绝，DAO 层将产生的 `DuplicateKeyException` 视为跳过，从而确保任务结果告警在并发投递时幂等。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* 为 `t_ds_alert` 表的 `sign` 新增唯一约束 `uk_alert_dedup`。升级脚本不会删除任何已有数据；如果已存在重复行，索引创建将失败，需要由用户自行处理。任务结果告警由应用代码插入，重复插入由该约束拒绝，DAO 层将产生的 `DuplicateKeyException` 视为跳过，从而确保任务结果告警在并发投递时幂等。任务结果的 `sign` 已嵌入任务实例 ID，因此同一任务重复成功对应相同 sign、不同任务即使结果相同也不会碰撞。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` 对 null `alertType` 增加了显式拒绝（抛 `IllegalArgumentException` 并记录错误日志），而非直接抛 `NullPointerException`。滚动升级时**必须先升级 Alert Server，再升级 Master/Worker**，否则旧 Alert Server 无法识别新增的 `alert_type` 枚举值（如 `TASK_RESULT`），会导致告警投递失败。详见[升级文档](upgrade.md#滚动升降级顺序)。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +> AI生成 diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index 23632b5149de..f53c3fc461f6 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -117,7 +117,7 @@ public int addAlert(Alert alert) { /** * Insert a task-result alert idempotently. The uk_alert_dedup unique constraint on - * (sign, workflow_instance_id, alert_type) rejects duplicates; a DuplicateKeyException + * sign rejects duplicates; a DuplicateKeyException * is caught here and treated as a skip. * * @return insert count (1 if inserted, 0 if skipped) diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index 597ff1e7e73f..848bd2171e48 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -286,7 +286,7 @@ CREATE TABLE t_ds_alert alert_type int(11) DEFAULT NULL, PRIMARY KEY (id), KEY idx_sign (sign), - UNIQUE KEY uk_alert_dedup (sign, workflow_instance_id, alert_type) + UNIQUE KEY uk_alert_dedup (sign) ); -- ---------------------------- diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index 0986b4da1f19..f12cda2117c8 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -292,7 +292,7 @@ CREATE TABLE `t_ds_alert` ( PRIMARY KEY (`id`), KEY `idx_status` (`alert_status`) USING BTREE, KEY `idx_sign` (`sign`) USING BTREE, - UNIQUE KEY `uk_alert_dedup` (`sign`, `workflow_instance_id`, `alert_type`) + UNIQUE KEY `uk_alert_dedup` (`sign`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin; -- ---------------------------- diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index 645db27dd747..fa33a7ceccee 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -226,7 +226,7 @@ comment on column t_ds_alert.sign is 'sign=sha1(content)'; create index idx_status on t_ds_alert (alert_status); create index idx_sign on t_ds_alert (sign); -create unique index uk_alert_dedup on t_ds_alert (sign, workflow_instance_id, alert_type); +create unique index uk_alert_dedup on t_ds_alert (sign); -- -- Table structure for table t_ds_alertgroup diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql index 1e9e0653712d..75203b231a34 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql @@ -16,5 +16,5 @@ */ -- If the table already has duplicate rows, index creation will fail; resolve them manually first. -ALTER TABLE `t_ds_alert` ADD UNIQUE INDEX `uk_alert_dedup` (`sign`, `workflow_instance_id`, `alert_type`); +ALTER TABLE `t_ds_alert` ADD UNIQUE INDEX `uk_alert_dedup` (`sign`); diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql index 429c501f2c86..81ebe5069873 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -16,4 +16,4 @@ */ -- If the table already has duplicate rows, index creation will fail; resolve them manually first. -CREATE UNIQUE INDEX IF NOT EXISTS uk_alert_dedup ON t_ds_alert (sign, workflow_instance_id, alert_type); +CREATE UNIQUE INDEX IF NOT EXISTS uk_alert_dedup ON t_ds_alert (sign); From 7596e453404a777e0c5d9b91053d2a6150bd83f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Sat, 10 Oct 2026 17:11:01 +0800 Subject: [PATCH 38/43] Drop redundant idx_sign superseded by the alert dedup unique index --- docs/docs/en/guide/upgrade/incompatible.md | 5 ++--- docs/docs/zh/guide/upgrade/incompatible.md | 5 ++--- .../src/main/resources/sql/dolphinscheduler_h2.sql | 1 - .../src/main/resources/sql/dolphinscheduler_mysql.sql | 1 - .../src/main/resources/sql/dolphinscheduler_postgresql.sql | 1 - .../sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql | 2 ++ .../upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql | 2 ++ 7 files changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index 9d9462be15b8..14555c5cffc5 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -58,7 +58,6 @@ This document records the incompatible updates between each version. You need to ## 3.5.0 -* Add a unique constraint `uk_alert_dedup` on `sign` to the `t_ds_alert` table. The upgrade script does not remove any existing data; if duplicate rows already exist, the index creation will fail and you need to resolve them manually. Task-result alerts are inserted by application code; the constraint rejects duplicates and the DAO layer treats the resulting `DuplicateKeyException` as a skip, ensuring idempotency for task-result alerts under concurrent delivery. The task-result `sign` already embeds the task instance id, so different tasks never collide while a repeated task maps to the same sign. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* Add a unique constraint `uk_alert_dedup` on `sign` to the `t_ds_alert` table. The upgrade script does not remove any existing data; if duplicate rows already exist, the index creation will fail and you need to resolve them manually. Task-result alerts are inserted by application code; the constraint rejects duplicates and the DAO layer treats the resulting `DuplicateKeyException` as a skip. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* Drop the legacy `idx_sign` index on `t_ds_alert`; the new `uk_alert_dedup` unique index on the same column fully covers it. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` now explicitly rejects a null `alertType` by throwing `IllegalArgumentException` with an error log, instead of throwing a raw `NullPointerException`. During a rolling upgrade, **the Alert Server must be upgraded before Master/Worker**, otherwise an old Alert Server cannot recognize newly introduced `alert_type` enum values (e.g. `TASK_RESULT`) and alert delivery will fail. See the [upgrade guide](upgrade.md#rolling-upgrade-order) for details.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) - -> AI生成 diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 841a131f5ee6..4d109289ae0d 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -58,7 +58,6 @@ ## 3.5.0 -* 为 `t_ds_alert` 表的 `sign` 新增唯一约束 `uk_alert_dedup`。升级脚本不会删除任何已有数据；如果已存在重复行，索引创建将失败，需要由用户自行处理。任务结果告警由应用代码插入，重复插入由该约束拒绝，DAO 层将产生的 `DuplicateKeyException` 视为跳过，从而确保任务结果告警在并发投递时幂等。任务结果的 `sign` 已嵌入任务实例 ID，因此同一任务重复成功对应相同 sign、不同任务即使结果相同也不会碰撞。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* 为 `t_ds_alert` 表的 `sign` 新增唯一约束 `uk_alert_dedup`。升级脚本不会删除任何已有数据；如果已存在重复行，索引创建将失败，需要由用户自行处理。任务结果告警由应用代码插入，重复插入由该约束拒绝，DAO 层将产生的 `DuplicateKeyException` 视为跳过。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* 删除 `t_ds_alert` 表上旧的 `idx_sign` 索引，同列上新增的 `uk_alert_dedup` 唯一索引已完全覆盖其用途。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` 对 null `alertType` 增加了显式拒绝（抛 `IllegalArgumentException` 并记录错误日志），而非直接抛 `NullPointerException`。滚动升级时**必须先升级 Alert Server，再升级 Master/Worker**，否则旧 Alert Server 无法识别新增的 `alert_type` 枚举值（如 `TASK_RESULT`），会导致告警投递失败。详见[升级文档](upgrade.md#滚动升降级顺序)。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) - -> AI生成 diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index 848bd2171e48..8f0cb148a07b 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -285,7 +285,6 @@ CREATE TABLE t_ds_alert workflow_instance_id int(11) DEFAULT NULL, alert_type int(11) DEFAULT NULL, PRIMARY KEY (id), - KEY idx_sign (sign), UNIQUE KEY uk_alert_dedup (sign) ); diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index f12cda2117c8..b658c2862911 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -291,7 +291,6 @@ CREATE TABLE `t_ds_alert` ( `alert_type` int(11) DEFAULT NULL COMMENT 'alert_type', PRIMARY KEY (`id`), KEY `idx_status` (`alert_status`) USING BTREE, - KEY `idx_sign` (`sign`) USING BTREE, UNIQUE KEY `uk_alert_dedup` (`sign`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin; diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index fa33a7ceccee..0476eb283c8e 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -225,7 +225,6 @@ CREATE TABLE t_ds_alert ( comment on column t_ds_alert.sign is 'sign=sha1(content)'; create index idx_status on t_ds_alert (alert_status); -create index idx_sign on t_ds_alert (sign); create unique index uk_alert_dedup on t_ds_alert (sign); -- diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql index 75203b231a34..c51aacc8c5b9 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql @@ -17,4 +17,6 @@ -- If the table already has duplicate rows, index creation will fail; resolve them manually first. ALTER TABLE `t_ds_alert` ADD UNIQUE INDEX `uk_alert_dedup` (`sign`); +-- The new unique index fully covers the legacy idx_sign index on the same column; drop the redundant one. +ALTER TABLE `t_ds_alert` DROP INDEX `idx_sign`; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql index 81ebe5069873..545270cf006e 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql @@ -17,3 +17,5 @@ -- If the table already has duplicate rows, index creation will fail; resolve them manually first. CREATE UNIQUE INDEX IF NOT EXISTS uk_alert_dedup ON t_ds_alert (sign); +-- The new unique index fully covers the legacy idx_sign index on the same column; drop the redundant one. +DROP INDEX IF EXISTS idx_sign; From 90d45770d6fc358173ca7c06ee7ef7e389e910be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Sat, 10 Oct 2026 17:33:53 +0800 Subject: [PATCH 39/43] update incompatible.md --- docs/docs/en/guide/upgrade/incompatible.md | 1 + docs/docs/zh/guide/upgrade/incompatible.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index 14555c5cffc5..d16bd4f41245 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -61,3 +61,4 @@ This document records the incompatible updates between each version. You need to * Add a unique constraint `uk_alert_dedup` on `sign` to the `t_ds_alert` table. The upgrade script does not remove any existing data; if duplicate rows already exist, the index creation will fail and you need to resolve them manually. Task-result alerts are inserted by application code; the constraint rejects duplicates and the DAO layer treats the resulting `DuplicateKeyException` as a skip. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * Drop the legacy `idx_sign` index on `t_ds_alert`; the new `uk_alert_dedup` unique index on the same column fully covers it. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` now explicitly rejects a null `alertType` by throwing `IllegalArgumentException` with an error log, instead of throwing a raw `NullPointerException`. During a rolling upgrade, **the Alert Server must be upgraded before Master/Worker**, otherwise an old Alert Server cannot recognize newly introduced `alert_type` enum values (e.g. `TASK_RESULT`) and alert delivery will fail. See the [upgrade guide](upgrade.md#rolling-upgrade-order) for details.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) + diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 4d109289ae0d..04eda97eaa2e 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -61,3 +61,4 @@ * 为 `t_ds_alert` 表的 `sign` 新增唯一约束 `uk_alert_dedup`。升级脚本不会删除任何已有数据；如果已存在重复行，索引创建将失败，需要由用户自行处理。任务结果告警由应用代码插入，重复插入由该约束拒绝，DAO 层将产生的 `DuplicateKeyException` 视为跳过。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * 删除 `t_ds_alert` 表上旧的 `idx_sign` 索引，同列上新增的 `uk_alert_dedup` 唯一索引已完全覆盖其用途。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` 对 null `alertType` 增加了显式拒绝（抛 `IllegalArgumentException` 并记录错误日志），而非直接抛 `NullPointerException`。滚动升级时**必须先升级 Alert Server，再升级 Master/Worker**，否则旧 Alert Server 无法识别新增的 `alert_type` 枚举值（如 `TASK_RESULT`），会导致告警投递失败。详见[升级文档](upgrade.md#滚动升降级顺序)。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) + From f4b37e55b72151e48102b2e48d2604fc52c42469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Mon, 12 Oct 2026 14:25:36 +0800 Subject: [PATCH 40/43] Limit SQL task-result alert content size and mark omissions --- docs/docs/en/guide/upgrade/incompatible.md | 1 + docs/docs/zh/guide/upgrade/incompatible.md | 1 + .../plugin/task/sql/SqlTask.java | 53 ++++++++++++++- .../plugin/task/sql/SqlTaskTest.java | 64 +++++++++++++++++-- 4 files changed, 111 insertions(+), 8 deletions(-) diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index d16bd4f41245..1f13cd845edc 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -61,4 +61,5 @@ This document records the incompatible updates between each version. You need to * Add a unique constraint `uk_alert_dedup` on `sign` to the `t_ds_alert` table. The upgrade script does not remove any existing data; if duplicate rows already exist, the index creation will fail and you need to resolve them manually. Task-result alerts are inserted by application code; the constraint rejects duplicates and the DAO layer treats the resulting `DuplicateKeyException` as a skip. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * Drop the legacy `idx_sign` index on `t_ds_alert`; the new `uk_alert_dedup` unique index on the same column fully covers it. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` now explicitly rejects a null `alertType` by throwing `IllegalArgumentException` with an error log, instead of throwing a raw `NullPointerException`. During a rolling upgrade, **the Alert Server must be upgraded before Master/Worker**, otherwise an old Alert Server cannot recognize newly introduced `alert_type` enum values (e.g. `TASK_RESULT`) and alert delivery will fail. See the [upgrade guide](upgrade.md#rolling-upgrade-order) for details.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* SQL task-result alert content is size-limited: at most 30 rows are kept, string field values longer than 100 characters are truncated with an ellipsis, and rows beyond 30 are represented by an omission marker. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 04eda97eaa2e..214c6cf321b4 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -61,4 +61,5 @@ * 为 `t_ds_alert` 表的 `sign` 新增唯一约束 `uk_alert_dedup`。升级脚本不会删除任何已有数据；如果已存在重复行，索引创建将失败，需要由用户自行处理。任务结果告警由应用代码插入，重复插入由该约束拒绝，DAO 层将产生的 `DuplicateKeyException` 视为跳过。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * 删除 `t_ds_alert` 表上旧的 `idx_sign` 索引，同列上新增的 `uk_alert_dedup` 唯一索引已完全覆盖其用途。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` 对 null `alertType` 增加了显式拒绝（抛 `IllegalArgumentException` 并记录错误日志），而非直接抛 `NullPointerException`。滚动升级时**必须先升级 Alert Server，再升级 Master/Worker**，否则旧 Alert Server 无法识别新增的 `alert_type` 枚举值（如 `TASK_RESULT`），会导致告警投递失败。详见[升级文档](upgrade.md#滚动升降级顺序)。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* SQL 任务结果告警内容有大小限制：最多保留 30 行，超过 100 字符的字符串字段值会被截断并以省略号标记，超出 30 行的部分以省略标记对象表示。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index 29e80244965b..b5d7e80baa26 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -57,6 +57,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -67,6 +68,7 @@ import lombok.extern.slf4j.Slf4j; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -91,6 +93,16 @@ public class SqlTask extends AbstractTask { */ private static final int QUERY_LIMIT = 10000; + /** + * max rows kept in the task-result alert content; rows beyond this are replaced by an omission marker + */ + private static final int ALERT_MAX_ROWS = 30; + + /** + * max length of a single string field value kept in the alert content; longer values are truncated with an ellipsis + */ + private static final int ALERT_MAX_ROW_VALUE_LENGTH = 100; + private final SQLTaskExecutionContext sqlTaskExecutionContext; private final DbType dbType; @@ -334,7 +346,7 @@ private void prepareTaskResultAlert(ArrayNode resultJSONArray) { taskAlertInfo.setTitle(StringUtils.isNotEmpty(sqlParameters.getTitle()) ? sqlParameters.getTitle() : taskExecutionContext.getTaskName() + " query result sets"); - taskAlertInfo.setContent(JSONUtils.toJsonString(resultJSONArray)); + taskAlertInfo.setContent(limitAlertContent(resultJSONArray)); taskAlertInfo.setAlertType(AlertType.TASK_RESULT); taskExecutionContext.setNeedAlert(true); @@ -344,6 +356,45 @@ private void prepareTaskResultAlert(ArrayNode resultJSONArray) { resultJSONArray.size()); } + /** + * Build a size-limited alert content that stays a valid JSON array. + * Rows beyond {@link #ALERT_MAX_ROWS} are dropped and a trailing marker object is appended; + * string field values longer than {@link #ALERT_MAX_ROW_VALUE_LENGTH} are truncated with an ellipsis. + */ + private String limitAlertContent(ArrayNode resultJSONArray) { + int totalRows = resultJSONArray.size(); + ArrayNode limited = JSONUtils.createArrayNode(); + int rowsToKeep = Math.min(totalRows, ALERT_MAX_ROWS); + for (int i = 0; i < rowsToKeep; i++) { + limited.add(limitRowValues(resultJSONArray.get(i))); + } + if (totalRows > ALERT_MAX_ROWS) { + ObjectNode omitted = JSONUtils.createObjectNode(); + omitted.put("...", (totalRows - ALERT_MAX_ROWS) + " more rows omitted"); + limited.add(omitted); + } + return JSONUtils.toJsonString(limited); + } + + private JsonNode limitRowValues(JsonNode row) { + if (!row.isObject()) { + return row; + } + ObjectNode limitedRow = JSONUtils.createObjectNode(); + Iterator> fields = row.fields(); + while (fields.hasNext()) { + Map.Entry entry = fields.next(); + JsonNode value = entry.getValue(); + if (value.isTextual() && value.textValue().length() > ALERT_MAX_ROW_VALUE_LENGTH) { + limitedRow.put(entry.getKey(), + value.textValue().substring(0, ALERT_MAX_ROW_VALUE_LENGTH) + "..."); + } else { + limitedRow.set(entry.getKey(), value); + } + } + return limitedRow; + } + private String executeQuery(Connection connection, SqlBinds sqlBinds, String handlerType) throws Exception { try (PreparedStatement statement = prepareStatementAndBind(connection, sqlBinds)) { log.info("{} statement execute query, for sql: {}", handlerType, sqlBinds.getSql()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java index bcd5677fc1a3..e7e7dd93a767 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java @@ -38,6 +38,7 @@ import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.spi.enums.DbType; +import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; @@ -53,6 +54,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; @@ -542,9 +544,10 @@ void testPrepareTaskResultAlertSetsNeedAlertAndTaskAlertInfo() { } @Test - void testPrepareTaskResultAlertKeepsFullContentRegardlessOfDisplayRows() { + void testPrepareTaskResultAlertKeepsFullContentRegardlessOfDisplayRows() throws Exception { + int totalRows = readStaticIntConstant("ALERT_MAX_ROWS") - 1; ArrayNode resultArray = JSONUtils.createArrayNode(); - for (int i = 0; i < 50; i++) { + for (int i = 0; i < totalRows; i++) { resultArray.add(JSONUtils.parseObject("{\"id\":\"" + i + "\"}")); } @@ -555,15 +558,16 @@ void testPrepareTaskResultAlertKeepsFullContentRegardlessOfDisplayRows() { Assertions.assertNotNull(alertInfo); ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); - Assertions.assertEquals(50, contentArray.size()); + Assertions.assertEquals(totalRows, contentArray.size()); Assertions.assertEquals("0", contentArray.get(0).get("id").asText()); - Assertions.assertEquals("49", contentArray.get(49).get("id").asText()); + Assertions.assertEquals(String.valueOf(totalRows - 1), contentArray.get(totalRows - 1).get("id").asText()); } @Test - void testPrepareTaskResultAlertKeepsFullContentWhenDisplayRowsUnset() { + void testPrepareTaskResultAlertKeepsFullContentWhenDisplayRowsUnset() throws Exception { + int totalRows = readStaticIntConstant("ALERT_MAX_ROWS") - 1; ArrayNode resultArray = JSONUtils.createArrayNode(); - for (int i = 0; i < 20; i++) { + for (int i = 0; i < totalRows; i++) { resultArray.add(JSONUtils.parseObject("{\"id\":\"" + i + "\"}")); } @@ -574,7 +578,7 @@ void testPrepareTaskResultAlertKeepsFullContentWhenDisplayRowsUnset() { Assertions.assertNotNull(alertInfo); ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); - Assertions.assertEquals(20, contentArray.size()); + Assertions.assertEquals(totalRows, contentArray.size()); } @Test @@ -601,6 +605,52 @@ void testPrepareTaskResultAlertUsesDefaultTitleWhenEmpty() { Assertions.assertEquals("test_sql_task query result sets", alertInfo.getTitle()); } + @Test + void testPrepareTaskResultAlertTruncatesRowsBeyondMax() throws Exception { + int maxRows = readStaticIntConstant("ALERT_MAX_ROWS"); + int totalRows = maxRows + 10; + ArrayNode resultArray = JSONUtils.createArrayNode(); + for (int i = 0; i < totalRows; i++) { + resultArray.add(JSONUtils.parseObject("{\"id\":\"" + i + "\"}")); + } + + TaskExecutionContext ctx = createSqlTaskWithAlert(resultArray, 10, "truncate rows test", 1); + + TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); + ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); + Assertions.assertEquals(maxRows + 1, contentArray.size()); + Assertions.assertEquals("0", contentArray.get(0).get("id").asText()); + JsonNode last = contentArray.get(contentArray.size() - 1); + Assertions.assertTrue(last.has("...")); + Assertions.assertEquals((totalRows - maxRows) + " more rows omitted", last.get("...").asText()); + } + + @Test + void testPrepareTaskResultAlertTruncatesLongFieldValue() throws Exception { + int maxLen = readStaticIntConstant("ALERT_MAX_ROW_VALUE_LENGTH"); + StringBuilder longValue = new StringBuilder(); + for (int i = 0; i < maxLen + 50; i++) { + longValue.append("x"); + } + ArrayNode resultArray = JSONUtils.createArrayNode(); + resultArray.add(JSONUtils.parseObject("{\"id\":\"1\",\"payload\":\"" + longValue + "\"}")); + + TaskExecutionContext ctx = createSqlTaskWithAlert(resultArray, 10, "truncate field test", 1); + + TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); + ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); + Assertions.assertEquals(1, contentArray.size()); + String payload = contentArray.get(0).get("payload").asText(); + Assertions.assertEquals(maxLen + 3, payload.length()); + Assertions.assertTrue(payload.endsWith("...")); + } + + private int readStaticIntConstant(String name) throws Exception { + Field field = SqlTask.class.getDeclaredField(name); + field.setAccessible(true); + return field.getInt(null); + } + @Test void testResultProcessSendEmailDisabledDoesNotSetNeedAlert() throws Exception { // Build a SqlTask with sendEmail = false From 63664d947e95c7f0259465c77e886afdaa7d13b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Mon, 12 Oct 2026 16:22:16 +0800 Subject: [PATCH 41/43] Drop the alert sign unique index and keep the schema unchanged --- docs/docs/en/guide/upgrade/incompatible.md | 2 - docs/docs/zh/guide/upgrade/incompatible.md | 3 - .../apache/dolphinscheduler/dao/AlertDao.java | 48 +--- .../resources/sql/dolphinscheduler_h2.sql | 2 +- .../resources/sql/dolphinscheduler_mysql.sql | 2 +- .../sql/dolphinscheduler_postgresql.sql | 2 +- .../mysql/dolphinscheduler_ddl.sql | 22 -- .../mysql/dolphinscheduler_dml.sql | 16 -- .../postgresql/dolphinscheduler_ddl.sql | 21 -- .../postgresql/dolphinscheduler_dml.sql | 16 -- .../dao/repository/impl/AlertDaoTest.java | 241 ------------------ .../TaskSuccessLifecycleEventHandlerTest.java | 4 +- 12 files changed, 18 insertions(+), 361 deletions(-) delete mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql delete mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql delete mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql delete mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index e31dbf2c174a..579db626e402 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -59,8 +59,6 @@ This document records the incompatible updates between each version. You need to ## 3.5.0 -* Add a unique constraint `uk_alert_dedup` on `sign` to the `t_ds_alert` table. The upgrade script does not remove any existing data; if duplicate rows already exist, the index creation will fail and you need to resolve them manually. Task-result alerts are inserted by application code; the constraint rejects duplicates and the DAO layer treats the resulting `DuplicateKeyException` as a skip. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) -* Drop the legacy `idx_sign` index on `t_ds_alert`; the new `uk_alert_dedup` unique index on the same column fully covers it. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` now explicitly rejects a null `alertType` by throwing `IllegalArgumentException` with an error log, instead of throwing a raw `NullPointerException`. During a rolling upgrade, **the Alert Server must be upgraded before Master/Worker**, otherwise an old Alert Server cannot recognize newly introduced `alert_type` enum values (e.g. `TASK_RESULT`) and alert delivery will fail. See the [upgrade guide](upgrade.md#rolling-upgrade-order) for details.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * SQL task-result alert content is size-limited: at most 30 rows are kept, string field values longer than 100 characters are truncated with an ellipsis, and rows beyond 30 are represented by an omission marker. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 214c6cf321b4..fc2ca0a28564 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -58,8 +58,5 @@ ## 3.5.0 -* 为 `t_ds_alert` 表的 `sign` 新增唯一约束 `uk_alert_dedup`。升级脚本不会删除任何已有数据；如果已存在重复行，索引创建将失败，需要由用户自行处理。任务结果告警由应用代码插入，重复插入由该约束拒绝，DAO 层将产生的 `DuplicateKeyException` 视为跳过。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) -* 删除 `t_ds_alert` 表上旧的 `idx_sign` 索引，同列上新增的 `uk_alert_dedup` 唯一索引已完全覆盖其用途。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * `AlertSender#getAlertData` 对 null `alertType` 增加了显式拒绝（抛 `IllegalArgumentException` 并记录错误日志），而非直接抛 `NullPointerException`。滚动升级时**必须先升级 Alert Server，再升级 Master/Worker**，否则旧 Alert Server 无法识别新增的 `alert_type` 枚举值（如 `TASK_RESULT`），会导致告警投递失败。详见[升级文档](upgrade.md#滚动升降级顺序)。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) * SQL 任务结果告警内容有大小限制：最多保留 30 行，超过 100 字符的字符串字段值会被截断并以省略号标记，超出 30 行的部分以省略标记对象表示。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) - diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index f53c3fc461f6..f5a4ab3c48d1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -55,7 +55,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Component; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; @@ -104,23 +103,15 @@ public int addAlert(Alert alert) { String sign = generateSign(alert); alert.setSign(sign); - try { - int count = alertMapper.insert(alert); - log.info("add alert to db , alert: {}", alert); - return count; - } catch (DuplicateKeyException e) { - // The uk_alert_dedup unique constraint caught a duplicate — treat as a skip. - log.info("skip duplicate alert, sign: {}, workflowInstanceId: {}", sign, alert.getWorkflowInstanceId()); - return 0; - } + int count = alertMapper.insert(alert); + log.info("add alert to db , alert: {}", alert); + return count; } /** - * Insert a task-result alert idempotently. The uk_alert_dedup unique constraint on - * sign rejects duplicates; a DuplicateKeyException - * is caught here and treated as a skip. + * Insert a task-result alert. * - * @return insert count (1 if inserted, 0 if skipped) + * @return insert count */ public int addTaskResultAlert(Alert alert) { if (null == alert.getAlertGroupId() || NumberUtils.INTEGER_ZERO.equals(alert.getAlertGroupId())) { @@ -130,16 +121,9 @@ public int addTaskResultAlert(Alert alert) { String sign = generateSign(alert); alert.setSign(sign); - try { - int count = alertMapper.insert(alert); - log.info("add task result alert to db , alert: {}", alert); - return count; - } catch (DuplicateKeyException e) { - // The uk_alert_dedup unique constraint caught a duplicate — treat as a skip. - log.info("skip duplicate task result alert, sign: {}, workflowInstanceId: {}", sign, - alert.getWorkflowInstanceId()); - return 0; - } + int count = alertMapper.insert(alert); + log.info("add task result alert to db , alert: {}", alert); + return count; } /** @@ -166,10 +150,10 @@ public int updateAlert(AlertStatus alertStatus, String log, int id) { * @return sign's str */ private String generateSign(Alert alert) { - // Task-result alerts include the task instance ID in the sign so that two - // different tasks returning identical results are not treated as duplicates. - // For other alert types taskInstanceId is null and the sign falls back to - // content-only, preserving the original behaviour. + // Task-result alerts include the task instance ID in the sign so that different + // tasks returning identical results get different signs, and repeated success + // events of the same task yield the same sign. For other alert types + // taskInstanceId is null and the sign falls back to content-only. if (alert.getTaskInstanceId() != null) { return DigestUtils.sha1Hex(alert.getTaskInstanceId() + "|" + alert.getContent()).toLowerCase(); } @@ -287,13 +271,7 @@ private void saveTimeoutAlert(Alert alert, String content, int alertGroupId) { alert.setUpdateTime(new Date()); String sign = generateSign(alert); alert.setSign(sign); - try { - alertMapper.insert(alert); - } catch (DuplicateKeyException e) { - // The uk_alert_dedup unique constraint caught a duplicate — treat as a skip. - log.info("skip duplicate timeout alert, sign: {}, workflowInstanceId: {}", sign, - alert.getWorkflowInstanceId()); - } + alertMapper.insert(alert); } /** diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index 8f0cb148a07b..d69728be6ef3 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -285,7 +285,7 @@ CREATE TABLE t_ds_alert workflow_instance_id int(11) DEFAULT NULL, alert_type int(11) DEFAULT NULL, PRIMARY KEY (id), - UNIQUE KEY uk_alert_dedup (sign) + KEY idx_sign (sign) ); -- ---------------------------- diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index b658c2862911..361cdc3ff7fd 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -291,7 +291,7 @@ CREATE TABLE `t_ds_alert` ( `alert_type` int(11) DEFAULT NULL COMMENT 'alert_type', PRIMARY KEY (`id`), KEY `idx_status` (`alert_status`) USING BTREE, - UNIQUE KEY `uk_alert_dedup` (`sign`) + KEY `idx_sign` (`sign`) USING BTREE ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE = utf8_bin; -- ---------------------------- diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index 0476eb283c8e..387716298493 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -225,7 +225,7 @@ CREATE TABLE t_ds_alert ( comment on column t_ds_alert.sign is 'sign=sha1(content)'; create index idx_status on t_ds_alert (alert_status); -create unique index uk_alert_dedup on t_ds_alert (sign); +create index idx_sign on t_ds_alert (sign); -- -- Table structure for table t_ds_alertgroup diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql deleted file mode 100644 index c51aacc8c5b9..000000000000 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_ddl.sql +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - --- If the table already has duplicate rows, index creation will fail; resolve them manually first. -ALTER TABLE `t_ds_alert` ADD UNIQUE INDEX `uk_alert_dedup` (`sign`); --- The new unique index fully covers the legacy idx_sign index on the same column; drop the redundant one. -ALTER TABLE `t_ds_alert` DROP INDEX `idx_sign`; - diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql deleted file mode 100644 index 4a14f326b985..000000000000 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/mysql/dolphinscheduler_dml.sql +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql deleted file mode 100644 index 545270cf006e..000000000000 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_ddl.sql +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - --- If the table already has duplicate rows, index creation will fail; resolve them manually first. -CREATE UNIQUE INDEX IF NOT EXISTS uk_alert_dedup ON t_ds_alert (sign); --- The new unique index fully covers the legacy idx_sign index on the same column; drop the redundant one. -DROP INDEX IF EXISTS idx_sign; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql deleted file mode 100644 index 4a14f326b985..000000000000 --- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.5.0_schema/postgresql/dolphinscheduler_dml.sql +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java index a877ebb1d64c..b284a4cd0967 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/AlertDaoTest.java @@ -18,23 +18,11 @@ package org.apache.dolphinscheduler.dao.repository.impl; import org.apache.dolphinscheduler.common.enums.AlertStatus; -import org.apache.dolphinscheduler.common.enums.AlertType; -import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.common.enums.WarningType; -import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.Alert; -import org.apache.dolphinscheduler.dao.entity.ProjectUser; -import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.dao.entity.WorkflowInstance; -import java.util.ArrayList; import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -78,233 +66,4 @@ void testSendServerStoppedAlert() { .count(); Assertions.assertEquals(1L, count); } - - @Test - void testAddTaskResultAlertIdempotent() { - String content = "[{\"taskName\":\"sql-task-1\",\"result\":\"ok\"}]"; - int workflowInstanceId = 999999; - int taskInstanceId = 5001; - - Alert alert = new Alert(); - alert.setTitle("SQL Task Result"); - alert.setContent(content); - alert.setWarningType(WarningType.SUCCESS); - alert.setAlertGroupId(1); - alert.setAlertStatus(AlertStatus.WAIT_EXECUTION); - alert.setWorkflowInstanceId(workflowInstanceId); - alert.setTaskInstanceId(taskInstanceId); - alert.setAlertType(AlertType.TASK_RESULT); - alert.setCreateTime(new java.util.Date()); - - // First insert should succeed - int firstCount = alertDao.addTaskResultAlert(alert); - Assertions.assertEquals(1, firstCount); - - // Second insert with the same dedup key should be skipped - Alert duplicateAlert = new Alert(); - duplicateAlert.setTitle("SQL Task Result"); - duplicateAlert.setContent(content); - duplicateAlert.setWarningType(WarningType.SUCCESS); - duplicateAlert.setAlertGroupId(1); - duplicateAlert.setAlertStatus(AlertStatus.WAIT_EXECUTION); - duplicateAlert.setWorkflowInstanceId(workflowInstanceId); - duplicateAlert.setTaskInstanceId(taskInstanceId); - duplicateAlert.setAlertType(AlertType.TASK_RESULT); - duplicateAlert.setCreateTime(new java.util.Date()); - - int secondCount = alertDao.addTaskResultAlert(duplicateAlert); - Assertions.assertEquals(0, secondCount); - - long count = alertDao.listAlerts(workflowInstanceId) - .stream() - .filter(a -> a.getAlertType() == AlertType.TASK_RESULT) - .count(); - Assertions.assertEquals(1L, count); - } - - /** - * Two different tasks in the same workflow instance returning identical results - * must NOT be treated as duplicates. - */ - @Test - void testAddTaskResultAlertDifferentTaskSameContentNotDuplicate() { - String content = "[{\"result\":\"ok\"}]"; - int workflowInstanceId = 999998; - - // First task alert - Alert alert1 = new Alert(); - alert1.setTitle("SQL Task A Result"); - alert1.setContent(content); - alert1.setWarningType(WarningType.SUCCESS); - alert1.setAlertGroupId(1); - alert1.setAlertStatus(AlertStatus.WAIT_EXECUTION); - alert1.setWorkflowInstanceId(workflowInstanceId); - alert1.setTaskInstanceId(6001); - alert1.setAlertType(AlertType.TASK_RESULT); - alert1.setCreateTime(new java.util.Date()); - int firstCount = alertDao.addTaskResultAlert(alert1); - Assertions.assertEquals(1, firstCount); - - // Second task alert — same content, different task instance - Alert alert2 = new Alert(); - alert2.setTitle("SQL Task B Result"); - alert2.setContent(content); - alert2.setWarningType(WarningType.SUCCESS); - alert2.setAlertGroupId(2); - alert2.setAlertStatus(AlertStatus.WAIT_EXECUTION); - alert2.setWorkflowInstanceId(workflowInstanceId); - alert2.setTaskInstanceId(6002); - alert2.setAlertType(AlertType.TASK_RESULT); - alert2.setCreateTime(new java.util.Date()); - int secondCount = alertDao.addTaskResultAlert(alert2); - Assertions.assertEquals(1, secondCount); - - long count = alertDao.listAlerts(workflowInstanceId) - .stream() - .filter(a -> a.getAlertType() == AlertType.TASK_RESULT) - .count(); - Assertions.assertEquals(2L, count); - } - - /** - * Concurrent calls to addTaskResultAlert with the same dedup key must insert exactly one row. - * The uk_alert_dedup unique constraint rejects duplicates; the DAO layer catches - * DuplicateKeyException and returns 0. - */ - @Test - void testConcurrentAddTaskResultAlertIdempotent() throws Exception { - String content = "[{\"taskName\":\"sql-concurrent\",\"result\":\"ok\"}]"; - int workflowInstanceId = 999997; - int taskInstanceId = 7001; - int threadCount = 8; - - CountDownLatch startLatch = new CountDownLatch(1); - ExecutorService executor = Executors.newFixedThreadPool(threadCount); - - List> futures = new ArrayList<>(); - for (int i = 0; i < threadCount; i++) { - futures.add(executor.submit(() -> { - startLatch.await(); - - Alert alert = new Alert(); - alert.setTitle("SQL Task Result"); - alert.setContent(content); - alert.setWarningType(WarningType.SUCCESS); - alert.setAlertGroupId(1); - alert.setAlertStatus(AlertStatus.WAIT_EXECUTION); - alert.setWorkflowInstanceId(workflowInstanceId); - alert.setTaskInstanceId(taskInstanceId); - alert.setAlertType(AlertType.TASK_RESULT); - alert.setCreateTime(new java.util.Date()); - - return alertDao.addTaskResultAlert(alert); - })); - } - - startLatch.countDown(); - executor.shutdown(); - - int successCount = 0; - for (Future f : futures) { - int inserted = f.get(); - if (inserted > 0) { - successCount++; - } - } - - Assertions.assertEquals(1, successCount); - - long dbCount = alertDao.listAlerts(workflowInstanceId) - .stream() - .filter(a -> a.getAlertType() == AlertType.TASK_RESULT) - .count(); - Assertions.assertEquals(1L, dbCount); - } - - /** - * Duplicate calls to addAlert with the same dedup key must be skipped instead of - * throwing, so existing workflow alert paths stay safe once the uk_alert_dedup - * unique constraint applies to all inserts. - */ - @Test - void testAddAlertIdempotent() { - String content = "[{\"workflowInstanceId\":888881,\"event\":\"FAILURE\"}]"; - int workflowInstanceId = 888881; - - Alert alert = new Alert(); - alert.setTitle("Workflow Failure"); - alert.setContent(content); - alert.setWarningType(WarningType.FAILURE); - alert.setAlertGroupId(1); - alert.setAlertStatus(AlertStatus.WAIT_EXECUTION); - alert.setWorkflowInstanceId(workflowInstanceId); - alert.setAlertType(AlertType.WORKFLOW_INSTANCE_FAILURE); - alert.setCreateTime(new java.util.Date()); - - int firstCount = alertDao.addAlert(alert); - Assertions.assertEquals(1, firstCount); - - // Second insert with the same dedup key must be skipped, not throw - Alert duplicateAlert = new Alert(); - duplicateAlert.setTitle("Workflow Failure"); - duplicateAlert.setContent(content); - duplicateAlert.setWarningType(WarningType.FAILURE); - duplicateAlert.setAlertGroupId(1); - duplicateAlert.setAlertStatus(AlertStatus.WAIT_EXECUTION); - duplicateAlert.setWorkflowInstanceId(workflowInstanceId); - duplicateAlert.setAlertType(AlertType.WORKFLOW_INSTANCE_FAILURE); - duplicateAlert.setCreateTime(new java.util.Date()); - - int secondCount = alertDao.addAlert(duplicateAlert); - Assertions.assertEquals(0, secondCount); - - long count = alertDao.listAlerts(workflowInstanceId) - .stream() - .filter(a -> a.getAlertType() == AlertType.WORKFLOW_INSTANCE_FAILURE) - .count(); - Assertions.assertEquals(1L, count); - } - - /** - * Repeated timeout alerts for the same task share the same dedup key; the unique - * constraint must skip the duplicate instead of throwing in the timeout alert path. - */ - @Test - void testSendTaskTimeoutAlertIdempotent() { - int workflowInstanceId = 888882; - - WorkflowInstance workflowInstance = new WorkflowInstance(); - workflowInstance.setId(workflowInstanceId); - workflowInstance.setWarningGroupId(1); - workflowInstance.setProjectCode(1L); - workflowInstance.setWorkflowDefinitionCode(1L); - workflowInstance.setName("timeout-workflow"); - workflowInstance.setCommandType(CommandType.START_PROCESS); - workflowInstance.setState(WorkflowExecutionStatus.RUNNING_EXECUTION); - workflowInstance.setRunTimes(1); - workflowInstance.setStartTime(new java.util.Date()); - workflowInstance.setHost("127.0.0.1:5678"); - - TaskInstance taskInstance = new TaskInstance(); - taskInstance.setId(1); - taskInstance.setName("timeout-task"); - taskInstance.setTaskType("SQL"); - taskInstance.setStartTime(new java.util.Date()); - taskInstance.setHost("127.0.0.1:1234"); - - ProjectUser projectUser = new ProjectUser(); - projectUser.setProjectCode(1L); - projectUser.setProjectName("timeout-project"); - projectUser.setUserName("admin"); - - // First insert should succeed, the duplicate must be skipped without throwing - alertDao.sendTaskTimeoutAlert(workflowInstance, taskInstance, projectUser); - alertDao.sendTaskTimeoutAlert(workflowInstance, taskInstance, projectUser); - - long count = alertDao.listAlerts(workflowInstanceId) - .stream() - .filter(a -> a.getAlertType() == AlertType.TASK_TIMEOUT) - .count(); - Assertions.assertEquals(1L, count); - } } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskSuccessLifecycleEventHandlerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskSuccessLifecycleEventHandlerTest.java index 0afabd32417d..8545becfd958 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskSuccessLifecycleEventHandlerTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskSuccessLifecycleEventHandlerTest.java @@ -135,8 +135,8 @@ void shouldStillSendResultAlertForRepeatedSucceedEvent() { taskSuccessLifecycleEventHandler.handle( taskStateAction, workflowExecution, taskExecution, taskSuccessEvent); - // Repeated success events are accepted idempotently: the alert is sent again and - // deduplicated by the DB unique constraint. + // Repeated success events are accepted idempotently: the alert is sent again, + // which may insert a duplicate row; occasional duplicates are accepted. verify(workflowAlertManager).sendTaskResultAlert( any(), eq(taskInstance), any()); verify(taskExecutorClient).ackTaskExecutorLifecycleEvent( From 4f53d83e9309f3a9015d88ddda070841e59c57e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Mon, 12 Oct 2026 16:39:58 +0800 Subject: [PATCH 42/43] Bound task-result alert content by serialized UTF-8 byte budget --- docs/docs/en/guide/upgrade/incompatible.md | 2 +- docs/docs/zh/guide/upgrade/incompatible.md | 3 +- .../plugin/task/sql/SqlTask.java | 61 ++++++++-- .../plugin/task/sql/SqlTaskTest.java | 109 ++++++++++++++++++ 4 files changed, 165 insertions(+), 10 deletions(-) diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index 579db626e402..c4eb40e10b28 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -60,5 +60,5 @@ This document records the incompatible updates between each version. You need to ## 3.5.0 * `AlertSender#getAlertData` now explicitly rejects a null `alertType` by throwing `IllegalArgumentException` with an error log, instead of throwing a raw `NullPointerException`. During a rolling upgrade, **the Alert Server must be upgraded before Master/Worker**, otherwise an old Alert Server cannot recognize newly introduced `alert_type` enum values (e.g. `TASK_RESULT`) and alert delivery will fail. See the [upgrade guide](upgrade.md#rolling-upgrade-order) for details.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) -* SQL task-result alert content is size-limited: at most 30 rows are kept, string field values longer than 100 characters are truncated with an ellipsis, and rows beyond 30 are represented by an omission marker. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* SQL task-result alert content is size-limited: at most 30 rows are kept, string field values longer than 100 characters are truncated with an ellipsis, and the serialized content is bounded to 60000 UTF-8 bytes (below the 65535-byte capacity of the MySQL TEXT column); rows beyond the budget are represented by an omission marker. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index fc2ca0a28564..d3f9d1c802f1 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -59,4 +59,5 @@ ## 3.5.0 * `AlertSender#getAlertData` 对 null `alertType` 增加了显式拒绝（抛 `IllegalArgumentException` 并记录错误日志），而非直接抛 `NullPointerException`。滚动升级时**必须先升级 Alert Server，再升级 Master/Worker**，否则旧 Alert Server 无法识别新增的 `alert_type` 枚举值（如 `TASK_RESULT`），会导致告警投递失败。详见[升级文档](upgrade.md#滚动升降级顺序)。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) -* SQL 任务结果告警内容有大小限制：最多保留 30 行，超过 100 字符的字符串字段值会被截断并以省略号标记，超出 30 行的部分以省略标记对象表示。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* SQL 任务结果告警内容有大小限制：最多保留 30 行，超过 100 字符的字符串字段值会被截断并以省略号标记；序列化后内容总大小限制在 60000 UTF-8 字节以内（兼容 MySQL TEXT 列 65535 字节容量），超出预算的行以省略标记对象表示。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index b5d7e80baa26..6628b2b0a53b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -103,6 +103,12 @@ public class SqlTask extends AbstractTask { */ private static final int ALERT_MAX_ROW_VALUE_LENGTH = 100; + /** + * max serialized UTF-8 bytes of the alert content, kept below the 65535-byte capacity + * of the MySQL TEXT column that stores t_ds_alert.content + */ + private static final int ALERT_MAX_CONTENT_BYTES = 60000; + private final SQLTaskExecutionContext sqlTaskExecutionContext; private final DbType dbType; @@ -358,8 +364,11 @@ private void prepareTaskResultAlert(ArrayNode resultJSONArray) { /** * Build a size-limited alert content that stays a valid JSON array. - * Rows beyond {@link #ALERT_MAX_ROWS} are dropped and a trailing marker object is appended; - * string field values longer than {@link #ALERT_MAX_ROW_VALUE_LENGTH} are truncated with an ellipsis. + * At most {@link #ALERT_MAX_ROWS} rows are kept; string field values longer than + * {@link #ALERT_MAX_ROW_VALUE_LENGTH} are truncated with an ellipsis. The serialized + * content is bounded by {@link #ALERT_MAX_CONTENT_BYTES} UTF-8 bytes: rows are dropped + * from the tail until it fits; a row that alone exceeds the budget is replaced by the + * omission marker. */ private String limitAlertContent(ArrayNode resultJSONArray) { int totalRows = resultJSONArray.size(); @@ -368,12 +377,38 @@ private String limitAlertContent(ArrayNode resultJSONArray) { for (int i = 0; i < rowsToKeep; i++) { limited.add(limitRowValues(resultJSONArray.get(i))); } - if (totalRows > ALERT_MAX_ROWS) { - ObjectNode omitted = JSONUtils.createObjectNode(); - omitted.put("...", (totalRows - ALERT_MAX_ROWS) + " more rows omitted"); - limited.add(omitted); + + int omittedRows = totalRows - rowsToKeep; + while (true) { + if (omittedRows > 0) { + limited.add(createOmissionMarker(omittedRows)); + } + String content = JSONUtils.toJsonString(limited); + if (content.getBytes(StandardCharsets.UTF_8).length <= ALERT_MAX_CONTENT_BYTES) { + return content; + } + if (omittedRows > 0) { + limited.remove(limited.size() - 1); + } + if (limited.isEmpty()) { + // Even a lone marker exceeds the budget (pathological budget); return the minimal notice. + return JSONUtils.toJsonString(createOmissionMarkerArray(totalRows)); + } + limited.remove(limited.size() - 1); + omittedRows++; } - return JSONUtils.toJsonString(limited); + } + + private ObjectNode createOmissionMarker(int omittedRows) { + ObjectNode omitted = JSONUtils.createObjectNode(); + omitted.put("...", omittedRows + " more rows omitted"); + return omitted; + } + + private ArrayNode createOmissionMarkerArray(int totalRows) { + ArrayNode markerArray = JSONUtils.createArrayNode(); + markerArray.add(createOmissionMarker(totalRows)); + return markerArray; } private JsonNode limitRowValues(JsonNode row) { @@ -387,7 +422,7 @@ private JsonNode limitRowValues(JsonNode row) { JsonNode value = entry.getValue(); if (value.isTextual() && value.textValue().length() > ALERT_MAX_ROW_VALUE_LENGTH) { limitedRow.put(entry.getKey(), - value.textValue().substring(0, ALERT_MAX_ROW_VALUE_LENGTH) + "..."); + truncateAtCodePointBoundary(value.textValue(), ALERT_MAX_ROW_VALUE_LENGTH) + "..."); } else { limitedRow.set(entry.getKey(), value); } @@ -395,6 +430,16 @@ private JsonNode limitRowValues(JsonNode row) { return limitedRow; } + /** + * Cut a string to at most maxChars characters without splitting a surrogate pair. + */ + private String truncateAtCodePointBoundary(String text, int maxChars) { + if (text.codePointCount(0, text.length()) <= maxChars) { + return text; + } + return text.substring(0, text.offsetByCodePoints(0, maxChars)); + } + private String executeQuery(Connection connection, SqlBinds sqlBinds, String handlerType) throws Exception { try (PreparedStatement statement = prepareStatementAndBind(connection, sqlBinds)) { log.info("{} statement execute query, for sql: {}", handlerType, sqlBinds.getSql()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java index e7e7dd93a767..4786ee87abb6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java @@ -645,6 +645,115 @@ void testPrepareTaskResultAlertTruncatesLongFieldValue() throws Exception { Assertions.assertTrue(payload.endsWith("...")); } + @Test + void testPrepareTaskResultAlertBoundsTotalBytesWithWideRows() throws Exception { + int maxRows = readStaticIntConstant("ALERT_MAX_ROWS"); + int maxLen = readStaticIntConstant("ALERT_MAX_ROW_VALUE_LENGTH"); + int budget = readStaticIntConstant("ALERT_MAX_CONTENT_BYTES"); + + // Wide rows: enough columns that keeping all rows overflows the byte budget + // even though every single value stays under the per-value character limit. + int columns = budget / (maxRows * (maxLen - 1)) + 5; + int totalRows = maxRows; + ArrayNode resultArray = JSONUtils.createArrayNode(); + for (int i = 0; i < totalRows; i++) { + ObjectNode row = JSONUtils.createObjectNode(); + for (int c = 0; c < columns; c++) { + row.put("col_" + c, repeatChar('v', maxLen - 1)); + } + resultArray.add(row); + } + + TaskExecutionContext ctx = createSqlTaskWithAlert(resultArray, 10, "wide rows test", 1); + + TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); + int contentBytes = alertInfo.getContent().getBytes(StandardCharsets.UTF_8).length; + Assertions.assertTrue(contentBytes <= budget, + "serialized alert content must stay within the byte budget: " + contentBytes); + + ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); + int keptRows = contentArray.size() - 1; + Assertions.assertTrue(keptRows < totalRows, "rows must be dropped from the tail to fit the budget"); + Assertions.assertTrue(contentArray.get(0).has("col_0")); + JsonNode last = contentArray.get(contentArray.size() - 1); + Assertions.assertTrue(last.has("...")); + Assertions.assertEquals((totalRows - keptRows) + " more rows omitted", last.get("...").asText()); + } + + @Test + void testPrepareTaskResultAlertBoundsTotalBytesWithMultibyteText() throws Exception { + int maxRows = readStaticIntConstant("ALERT_MAX_ROWS"); + int maxLen = readStaticIntConstant("ALERT_MAX_ROW_VALUE_LENGTH"); + int budget = readStaticIntConstant("ALERT_MAX_CONTENT_BYTES"); + + // Each character is 3 UTF-8 bytes, so rows within the character limits still + // overflow the byte budget and rows must be dropped from the tail. + char cjk = '\u4e2d'; + int columns = budget / (maxRows * 3 * maxLen) + 5; + int totalRows = maxRows; + ArrayNode resultArray = JSONUtils.createArrayNode(); + for (int i = 0; i < totalRows; i++) { + ObjectNode row = JSONUtils.createObjectNode(); + for (int c = 0; c < columns; c++) { + row.put("col_" + c, repeatChar(cjk, maxLen)); + } + resultArray.add(row); + } + + TaskExecutionContext ctx = createSqlTaskWithAlert(resultArray, 10, "multibyte test", 1); + + TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); + int contentBytes = alertInfo.getContent().getBytes(StandardCharsets.UTF_8).length; + Assertions.assertTrue(contentBytes <= budget, + "multibyte alert content must stay within the byte budget: " + contentBytes); + + ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); + int keptRows = contentArray.size() - 1; + Assertions.assertTrue(keptRows < totalRows, "rows must be dropped from the tail to fit the budget"); + // Kept values are exactly at the character limit, so they must stay complete. + String firstValue = contentArray.get(0).get("col_0").asText(); + Assertions.assertEquals(repeatChar(cjk, maxLen), firstValue); + JsonNode last = contentArray.get(contentArray.size() - 1); + Assertions.assertTrue(last.has("...")); + Assertions.assertEquals((totalRows - keptRows) + " more rows omitted", last.get("...").asText()); + } + + @Test + void testPrepareTaskResultAlertReplacesRowExceedingByteBudgetAlone() throws Exception { + int budget = readStaticIntConstant("ALERT_MAX_CONTENT_BYTES"); + + // Nested values are not truncated per-field, so a single row can exceed the + // budget alone; it must be replaced by the omission marker. + ObjectNode row = JSONUtils.createObjectNode(); + row.put("id", "1"); + ObjectNode payload = JSONUtils.createObjectNode(); + payload.put("blob", repeatChar('n', budget * 2)); + row.set("payload", payload); + + ArrayNode resultArray = JSONUtils.createArrayNode(); + resultArray.add(row); + + TaskExecutionContext ctx = createSqlTaskWithAlert(resultArray, 10, "huge nested value test", 1); + + TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); + int contentBytes = alertInfo.getContent().getBytes(StandardCharsets.UTF_8).length; + Assertions.assertTrue(contentBytes <= budget); + + ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); + Assertions.assertEquals(1, contentArray.size()); + JsonNode only = contentArray.get(0); + Assertions.assertTrue(only.has("...")); + Assertions.assertEquals("1 more rows omitted", only.get("...").asText()); + } + + private String repeatChar(char c, int count) { + StringBuilder builder = new StringBuilder(count); + for (int i = 0; i < count; i++) { + builder.append(c); + } + return builder.toString(); + } + private int readStaticIntConstant(String name) throws Exception { Field field = SqlTask.class.getDeclaredField(name); field.setAccessible(true); From eb286243c078cd5a95f4613bf4db5a4302ad1c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E4=B9=89=E8=B6=85?= Date: Mon, 12 Oct 2026 19:44:22 +0800 Subject: [PATCH 43/43] Make SQL task-result alert limits configurable via common.properties --- deploy/kubernetes/dolphinscheduler/README.md | 3 + .../kubernetes/dolphinscheduler/values.yaml | 9 +++ docs/docs/en/architecture/configuration.md | 3 + docs/docs/en/guide/upgrade/incompatible.md | 2 +- docs/docs/zh/architecture/configuration.md | 3 + docs/docs/zh/guide/upgrade/incompatible.md | 2 +- .../common/constants/Constants.java | 17 ++++++ .../src/main/resources/common.properties | 8 +++ .../src/test/resources/common.properties | 8 ++- .../plugin/task/sql/SqlTask.java | 45 +++++++++------ .../plugin/task/sql/SqlTaskTest.java | 56 +++++++++++++++---- 11 files changed, 126 insertions(+), 30 deletions(-) diff --git a/deploy/kubernetes/dolphinscheduler/README.md b/deploy/kubernetes/dolphinscheduler/README.md index 69b5590a0e4f..99de38ec818a 100644 --- a/deploy/kubernetes/dolphinscheduler/README.md +++ b/deploy/kubernetes/dolphinscheduler/README.md @@ -203,6 +203,9 @@ The paths above match the API and Worker server classpaths. Check the configurat | conf.common."resource.storage.type" | string | `"S3"` | resource storage type: HDFS, S3, OSS, GCS, ABS, NONE | | conf.common."resource.storage.upload.base.path" | string | `"/dolphinscheduler"` | resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended | | conf.common."shell.kill.wait.timeout" | int | `10` | If the shell process is still active after this timeout value (in seconds), then will use kill -9 to kill it | +| conf.common."sql.task.alert.max.content.bytes" | int | `60000` | max serialized UTF-8 bytes of the SQL task-result alert content, kept below the 65535-byte capacity of the MySQL TEXT column | +| conf.common."sql.task.alert.max.row.value.length" | int | `100` | max chars of a single string value kept in the SQL task-result alert content, longer values are truncated with an ellipsis | +| conf.common."sql.task.alert.max.rows" | int | `30` | max rows kept in the SQL task-result alert content, rows beyond this are replaced by an omission marker | | conf.common."sudo.enable" | bool | `true` | use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions | | conf.common."support.hive.oneSession" | bool | `false` | Whether hive SQL is executed in the same session | | conf.common."task.resource.limit.state" | bool | `false` | Task resource limit state | diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml index 048da5aa7f99..8f168534e4c1 100644 --- a/deploy/kubernetes/dolphinscheduler/values.yaml +++ b/deploy/kubernetes/dolphinscheduler/values.yaml @@ -393,6 +393,15 @@ conf: # -- way to collect applicationId: log, aop appId.collect: log + # -- max rows kept in the SQL task-result alert content, rows beyond this are replaced by an omission marker + sql.task.alert.max.rows: 30 + + # -- max chars of a single string value kept in the SQL task-result alert content, longer values are truncated with an ellipsis + sql.task.alert.max.row.value.length: 100 + + # -- max serialized UTF-8 bytes of the SQL task-result alert content, kept below the 65535-byte capacity of the MySQL TEXT column + sql.task.alert.max.content.bytes: 60000 + common: ## Configmap configmap: diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md index cb37799a853e..5b5c128ba408 100644 --- a/docs/docs/en/architecture/configuration.md +++ b/docs/docs/en/architecture/configuration.md @@ -223,6 +223,9 @@ The default configuration is as follows: | sudo.enable | true | whether to enable sudo | | zeppelin.rest.url | http://localhost:8080 | the RESTful API url of zeppelin | | appId.collect | log | way to collect applicationId, if use aop, alter the configuration from log to aop, annotation of applicationId auto collection related configuration in `bin/env/dolphinscheduler_env.sh` should be removed. Note: Aop way doesn't support submitting yarn job on remote host by client mode like Beeline, and will failure if override applicationId collection-related environment configuration in dolphinscheduler_env.sh, and . | +| sql.task.alert.max.rows | 30 | max rows kept in the SQL task-result alert content, rows beyond this are replaced by an omission marker. Takes effect after restarting Worker Server | +| sql.task.alert.max.row.value.length | 100 | max chars of a single string value kept in the SQL task-result alert content, longer values are truncated with an ellipsis. Takes effect after restarting Worker Server | +| sql.task.alert.max.content.bytes | 60000 | max serialized UTF-8 bytes of the SQL task-result alert content, kept below the 65535-byte capacity of the MySQL TEXT column that stores alert content. Takes effect after restarting Worker Server | ### Api-server related configuration diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index c4eb40e10b28..2b6649dad2ed 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -60,5 +60,5 @@ This document records the incompatible updates between each version. You need to ## 3.5.0 * `AlertSender#getAlertData` now explicitly rejects a null `alertType` by throwing `IllegalArgumentException` with an error log, instead of throwing a raw `NullPointerException`. During a rolling upgrade, **the Alert Server must be upgraded before Master/Worker**, otherwise an old Alert Server cannot recognize newly introduced `alert_type` enum values (e.g. `TASK_RESULT`) and alert delivery will fail. See the [upgrade guide](upgrade.md#rolling-upgrade-order) for details.([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) -* SQL task-result alert content is size-limited: at most 30 rows are kept, string field values longer than 100 characters are truncated with an ellipsis, and the serialized content is bounded to 60000 UTF-8 bytes (below the 65535-byte capacity of the MySQL TEXT column); rows beyond the budget are represented by an omission marker. ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* SQL task-result alert content is size-limited: by default at most 30 rows are kept, string field values longer than 100 characters are truncated with an ellipsis, and the serialized content is bounded to 60000 UTF-8 bytes (below the 65535-byte capacity of the MySQL TEXT column); rows beyond the budget are represented by an omission marker. These defaults are tunable via `sql.task.alert.max.rows`, `sql.task.alert.max.row.value.length` and `sql.task.alert.max.content.bytes` in `common.properties` (worker restart required). ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md index a2b94379555b..00012cf0eadd 100644 --- a/docs/docs/zh/architecture/configuration.md +++ b/docs/docs/zh/architecture/configuration.md @@ -223,6 +223,9 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId | sudo.enable | true | 是否开启sudo | | zeppelin.rest.url | http://localhost:8080 | zeppelin RESTful API 接口地址 | | appId.collect | log | 收集applicationId方式， 如果用aop方法，将配置log替换为aop，并将`bin/env/dolphinscheduler_env.sh`自动收集applicationId相关环境变量配置的注释取消掉，注意：aop不支持远程主机提交yarn作业的方式比如Beeline客户端提交，且如果用户环境覆盖了dolphinscheduler_env.sh收集applicationId相关环境变量配置，aop方法会失效 | +| sql.task.alert.max.rows | 30 | SQL 任务结果告警内容最多保留的行数，超出的行以省略标记表示。修改后需重启 Worker Server 生效 | +| sql.task.alert.max.row.value.length | 100 | SQL 任务结果告警内容中单个字符串字段值最多保留的字符数，超出的值以省略号截断。修改后需重启 Worker Server 生效 | +| sql.task.alert.max.content.bytes | 60000 | SQL 任务结果告警内容序列化后的最大 UTF-8 字节数，小于存储告警内容的 MySQL TEXT 列 65535 字节容量。修改后需重启 Worker Server 生效 | ## Api-server相关配置 diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index d3f9d1c802f1..7e3de19628b7 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -59,5 +59,5 @@ ## 3.5.0 * `AlertSender#getAlertData` 对 null `alertType` 增加了显式拒绝（抛 `IllegalArgumentException` 并记录错误日志），而非直接抛 `NullPointerException`。滚动升级时**必须先升级 Alert Server，再升级 Master/Worker**，否则旧 Alert Server 无法识别新增的 `alert_type` 枚举值（如 `TASK_RESULT`），会导致告警投递失败。详见[升级文档](upgrade.md#滚动升降级顺序)。([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) -* SQL 任务结果告警内容有大小限制：最多保留 30 行，超过 100 字符的字符串字段值会被截断并以省略号标记；序列化后内容总大小限制在 60000 UTF-8 字节以内（兼容 MySQL TEXT 列 65535 字节容量），超出预算的行以省略标记对象表示。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) +* SQL 任务结果告警内容有大小限制：默认最多保留 30 行，超过 100 字符的字符串字段值会被截断并以省略号标记；序列化后内容总大小限制在 60000 UTF-8 字节以内（兼容 MySQL TEXT 列 65535 字节容量），超出预算的行以省略标记对象表示。以上默认值可通过 `common.properties` 中的 `sql.task.alert.max.rows`、`sql.task.alert.max.row.value.length`、`sql.task.alert.max.content.bytes` 配置调整，修改后需重启 Worker 生效。 ([#18549](https://github.com/apache/dolphinscheduler/pull/18549)) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index 81783ed28528..e5815701df9e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -59,6 +59,23 @@ public final class Constants { */ public static final String SHELL_KILL_WAIT_TIMEOUT = "shell.kill.wait.timeout"; + /** + * sql.task.alert.max.rows: this property defines the max rows kept in the task-result alert content. + */ + public static final String SQL_TASK_ALERT_MAX_ROWS = "sql.task.alert.max.rows"; + + /** + * sql.task.alert.max.row.value.length: this property defines the max chars of a single string value kept in the + * task-result alert content. + */ + public static final String SQL_TASK_ALERT_MAX_ROW_VALUE_LENGTH = "sql.task.alert.max.row.value.length"; + + /** + * sql.task.alert.max.content.bytes: this property defines the max serialized UTF-8 bytes of the task-result alert + * content. + */ + public static final String SQL_TASK_ALERT_MAX_CONTENT_BYTES = "sql.task.alert.max.content.bytes"; + public static final String SUDO_ENABLE = "sudo.enable"; public static final String RESOURCE_STORAGE_TYPE = "resource.storage.type"; diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties index 9e5adea62ea1..fa397d7ce904 100644 --- a/dolphinscheduler-common/src/main/resources/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -108,3 +108,11 @@ shell.interceptor.type=bash # Whether to enable remote logging remote.logging.enable=false + +# SQL task-result alert content limits, take effect on worker server restart: +# sql.task.alert.max.rows: max rows kept in the alert content, default: 30 +# sql.task.alert.max.row.value.length: max chars of a single string value kept in the alert content, default: 100 +# sql.task.alert.max.content.bytes: max serialized UTF-8 bytes of the alert content, default: 60000 +sql.task.alert.max.rows=30 +sql.task.alert.max.row.value.length=100 +sql.task.alert.max.content.bytes=60000 diff --git a/dolphinscheduler-common/src/test/resources/common.properties b/dolphinscheduler-common/src/test/resources/common.properties index f849b1c3e04d..99ec046b940d 100644 --- a/dolphinscheduler-common/src/test/resources/common.properties +++ b/dolphinscheduler-common/src/test/resources/common.properties @@ -201,4 +201,10 @@ remote.logging.google.cloud.storage.credential=/path/to/credential # gcs bucket name, required if you set remote.logging.target=GCS remote.logging.google.cloud.storage.bucket.name= - +# SQL task-result alert content limits, take effect on worker server restart: +# sql.task.alert.max.rows: max rows kept in the alert content, default: 30 +# sql.task.alert.max.row.value.length: max chars of a single string value kept in the alert content, default: 100 +# sql.task.alert.max.content.bytes: max serialized UTF-8 bytes of the alert content, default: 60000 +sql.task.alert.max.rows=30 +sql.task.alert.max.row.value.length=100 +sql.task.alert.max.content.bytes=60000 diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index 6628b2b0a53b..12c0cce3231a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -17,9 +17,11 @@ package org.apache.dolphinscheduler.plugin.task.sql; +import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourcePluginManager; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; @@ -94,20 +96,32 @@ public class SqlTask extends AbstractTask { private static final int QUERY_LIMIT = 10000; /** - * max rows kept in the task-result alert content; rows beyond this are replaced by an omission marker + * default max rows kept in the task-result alert content; rows beyond this are replaced by an omission marker, + * overridable via the sql.task.alert.max.rows property in common.properties */ - private static final int ALERT_MAX_ROWS = 30; + private static final int DEFAULT_ALERT_MAX_ROWS = 30; /** - * max length of a single string field value kept in the alert content; longer values are truncated with an ellipsis + * default max length of a single string field value kept in the alert content; longer values are truncated + * with an ellipsis, overridable via the sql.task.alert.max.row.value.length property in common.properties */ - private static final int ALERT_MAX_ROW_VALUE_LENGTH = 100; + private static final int DEFAULT_ALERT_MAX_ROW_VALUE_LENGTH = 100; /** - * max serialized UTF-8 bytes of the alert content, kept below the 65535-byte capacity - * of the MySQL TEXT column that stores t_ds_alert.content + * default max serialized UTF-8 bytes of the alert content, kept below the 65535-byte capacity + * of the MySQL TEXT column that stores t_ds_alert.content, overridable via the + * sql.task.alert.max.content.bytes property in common.properties */ - private static final int ALERT_MAX_CONTENT_BYTES = 60000; + private static final int DEFAULT_ALERT_MAX_CONTENT_BYTES = 60000; + + private final int alertMaxRows = + PropertyUtils.getInt(Constants.SQL_TASK_ALERT_MAX_ROWS, DEFAULT_ALERT_MAX_ROWS); + + private final int alertMaxRowValueLength = + PropertyUtils.getInt(Constants.SQL_TASK_ALERT_MAX_ROW_VALUE_LENGTH, DEFAULT_ALERT_MAX_ROW_VALUE_LENGTH); + + private final int alertMaxContentBytes = + PropertyUtils.getInt(Constants.SQL_TASK_ALERT_MAX_CONTENT_BYTES, DEFAULT_ALERT_MAX_CONTENT_BYTES); private final SQLTaskExecutionContext sqlTaskExecutionContext; @@ -364,16 +378,15 @@ private void prepareTaskResultAlert(ArrayNode resultJSONArray) { /** * Build a size-limited alert content that stays a valid JSON array. - * At most {@link #ALERT_MAX_ROWS} rows are kept; string field values longer than - * {@link #ALERT_MAX_ROW_VALUE_LENGTH} are truncated with an ellipsis. The serialized - * content is bounded by {@link #ALERT_MAX_CONTENT_BYTES} UTF-8 bytes: rows are dropped - * from the tail until it fits; a row that alone exceeds the budget is replaced by the - * omission marker. + * At most the configured max rows are kept; string field values longer than the configured + * max value length are truncated with an ellipsis. The serialized content is bounded by + * the configured max UTF-8 bytes: rows are dropped from the tail until it fits; a row that + * alone exceeds the budget is replaced by the omission marker. */ private String limitAlertContent(ArrayNode resultJSONArray) { int totalRows = resultJSONArray.size(); ArrayNode limited = JSONUtils.createArrayNode(); - int rowsToKeep = Math.min(totalRows, ALERT_MAX_ROWS); + int rowsToKeep = Math.min(totalRows, alertMaxRows); for (int i = 0; i < rowsToKeep; i++) { limited.add(limitRowValues(resultJSONArray.get(i))); } @@ -384,7 +397,7 @@ private String limitAlertContent(ArrayNode resultJSONArray) { limited.add(createOmissionMarker(omittedRows)); } String content = JSONUtils.toJsonString(limited); - if (content.getBytes(StandardCharsets.UTF_8).length <= ALERT_MAX_CONTENT_BYTES) { + if (content.getBytes(StandardCharsets.UTF_8).length <= alertMaxContentBytes) { return content; } if (omittedRows > 0) { @@ -420,9 +433,9 @@ private JsonNode limitRowValues(JsonNode row) { while (fields.hasNext()) { Map.Entry entry = fields.next(); JsonNode value = entry.getValue(); - if (value.isTextual() && value.textValue().length() > ALERT_MAX_ROW_VALUE_LENGTH) { + if (value.isTextual() && value.textValue().length() > alertMaxRowValueLength) { limitedRow.put(entry.getKey(), - truncateAtCodePointBoundary(value.textValue(), ALERT_MAX_ROW_VALUE_LENGTH) + "..."); + truncateAtCodePointBoundary(value.textValue(), alertMaxRowValueLength) + "..."); } else { limitedRow.set(entry.getKey(), value); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java index 4786ee87abb6..7dc36bc9e22c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/test/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskTest.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; @@ -53,6 +54,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -545,7 +548,7 @@ void testPrepareTaskResultAlertSetsNeedAlertAndTaskAlertInfo() { @Test void testPrepareTaskResultAlertKeepsFullContentRegardlessOfDisplayRows() throws Exception { - int totalRows = readStaticIntConstant("ALERT_MAX_ROWS") - 1; + int totalRows = readStaticIntConstant("DEFAULT_ALERT_MAX_ROWS") - 1; ArrayNode resultArray = JSONUtils.createArrayNode(); for (int i = 0; i < totalRows; i++) { resultArray.add(JSONUtils.parseObject("{\"id\":\"" + i + "\"}")); @@ -565,7 +568,7 @@ void testPrepareTaskResultAlertKeepsFullContentRegardlessOfDisplayRows() throws @Test void testPrepareTaskResultAlertKeepsFullContentWhenDisplayRowsUnset() throws Exception { - int totalRows = readStaticIntConstant("ALERT_MAX_ROWS") - 1; + int totalRows = readStaticIntConstant("DEFAULT_ALERT_MAX_ROWS") - 1; ArrayNode resultArray = JSONUtils.createArrayNode(); for (int i = 0; i < totalRows; i++) { resultArray.add(JSONUtils.parseObject("{\"id\":\"" + i + "\"}")); @@ -607,7 +610,7 @@ void testPrepareTaskResultAlertUsesDefaultTitleWhenEmpty() { @Test void testPrepareTaskResultAlertTruncatesRowsBeyondMax() throws Exception { - int maxRows = readStaticIntConstant("ALERT_MAX_ROWS"); + int maxRows = readStaticIntConstant("DEFAULT_ALERT_MAX_ROWS"); int totalRows = maxRows + 10; ArrayNode resultArray = JSONUtils.createArrayNode(); for (int i = 0; i < totalRows; i++) { @@ -627,7 +630,7 @@ void testPrepareTaskResultAlertTruncatesRowsBeyondMax() throws Exception { @Test void testPrepareTaskResultAlertTruncatesLongFieldValue() throws Exception { - int maxLen = readStaticIntConstant("ALERT_MAX_ROW_VALUE_LENGTH"); + int maxLen = readStaticIntConstant("DEFAULT_ALERT_MAX_ROW_VALUE_LENGTH"); StringBuilder longValue = new StringBuilder(); for (int i = 0; i < maxLen + 50; i++) { longValue.append("x"); @@ -647,9 +650,9 @@ void testPrepareTaskResultAlertTruncatesLongFieldValue() throws Exception { @Test void testPrepareTaskResultAlertBoundsTotalBytesWithWideRows() throws Exception { - int maxRows = readStaticIntConstant("ALERT_MAX_ROWS"); - int maxLen = readStaticIntConstant("ALERT_MAX_ROW_VALUE_LENGTH"); - int budget = readStaticIntConstant("ALERT_MAX_CONTENT_BYTES"); + int maxRows = readStaticIntConstant("DEFAULT_ALERT_MAX_ROWS"); + int maxLen = readStaticIntConstant("DEFAULT_ALERT_MAX_ROW_VALUE_LENGTH"); + int budget = readStaticIntConstant("DEFAULT_ALERT_MAX_CONTENT_BYTES"); // Wide rows: enough columns that keeping all rows overflows the byte budget // even though every single value stays under the per-value character limit. @@ -682,9 +685,9 @@ void testPrepareTaskResultAlertBoundsTotalBytesWithWideRows() throws Exception { @Test void testPrepareTaskResultAlertBoundsTotalBytesWithMultibyteText() throws Exception { - int maxRows = readStaticIntConstant("ALERT_MAX_ROWS"); - int maxLen = readStaticIntConstant("ALERT_MAX_ROW_VALUE_LENGTH"); - int budget = readStaticIntConstant("ALERT_MAX_CONTENT_BYTES"); + int maxRows = readStaticIntConstant("DEFAULT_ALERT_MAX_ROWS"); + int maxLen = readStaticIntConstant("DEFAULT_ALERT_MAX_ROW_VALUE_LENGTH"); + int budget = readStaticIntConstant("DEFAULT_ALERT_MAX_CONTENT_BYTES"); // Each character is 3 UTF-8 bytes, so rows within the character limits still // overflow the byte budget and rows must be dropped from the tail. @@ -720,7 +723,7 @@ void testPrepareTaskResultAlertBoundsTotalBytesWithMultibyteText() throws Except @Test void testPrepareTaskResultAlertReplacesRowExceedingByteBudgetAlone() throws Exception { - int budget = readStaticIntConstant("ALERT_MAX_CONTENT_BYTES"); + int budget = readStaticIntConstant("DEFAULT_ALERT_MAX_CONTENT_BYTES"); // Nested values are not truncated per-field, so a single row can exceed the // budget alone; it must be replaced by the omission marker. @@ -754,6 +757,37 @@ private String repeatChar(char c, int count) { return builder.toString(); } + @Test + void testPrepareTaskResultAlertRespectsConfiguredLimits() throws Exception { + int configuredMaxRows = 3; + int configuredMaxValueLength = 10; + int configuredMaxBytes = 60000; + int totalRows = 10; + + ArrayNode resultArray = JSONUtils.createArrayNode(); + for (int i = 0; i < totalRows; i++) { + resultArray.add(JSONUtils.parseObject("{\"id\":\"" + i + "\"}")); + } + + try (MockedStatic propertyUtils = Mockito.mockStatic(PropertyUtils.class)) { + propertyUtils.when(() -> PropertyUtils.getInt("sql.task.alert.max.rows", 30)).thenReturn(configuredMaxRows); + propertyUtils.when(() -> PropertyUtils.getInt("sql.task.alert.max.row.value.length", 100)) + .thenReturn(configuredMaxValueLength); + propertyUtils.when(() -> PropertyUtils.getInt("sql.task.alert.max.content.bytes", 60000)) + .thenReturn(configuredMaxBytes); + + TaskExecutionContext ctx = createSqlTaskWithAlert(resultArray, 10, "configured limits test", 1); + + TaskAlertInfo alertInfo = ctx.getTaskAlertInfo(); + ArrayNode contentArray = JSONUtils.parseArray(alertInfo.getContent()); + Assertions.assertEquals(configuredMaxRows + 1, contentArray.size()); + JsonNode last = contentArray.get(contentArray.size() - 1); + Assertions.assertTrue(last.has("...")); + Assertions.assertEquals((totalRows - configuredMaxRows) + " more rows omitted", + last.get("...").asText()); + } + } + private int readStaticIntConstant(String name) throws Exception { Field field = SqlTask.class.getDeclaredField(name); field.setAccessible(true);