[KYLIN-6087] Support built-in auto scheduled segment build with model-level config and scheduler

kystudio/src/components/setting/SettingBasic/SettingBasic.vue

@@ -93,6 +93,16 @@
       @submit="(scb, ecb) => handleSubmit('segment-settings', scb, ecb)"
       @cancel="(scb, ecb) => handleResetForm('segment-settings', scb, ecb)">
       <el-form ref="segment-setting-form" :model="form" :rules="rules">
+        <div class="setting-item">
+          <span class="setting-label font-medium">{{$t('autoSegmentBuild')}}</span><span class="setting-value fixed">
+            <el-switch
+              v-model="form.auto_segment_build_enabled"
+              :active-text="$t('kylinLang.common.OFF')"
+              :inactive-text="$t('kylinLang.common.ON')">
+            </el-switch>
+          </span>
+          <div class="setting-desc">{{$t('autoSegmentBuildDesc')}}</div>
+        </div>
         <div class="setting-item">
           <span class="setting-label font-medium">{{$t('segmentMerge')}}</span><span class="setting-value fixed">
             <el-switch

kystudio/src/components/setting/SettingBasic/handler.js

@@ -24,6 +24,7 @@ export const initialFormValue = {
   push_down_range_limited: true,
   auto_merge_enabled: true,
   auto_merge_time_ranges: [ 'WEEK', 'MONTH' ],
+  auto_segment_build_enabled: false,
   storage_garbage: true,
   storage_quota_size: 0,
   storage_quota_tb_size: 0,
@@ -81,6 +82,7 @@ export function _getSegmentSettings (data, project) {
     project: data.project,
     auto_merge_time_ranges: data.auto_merge_time_ranges,
     auto_merge_enabled: data.auto_merge_enabled,
+    auto_segment_build_enabled: data.auto_segment_build_enabled,
     volatile_range: {
       ...data.volatile_range,
       volatile_range_number: data.volatile_range.volatile_range_number + '',

kystudio/src/components/setting/SettingBasic/locales.js

@@ -11,6 +11,8 @@ export default {
     pushdownRange: 'Pushdown Range',
     pushdownRangeDesc: 'Turn off the pushdown range setting may cause the result difference between query on source data and indexes, especially when a source table has new data yet indexes don\'t.',
     segmentSettings: 'Segment Settings',
+    autoSegmentBuild: 'Auto Segment Build',
+    autoSegmentBuildDesc: 'When enabled, the master node periodically scans this project and submits jobs for eligible model-level auto segment build schedules.',
     datasourceSetting: 'Data Source Setting',
     gbaseConnectInfo: 'GBase Data Source Information',
     JDBCDataSource: 'JDBC Datasource',

kystudio/src/components/setting/SettingModel/SettingModel.vue

@@ -46,6 +46,15 @@
               <i class="el-icon-ksd-symbol_type" @click="removeAutoMerge(scope.row, 'retention_range')"></i>
             </common-tip>
           </div>
+          <div v-if="scope.row.auto_segment_build && scope.row.auto_segment_build.enabled">
+            <span class="model-setting-item" @click="editAutoSegmentBuildItem(scope.row)">
+              {{$t('autoSegmentBuild')}}<span>{{formatAutoSegmentBuild(scope.row.auto_segment_build)}}</span>
+            </span><common-tip :content="$t('kylinLang.common.edit')">
+              <i class="el-icon-ksd-table_edit ksd-mr-5 ksd-ml-10" @click="editAutoSegmentBuildItem(scope.row)"></i>
+            </common-tip><common-tip :content="$t('kylinLang.common.delete')">
+              <i class="el-icon-ksd-symbol_type" @click="removeAutoMerge(scope.row, 'auto_segment_build')"></i>
+            </common-tip>
+          </div>
           <div v-if="Object.keys(scope.row.override_props).length">
             <div v-for="(propValue, key) in scope.row.override_props" :key="key">
               <template v-if="key.includes('kylin.engine.spark-conf') && defaultConfigs.includes(key.replace(/^kylin.engine.spark-conf./, ''))">
@@ -141,6 +150,44 @@
             </el-option>
           </el-select>
         </el-form-item>
+        <template v-if="step=='stepTwo'&&modelSettingForm.settingItem==='Auto Segment Build'">
+          <el-form-item :label="$t('autoSegmentBuildTriggerTime')">
+            <el-time-picker
+              style="width: 180px;"
+              size="small"
+              format="HH:mm:ss"
+              value-format="HH:mm:ss"
+              v-model="modelSettingForm.autoSegmentBuild.trigger_time">
+            </el-time-picker>
+          </el-form-item>
+          <el-form-item :label="$t('autoSegmentBuildLogicalOffset')">
+            <el-input v-model="modelSettingForm.autoSegmentBuild.logical_date_offset_days" v-number="modelSettingForm.autoSegmentBuild.logical_date_offset_days" class="retention-input"></el-input>
+          </el-form-item>
+          <el-form-item :label="$t('autoSegmentBuildRangeStart')">
+            <el-time-picker
+              style="width: 180px;"
+              size="small"
+              format="HH:mm:ss"
+              value-format="HH:mm:ss"
+              v-model="modelSettingForm.autoSegmentBuild.data_range_start_time">
+            </el-time-picker>
+          </el-form-item>
+          <el-form-item :label="$t('autoSegmentBuildRangeEnd')">
+            <el-time-picker
+              v-if="!isAutoSegmentBuildEndOfDay"
+              style="width: 180px;"
+              size="small"
+              format="HH:mm:ss"
+              value-format="HH:mm:ss"
+              v-model="modelSettingForm.autoSegmentBuild.data_range_end_time">
+            </el-time-picker>
+            <el-checkbox
+              :class="{'ksd-ml-10': !isAutoSegmentBuildEndOfDay}"
+              v-model="isAutoSegmentBuildEndOfDay">
+              {{$t('autoSegmentBuildEndOfDay')}}
+            </el-checkbox>
+          </el-form-item>
+        </template>
         <el-form-item :label="settingMap[modelSettingForm.settingItem]" v-if="step=='stepTwo'&&modelSettingForm.settingItem.indexOf('spark.')!==-1">
           <el-input v-model="modelSettingForm[modelSettingForm.settingItem]" v-number="modelSettingForm[modelSettingForm.settingItem]" :placeholder="$t('kylinLang.common.pleaseInput')" class="retention-input"></el-input><span
           class="ksd-ml-5" v-if="modelSettingForm.settingItem==='spark.executor.memory'">G</span>
@@ -196,12 +243,15 @@ import { handleSuccess, transToGmtTime, kylinConfirm } from '../../../util/busin
 import { handleSuccessAsync, handleError, objectClone, ArrayFlat } from '../../../util/index'
 import { retentionTypes } from '../handler'
 
+const END_OF_DAY = '24:00:00'
+
 const initialSettingForm = JSON.stringify({
   name: '',
   settingItem: '',
   autoMerge: [],
   volatileRange: {volatile_range_number: 0, volatile_range_type: '', volatile_range_enabled: true},
   retentionThreshold: {retention_range_number: 0, retention_range_type: '', retention_range_enabled: true},
+  autoSegmentBuild: {enabled: true, trigger_time: '01:00:00', logical_date_offset_days: 1, data_range_start_time: '00:00:00', data_range_end_time: END_OF_DAY},
   'spark.executor.cores': null,
   'spark.executor.instances': null,
   'spark.executor.memory': null,
@@ -255,6 +305,7 @@ export default class SettingStorage extends Vue {
     'Auto-merge': 'Auto-merge',
     'Volatile Range': 'Volatile Range',
     'Retention Threshold': 'Retention Threshold',
+    'Auto Segment Build': 'Auto Segment Build',
     'spark.executor.cores': 'kylin.engine.spark-conf.spark.executor.cores',
     'spark.executor.instances': 'kylin.engine.spark-conf.spark.executor.instances',
     'spark.executor.memory': 'kylin.engine.spark-conf.spark.executor.memory',
@@ -275,6 +326,7 @@ export default class SettingStorage extends Vue {
       'Auto-merge',
       'Volatile Range',
       'Retention Threshold',
+      'Auto Segment Build',
       'spark.executor.cores',
       'spark.executor.instances',
       'spark.executor.memory',
@@ -295,6 +347,12 @@ export default class SettingStorage extends Vue {
     // return largestRange || ''
     return 'DAY'
   }
+  get isAutoSegmentBuildEndOfDay () {
+    return this.modelSettingForm.autoSegmentBuild.data_range_end_time === END_OF_DAY
+  }
+  set isAutoSegmentBuildEndOfDay (isEndOfDay) {
+    this.modelSettingForm.autoSegmentBuild.data_range_end_time = isEndOfDay ? END_OF_DAY : null
+  }
   validateSettingItem (rule, value, callback) {
     const autoMergeRanges = this.activeRow && this.activeRow.auto_merge_time_ranges || []
     if (this.step === 'stepOne' && value === 'Retention Threshold' && !autoMergeRanges.length) {
@@ -308,6 +366,7 @@ export default class SettingStorage extends Vue {
       'Auto-merge': this.$t('autoMergeTip'),
       'Volatile Range': this.$t('volatileTip'),
       'Retention Threshold': this.$t('retentionThresholdDesc'),
+      'Auto Segment Build': this.$t('autoSegmentBuildTip'),
       'kylin.engine.spark-conf.spark.executor.cores': this.$t('sparkCores'),
       'kylin.engine.spark-conf.spark.executor.instances': this.$t('sparkInstances'),
       'kylin.engine.spark-conf.spark.executor.memory': this.$t('sparkMemory'),
@@ -346,6 +405,8 @@ export default class SettingStorage extends Vue {
       return true
     } else if (this.modelSettingForm.settingItem === 'Retention Threshold' && !(this.modelSettingForm.retentionThreshold.retention_range_number >= 0 && this.modelSettingForm.retentionThreshold.retention_range_number !== '' && this.modelSettingForm.retentionThreshold.retention_range_type)) {
       return true
+    } else if (this.modelSettingForm.settingItem === 'Auto Segment Build' && !this.isValidAutoSegmentBuild()) {
+      return true
     } else if (this.modelSettingForm.settingItem.indexOf('spark.') !== -1 && !this.modelSettingForm[this.modelSettingForm.settingItem]) {
       return true
     } else if (this.modelSettingForm.settingItem === 'is-base-cuboid-always-valid' && this.modelSettingForm[this.modelSettingForm.settingItem] === '') {
@@ -439,6 +500,15 @@ export default class SettingStorage extends Vue {
     this.isEdit = true
     this.editModelSetting = true
   }
+  editAutoSegmentBuildItem (row) {
+    this.modelSettingForm.name = row.alias
+    this.modelSettingForm.settingItem = 'Auto Segment Build'
+    this.modelSettingForm.autoSegmentBuild = JSON.parse(JSON.stringify(row.auto_segment_build))
+    this.activeRow = row
+    this.step = 'stepTwo'
+    this.isEdit = true
+    this.editModelSetting = true
+  }
   editSparkItem (row, sparkItemKey) {
     this.modelSettingForm.name = row.alias
     this.modelSettingForm.settingItem = sparkItemKey.substring(24)
@@ -489,6 +559,13 @@ export default class SettingStorage extends Vue {
     if (this.modelSettingForm.settingItem === 'Retention Threshold') {
       this.activeRow.retention_range = this.modelSettingForm.retentionThreshold
     }
+    if (this.modelSettingForm.settingItem === 'Auto Segment Build') {
+      this.activeRow.auto_segment_build = {
+        ...this.modelSettingForm.autoSegmentBuild,
+        enabled: true,
+        logical_date_offset_days: Number(this.modelSettingForm.autoSegmentBuild.logical_date_offset_days)
+      }
+    }
     if (this.modelSettingForm.settingItem.indexOf('spark.') !== -1) {
       this.activeRow.override_props['kylin.engine.spark-conf.' + this.modelSettingForm.settingItem] = this.modelSettingForm[this.modelSettingForm.settingItem]
     }
@@ -544,6 +621,22 @@ export default class SettingStorage extends Vue {
   removeCustomSetting (index) {
     this.modelSettingForm[this.modelSettingForm.settingItem].splice(index, 1)
   }
+  isValidAutoSegmentBuild () {
+    const config = this.modelSettingForm.autoSegmentBuild
+    if (!config || !config.trigger_time || !config.data_range_start_time || !config.data_range_end_time) return false
+    if (!(+config.logical_date_offset_days >= 1)) return false
+    if (config.data_range_end_time === END_OF_DAY) return true
+    return config.data_range_start_time < config.data_range_end_time
+  }
+  formatAutoSegmentBuild (config) {
+    if (!config) return ''
+    return this.$t('autoSegmentBuildSummary', {
+      trigger: config.trigger_time,
+      offset: config.logical_date_offset_days,
+      start: config.data_range_start_time,
+      end: config.data_range_end_time
+    })
+  }
   created () {
     this.getConfigList()
   }

kystudio/src/components/setting/SettingModel/locales.js

@@ -6,6 +6,7 @@ export default {
     segmentMerge: 'Segment Merge:',
     volatileRange: 'Volatile Range:',
     retention: 'Retention Threshold:',
+    autoSegmentBuild: 'Auto Segment Build:',
     newSetting: 'Add Parameter Configuration',
     editSetting: 'Edit Parameter Configuration',
     modelName: 'Model Name',
@@ -34,13 +35,15 @@ export default {
     'isDel_kylin.engine.spark-conf.spark.executor.memory': 'Are you sure delete spark.executor.memory item?',
     'isDel_kylin.engine.spark-conf.spark.sql.shuffle.partitions': 'Are you sure delete spark.sql.shuffle.partitions item?',
     'isDel_kylin.cube.aggrgroup.is-base-cuboid-always-valid': 'Are you sure delete is-base-cuboid-always-valid item?',
+    isDel_auto_segment_build: 'Are you sure delete auto segment build configuration?',
     autoMergeTip: 'The system could auto-merge segment fragments over different merging threshold. Auto-merge will optimize storage to enhance query performance.',
     volatileTip: '"Auto-Merge" will not merge the latest segments defined in "Volatile Range". The default value is 0.',
     retentionThresholdDesc: 'The segments within the retention threshold would be kept. The rest would be removed automatically.',
     pleaseSetAutoMerge: 'Please add \'Auto Merge\' configuration first.',
     'Auto-merge': 'Auto Merge',
     'Volatile Range': 'Volatile Range',
     'Retention Threshold': 'Retention Threshold',
+    'Auto Segment Build': 'Auto Segment Build',
     'kylin.engine.spark-conf.spark.executor.cores': 'kylin.engine.spark-conf.spark.executor.cores',
     'kylin.engine.spark-conf.spark.executor.instances': 'kylin.engine.spark-conf.spark.executor.instances',
     'kylin.engine.spark-conf.spark.executor.memory': 'kylin.engine.spark-conf.spark.executor.memory',
@@ -55,6 +58,13 @@ export default {
     customOptions: 'Besides the defined configurations, you can also add some advanced settings.<br/><i class="el-icon-ksd-alert"></i>Note: It\'s highly recommended to use this feature with the support of Kylin 5 Team.',
     customSettingKeyPlaceholder: 'Configuration Name',
     customSettingValuePlaceholder: 'Value',
-    delCustomConfigTip: 'Are you sure you want to delete custom setting item {name}？'
+    delCustomConfigTip: 'Are you sure you want to delete custom setting item {name}？',
+    autoSegmentBuildTriggerTime: 'Trigger Time',
+    autoSegmentBuildLogicalOffset: 'Logical Date Offset (Day)',
+    autoSegmentBuildRangeStart: 'Range Start Time',
+    autoSegmentBuildRangeEnd: 'Range End Time',
+    autoSegmentBuildEndOfDay: 'End of logical day (next day 00:00)',
+    autoSegmentBuildTip: 'Build segment automatically every day with logical day offset and configured time range.',
+    autoSegmentBuildSummary: 'Daily {trigger}, D-{offset} {start}~{end}'
   }
 }

src/common-service/src/main/java/org/apache/kylin/rest/request/SegmentConfigRequest.java

@@ -40,4 +40,6 @@ public class SegmentConfigRequest {
     private RetentionRange retentionRange = new RetentionRange();
     @JsonProperty("create_empty_segment_enabled")
     private Boolean createEmptySegmentEnabled = false;
+    @JsonProperty("auto_segment_build_enabled")
+    private Boolean autoSegmentBuildEnabled;
 }

src/common-service/src/main/java/org/apache/kylin/rest/response/ProjectConfigResponse.java

@@ -82,6 +82,8 @@ public class ProjectConfigResponse {
     private VolatileRange volatileRange;
     @JsonProperty("create_empty_segment_enabled")
     private boolean createEmptySegmentEnabled = false;
+    @JsonProperty("auto_segment_build_enabled")
+    private boolean autoSegmentBuildEnabled = false;
 
     @JsonProperty("retention_range")
     private RetentionRange retentionRange;

src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java

@@ -602,6 +602,8 @@ public ProjectConfigResponse getProjectConfig0(String project) {
         response.setVolatileRange(projectInstance.getSegmentConfig().getVolatileRange());
         response.setRetentionRange(projectInstance.getSegmentConfig().getRetentionRange());
         response.setCreateEmptySegmentEnabled(projectInstance.getSegmentConfig().getCreateEmptySegmentEnabled());
+        response.setAutoSegmentBuildEnabled(
+                Boolean.TRUE.equals(projectInstance.getSegmentConfig().getAutoSegmentBuildEnabled()));
 
         response.setFavoriteQueryThreshold(config.getFavoriteQueryAccelerateThreshold());
         response.setFavoriteQueryTipsEnabled(config.getFavoriteQueryAccelerateTipsEnabled());
@@ -937,6 +939,10 @@ public void updateSegmentConfig(String project, SegmentConfigRequest segmentConf
             copyForWrite.getSegmentConfig().setRetentionRange(segmentConfigRequest.getRetentionRange());
             copyForWrite.getSegmentConfig()
                     .setCreateEmptySegmentEnabled(segmentConfigRequest.getCreateEmptySegmentEnabled());
+            if (segmentConfigRequest.getAutoSegmentBuildEnabled() != null) {
+                copyForWrite.getSegmentConfig()
+                        .setAutoSegmentBuildEnabled(segmentConfigRequest.getAutoSegmentBuildEnabled());
+            }
         });
     }
 
@@ -1199,6 +1205,8 @@ private void resetSegmentConfig(String project) {
                     .setAutoMergeTimeRanges(projectInstance.getSegmentConfig().getAutoMergeTimeRanges());
             copyForWrite.getSegmentConfig().setVolatileRange(projectInstance.getSegmentConfig().getVolatileRange());
             copyForWrite.getSegmentConfig().setRetentionRange(projectInstance.getSegmentConfig().getRetentionRange());
+            copyForWrite.getSegmentConfig()
+                    .setAutoSegmentBuildEnabled(projectInstance.getSegmentConfig().getAutoSegmentBuildEnabled());
         });
     }
 

src/core-common/src/main/resources/kylin-defaults0.properties

@@ -449,6 +449,8 @@ kylin.web.session.jdbc-encode-enabled=false
 kylin.security.user-password-encoder=org.apache.kylin.rest.security.CachedBCryptPasswordEncoder
 
 # model
+# System-level internal scan cadence in milliseconds; model trigger times are read from metadata on every scan.
+kylin.model.auto-segment-build.dispatcher-interval-ms=30000
 kylin.model.recommendation-page-size=500
 kylin.model.dimension-measure-name.max-length=300
 

src/core-metadata/src/main/java/org/apache/kylin/metadata/model/AutoSegmentBuildConfig.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.model;
+
+import java.io.Serializable;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.Getter;
+import lombok.Setter;
+
+@Getter
+@Setter
+public class AutoSegmentBuildConfig implements Serializable {
+
+    // LocalTime cannot represent 24:00:00; this value denotes midnight at the end of the logical date.
+    public static final String END_OF_DAY = "24:00:00";
+
+    @JsonProperty("enabled")
+    private boolean enabled = false;
+
+    @JsonProperty("trigger_time")
+    private String triggerTime;
+
+    @JsonProperty("logical_date_offset_days")
+    private Integer logicalDateOffsetDays;
+
+    @JsonProperty("data_range_start_time")
+    private String dataRangeStartTime;
+
+    @JsonProperty("data_range_end_time")
+    private String dataRangeEndTime;
+}

src/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentConfig.java

@@ -21,6 +21,7 @@
 import java.io.Serializable;
 import java.util.List;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 import lombok.AllArgsConstructor;
@@ -49,6 +50,13 @@ public class SegmentConfig implements Serializable {
     @JsonProperty("create_empty_segment_enabled")
     private Boolean createEmptySegmentEnabled = false;
 
+    @JsonProperty("auto_segment_build")
+    private AutoSegmentBuildConfig autoSegmentBuild = new AutoSegmentBuildConfig();
+
+    @JsonProperty("auto_segment_build_enabled")
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    private Boolean autoSegmentBuildEnabled;
+
     public boolean canSkipAutoMerge() {
         return !autoMergeEnabled;
     }