diff --git a/packaging/src/docker/entrypoint.sh b/packaging/src/docker/entrypoint.sh index a043ef095026..baa540efaf28 100644 --- a/packaging/src/docker/entrypoint.sh +++ b/packaging/src/docker/entrypoint.sh @@ -177,7 +177,8 @@ if [ -d "${HIVE_CUSTOM_CONF_DIR:-}" ]; then export TEZ_CONF_DIR=$HIVE_CONF_DIR fi -export HADOOP_CLIENT_OPTS="${HADOOP_CLIENT_OPTS:-} -Xmx1G ${SERVICE_OPTS:-}" +# default heap first so user-provided -Xmx wins (last one applies) +export HADOOP_CLIENT_OPTS="-Xmx1G ${HADOOP_CLIENT_OPTS:-} ${SERVICE_OPTS:-}" if [[ "${SKIP_SCHEMA_INIT}" == "false" && ( "${SERVICE_NAME}" == "hiveserver2" || "${SERVICE_NAME}" == "metastore" ) ]]; then # handles schema initialization initialize_hive diff --git a/packaging/src/kubernetes/README.md b/packaging/src/kubernetes/README.md index 1135413f612f..c1cd9fe87b6e 100644 --- a/packaging/src/kubernetes/README.md +++ b/packaging/src/kubernetes/README.md @@ -111,12 +111,20 @@ and an equivalent values file. Each example below shows both the `helm install` CLI command and the equivalent `values.yaml` file. Use whichever approach you prefer. +> **Give each cluster its own release name.** `-hive-operator` is a ClusterRole, so +> a second cluster under the same release name fails even in another namespace. Naming the +> release after its namespace is enough, since namespaces are already unique: +> +> ```bash +> export RELEASE=hive-dev +> ``` + ### Ozone (Full-HA, default behavior) **CLI:** ```bash -helm install hive ./helm/hive-operator \ +helm install "$RELEASE" ./helm/hive-operator \ --set cluster.database.type=postgres \ --set cluster.database.url="jdbc:postgresql://postgres-postgresql:5432/metastore" \ --set cluster.database.driver="org.postgresql.Driver" \ @@ -169,7 +177,7 @@ cluster: ``` ```bash -helm install hive ./helm/hive-operator -f values.yaml +helm install "$RELEASE" ./helm/hive-operator -f values.yaml ``` --- @@ -188,7 +196,7 @@ kubectl create secret generic aws-s3-creds \ Then install the operator and HiveCluster with the appropriate storage config: ```bash -helm install hive ./helm/hive-operator \ +helm install "$RELEASE" ./helm/hive-operator \ --set cluster.database.type=postgres \ --set cluster.database.url="jdbc:postgresql://postgres-postgresql:5432/metastore" \ --set cluster.database.driver="org.postgresql.Driver" \ @@ -245,7 +253,7 @@ cluster: ``` ```bash -helm install hive ./helm/hive-operator -f values.yaml +helm install "$RELEASE" ./helm/hive-operator -f values.yaml ``` --- @@ -261,7 +269,7 @@ kubectl create secret generic gcs-creds --from-file=key.json=.json **CLI:** ```bash -helm install hive ./helm/hive-operator \ +helm install "$RELEASE" ./helm/hive-operator \ --set cluster.database.type=postgres \ --set cluster.database.url="jdbc:postgresql://postgres-postgresql:5432/metastore" \ --set cluster.database.driver="org.postgresql.Driver" \ @@ -321,7 +329,7 @@ cluster: ``` ```bash -helm install hive ./helm/hive-operator -f values.yaml +helm install "$RELEASE" ./helm/hive-operator -f values.yaml ``` --- @@ -333,7 +341,7 @@ helm install hive ./helm/hive-operator -f values.yaml **CLI:** ```bash -helm install hive ./helm/hive-operator \ +helm install "$RELEASE" ./helm/hive-operator \ --set cluster.database.type=postgres \ --set cluster.database.url="jdbc:postgresql://postgres-postgresql:5432/metastore" \ --set cluster.database.driver="org.postgresql.Driver" \ @@ -398,7 +406,7 @@ cluster: ``` ```bash -helm install hive ./helm/hive-operator -f values.yaml +helm install "$RELEASE" ./helm/hive-operator -f values.yaml ``` --- @@ -408,7 +416,7 @@ helm install hive ./helm/hive-operator -f values.yaml **CLI:** ```bash -helm install hive ./helm/hive-operator \ +helm install "$RELEASE" ./helm/hive-operator \ --set cluster.zookeeper.quorum="zookeeper:2181" \ --set cluster.metastore.enabled=false \ --set cluster.metastore.externalUri="thrift://my-external-metastore:9083" \ @@ -448,7 +456,7 @@ cluster: ``` ```bash -helm install hive ./helm/hive-operator -f values.yaml +helm install "$RELEASE" ./helm/hive-operator -f values.yaml ``` --- @@ -492,7 +500,7 @@ cluster: ``` ```bash -helm install hive ./helm/hive-operator -f values.yaml +helm install "$RELEASE" ./helm/hive-operator -f values.yaml ``` --- @@ -609,7 +617,7 @@ cluster: ``` ```bash -helm install hive ./helm/hive-operator -f values-multi-tenant.yaml +helm install "$RELEASE" ./helm/hive-operator -f values-multi-tenant.yaml ``` ### Resulting Kubernetes Resources @@ -693,7 +701,7 @@ This means scaling up `production` never affects `analytics` or `dev` replicas. To add a new cluster, append to `llapClusters[]` and run `helm upgrade`: ```bash -helm upgrade hive ./helm/hive-operator -f values-multi-tenant.yaml +helm upgrade "$RELEASE" ./helm/hive-operator -f values-multi-tenant.yaml ``` To remove a cluster, delete the entry from `llapClusters[]` and upgrade. The operator @@ -1106,7 +1114,7 @@ Each component has sensible per-component defaults (see [Configuration Reference Only `enabled=true` is needed to turn on autoscaling: ```bash -helm install hive ./helm/hive-operator \ +helm install "$RELEASE" ./helm/hive-operator \ --set cluster.database.type=postgres \ --set cluster.database.url="jdbc:postgresql://postgres-postgresql:5432/metastore" \ --set cluster.database.driver="org.postgresql.Driver" \ @@ -1208,7 +1216,7 @@ cluster: ``` ```bash -helm install hive ./helm/hive-operator -f values-autoscaling.yaml +helm install "$RELEASE" ./helm/hive-operator -f values-autoscaling.yaml ``` When autoscaling is enabled, the operator automatically: @@ -1468,7 +1476,7 @@ controls shared settings (enabled flag, scratch PVC). Per-LLAP TezAM settings ### Upgrade (values only, no CRD changes) ```bash -helm upgrade hive ./helm/hive-operator -f my-values.yaml +helm upgrade "$RELEASE" ./helm/hive-operator -f my-values.yaml ``` ### Upgrade (with CRD schema changes) @@ -1479,14 +1487,14 @@ re-apply the CRD manually: ```bash kubectl apply -f helm/hive-operator/crds/hiveclusters.hive.apache.org-v1.yml -helm upgrade hive ./helm/hive-operator -f my-values.yaml +helm upgrade "$RELEASE" ./helm/hive-operator -f my-values.yaml ``` ### Full Uninstall and Reinstall (clean slate) ```bash # Uninstall (removes operator + HiveCluster CR + all managed pods) -helm uninstall hive +helm uninstall "$RELEASE" # IMPORTANT: Always delete the CRD before reinstalling to ensure # the updated schema is applied. Helm only creates CRDs on install, @@ -1494,14 +1502,14 @@ helm uninstall hive kubectl delete crd hiveclusters.hive.apache.org # Reinstall -helm install hive ./helm/hive-operator -f my-values.yaml +helm install "$RELEASE" ./helm/hive-operator -f my-values.yaml ``` ### Remove Everything (including dependencies) ```bash kubectl delete hivecluster --all -A --wait=false --ignore-not-found -helm uninstall hive --ignore-not-found +helm uninstall "$RELEASE" --ignore-not-found kubectl delete crd hiveclusters.hive.apache.org --wait=false --ignore-not-found helm uninstall ozone --ignore-not-found helm uninstall postgres --ignore-not-found diff --git a/packaging/src/kubernetes/config/operator/deployment.yaml b/packaging/src/kubernetes/config/operator/deployment.yaml index b7d9625daacf..106cfdee2c58 100644 --- a/packaging/src/kubernetes/config/operator/deployment.yaml +++ b/packaging/src/kubernetes/config/operator/deployment.yaml @@ -35,11 +35,6 @@ spec: - name: hive-operator image: apache/hive:operator-${HIVE_VERSION} imagePullPolicy: IfNotPresent - env: - - name: OPERATOR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace resources: requests: cpu: 200m diff --git a/packaging/src/kubernetes/helm/hive-operator/crds/hiveclusters.hive.apache.org-v1.yml b/packaging/src/kubernetes/helm/hive-operator/crds/hiveclusters.hive.apache.org-v1.yml index 9bdb3863c526..516e4309d281 100644 --- a/packaging/src/kubernetes/helm/hive-operator/crds/hiveclusters.hive.apache.org-v1.yml +++ b/packaging/src/kubernetes/helm/hive-operator/crds/hiveclusters.hive.apache.org-v1.yml @@ -79,6 +79,11 @@ spec: hiveServer2: description: HiveServer2 component configuration properties: + affinity: + description: Affinity override; replaces the default spread anti-affinity + when set + type: object + x-kubernetes-preserve-unknown-fields: true autoscaling: description: "Autoscaling configuration (operator-driven, no external\ \ dependencies)" @@ -154,6 +159,13 @@ spec: type: string description: Additional configuration overrides as key-value pairs type: object + envVars: + description: "Component-scoped env vars, appended after the cluster-wide\ + \ envVars" + items: + type: object + type: array + x-kubernetes-preserve-unknown-fields: true externalJars: description: 'List of URIs to external JARs to download and add to HS2 classpath ' @@ -224,28 +236,24 @@ spec: description: Number of replicas type: integer resources: + default: + requests: + cpu: 500m + memory: 1Gi description: Resource requirements for pods - properties: - limitsCpu: - description: "CPU limit (e.g. 2, 1000m)" - type: string - limitsMemory: - description: "Memory limit (e.g. 2Gi, 1024Mi)" - type: string - requestsCpu: - default: 500m - description: "CPU request (e.g. 500m, 1)" - type: string - requestsMemory: - default: 1Gi - description: "Memory request (e.g. 1Gi, 512Mi)" - type: string type: object + x-kubernetes-preserve-unknown-fields: true serviceType: default: ClusterIP description: "Kubernetes Service type: ClusterIP, LoadBalancer,\ \ or NodePort" type: string + tolerations: + description: Tolerations for scheduling onto tainted nodes + items: + type: object + type: array + x-kubernetes-preserve-unknown-fields: true type: object x-kubernetes-preserve-unknown-fields: true image: @@ -267,6 +275,11 @@ spec: \ in their session." items: properties: + affinity: + description: Affinity override; replaces the default spread + anti-affinity when set + type: object + x-kubernetes-preserve-unknown-fields: true autoscaling: description: "Autoscaling configuration (operator-driven, no\ \ external dependencies)" @@ -348,6 +361,13 @@ spec: default: true description: Whether LLAP is enabled type: boolean + envVars: + description: "Component-scoped env vars, appended after the\ + \ cluster-wide envVars" + items: + type: object + type: array + x-kubernetes-preserve-unknown-fields: true executors: default: 1 description: Number of LLAP executors per daemon @@ -402,23 +422,13 @@ spec: description: Number of replicas type: integer resources: + default: + requests: + cpu: 500m + memory: 1Gi description: Resource requirements for pods - properties: - limitsCpu: - description: "CPU limit (e.g. 2, 1000m)" - type: string - limitsMemory: - description: "Memory limit (e.g. 2Gi, 1024Mi)" - type: string - requestsCpu: - default: 500m - description: "CPU request (e.g. 500m, 1)" - type: string - requestsMemory: - default: 1Gi - description: "Memory request (e.g. 1Gi, 512Mi)" - type: string type: object + x-kubernetes-preserve-unknown-fields: true serviceHosts: description: "LLAP service hosts identifier for ZooKeeper registration.\ \ Defaults to @{name} (e.g. @llap0)." @@ -427,6 +437,12 @@ spec: description: Per-LLAP TezAM configuration. Each LLAP cluster gets its own TezAM with independent replica count and autoscaling. properties: + affinity: + description: "Affinity for this LLAP cluster's TezAM, overriding\ + \ spec.tezAm.affinity. Set it with more than one LLAP\ + \ cluster." + type: object + x-kubernetes-preserve-unknown-fields: true autoscaling: description: Autoscaling configuration for this LLAP cluster's TezAM @@ -504,7 +520,21 @@ spec: description: Max number of TezAM replicas for this LLAP cluster type: integer + tolerations: + description: "Tolerations for this LLAP cluster's TezAM,\ + \ overriding spec.tezAm.tolerations" + items: + type: object + type: array + x-kubernetes-preserve-unknown-fields: true type: object + x-kubernetes-preserve-unknown-fields: true + tolerations: + description: Tolerations for scheduling onto tainted nodes + items: + type: object + type: array + x-kubernetes-preserve-unknown-fields: true required: - name type: object @@ -513,6 +543,11 @@ spec: metastore: description: Metastore component configuration properties: + affinity: + description: Affinity override; replaces the default spread anti-affinity + when set + type: object + x-kubernetes-preserve-unknown-fields: true autoscaling: description: "Autoscaling configuration (operator-driven, no external\ \ dependencies)" @@ -625,6 +660,13 @@ spec: default: true description: Whether the operator should deploy and manage a Metastore type: boolean + envVars: + description: "Component-scoped env vars, appended after the cluster-wide\ + \ envVars" + items: + type: object + type: array + x-kubernetes-preserve-unknown-fields: true externalUri: description: Thrift URI of the external Metastore (if enabled is false) @@ -693,23 +735,19 @@ spec: description: Number of replicas type: integer resources: + default: + requests: + cpu: 500m + memory: 1Gi description: Resource requirements for pods - properties: - limitsCpu: - description: "CPU limit (e.g. 2, 1000m)" - type: string - limitsMemory: - description: "Memory limit (e.g. 2Gi, 1024Mi)" - type: string - requestsCpu: - default: 500m - description: "CPU request (e.g. 500m, 1)" - type: string - requestsMemory: - default: 1Gi - description: "Memory request (e.g. 1Gi, 512Mi)" - type: string type: object + x-kubernetes-preserve-unknown-fields: true + tolerations: + description: Tolerations for scheduling onto tainted nodes + items: + type: object + type: array + x-kubernetes-preserve-unknown-fields: true warehouseDir: default: /hive/warehouse description: Warehouse directory path @@ -727,6 +765,11 @@ spec: tezAm: description: Tez Application Master configuration. Enabled by default. properties: + affinity: + description: Affinity override; replaces the default spread anti-affinity + when set + type: object + x-kubernetes-preserve-unknown-fields: true autoscaling: description: "Autoscaling configuration (operator-driven, no external\ \ dependencies)" @@ -806,6 +849,13 @@ spec: default: true description: Whether Tez AM is enabled type: boolean + envVars: + description: "Component-scoped env vars, appended after the cluster-wide\ + \ envVars" + items: + type: object + type: array + x-kubernetes-preserve-unknown-fields: true extraVolumeMounts: description: Additional volume mounts for the container items: @@ -824,32 +874,28 @@ spec: description: Number of replicas type: integer resources: + default: + requests: + cpu: 500m + memory: 1Gi description: Resource requirements for pods - properties: - limitsCpu: - description: "CPU limit (e.g. 2, 1000m)" - type: string - limitsMemory: - description: "Memory limit (e.g. 2Gi, 1024Mi)" - type: string - requestsCpu: - default: 500m - description: "CPU request (e.g. 500m, 1)" - type: string - requestsMemory: - default: 1Gi - description: "Memory request (e.g. 1Gi, 512Mi)" - type: string type: object + x-kubernetes-preserve-unknown-fields: true scratchStorageClassName: description: "StorageClass for the shared scratch PVC. Must support\ \ ReadWriteMany access. If null, uses cluster default." type: string scratchStorageSize: default: 1Gi - description: Storage size for the shared scratch PVC (ReadWriteMany) - mounted on HS2 and TezAM at /opt/hive/scratch + description: "Storage size for the shared scratch PVC (ReadWriteMany)\ + \ mounted on HS2, TezAM and LLAP at /opt/hive/scratch" type: string + tolerations: + description: Tolerations for scheduling onto tainted nodes + items: + type: object + type: array + x-kubernetes-preserve-unknown-fields: true type: object x-kubernetes-preserve-unknown-fields: true volumeMounts: diff --git a/packaging/src/kubernetes/helm/hive-operator/templates/deployment.yaml b/packaging/src/kubernetes/helm/hive-operator/templates/deployment.yaml index 1c57badfeec0..575fb31db0ef 100644 --- a/packaging/src/kubernetes/helm/hive-operator/templates/deployment.yaml +++ b/packaging/src/kubernetes/helm/hive-operator/templates/deployment.yaml @@ -37,10 +37,5 @@ spec: - name: operator image: "{{ .Values.operator.image.repository }}:{{ .Values.operator.image.tag }}" imagePullPolicy: {{ .Values.operator.image.pullPolicy }} - env: - - name: OPERATOR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace resources: {{- toYaml .Values.operator.resources | nindent 12 }} diff --git a/packaging/src/kubernetes/helm/hive-operator/templates/hivecluster.yaml b/packaging/src/kubernetes/helm/hive-operator/templates/hivecluster.yaml index a2b465ba0b37..6fd7fcb4fc17 100644 --- a/packaging/src/kubernetes/helm/hive-operator/templates/hivecluster.yaml +++ b/packaging/src/kubernetes/helm/hive-operator/templates/hivecluster.yaml @@ -70,6 +70,18 @@ spec: extraVolumeMounts: {{- toYaml .Values.cluster.metastore.extraVolumeMounts | nindent 6 }} {{- end }} + {{- if .Values.cluster.metastore.tolerations }} + tolerations: + {{- toYaml .Values.cluster.metastore.tolerations | nindent 6 }} + {{- end }} + {{- if .Values.cluster.metastore.affinity }} + affinity: + {{- toYaml .Values.cluster.metastore.affinity | nindent 6 }} + {{- end }} + {{- if .Values.cluster.metastore.envVars }} + envVars: + {{- toYaml .Values.cluster.metastore.envVars | nindent 6 }} + {{- end }} {{- if and .Values.cluster.metastore.autoscaling .Values.cluster.metastore.autoscaling.enabled }} autoscaling: enabled: true @@ -103,6 +115,18 @@ spec: externalJars: {{- toYaml .Values.cluster.hiveServer2.externalJars | nindent 6 }} {{- end }} + {{- if .Values.cluster.hiveServer2.tolerations }} + tolerations: + {{- toYaml .Values.cluster.hiveServer2.tolerations | nindent 6 }} + {{- end }} + {{- if .Values.cluster.hiveServer2.affinity }} + affinity: + {{- toYaml .Values.cluster.hiveServer2.affinity | nindent 6 }} + {{- end }} + {{- if .Values.cluster.hiveServer2.envVars }} + envVars: + {{- toYaml .Values.cluster.hiveServer2.envVars | nindent 6 }} + {{- end }} {{- if .Values.cluster.hiveServer2.extraVolumes }} extraVolumes: {{- toYaml .Values.cluster.hiveServer2.extraVolumes | nindent 6 }} @@ -144,6 +168,10 @@ spec: configOverrides: {{- toYaml .configOverrides | nindent 6 }} {{- end }} + {{- if .envVars }} + envVars: + {{- toYaml .envVars | nindent 6 }} + {{- end }} {{- if .extraVolumes }} extraVolumes: {{- toYaml .extraVolumes | nindent 6 }} @@ -152,6 +180,14 @@ spec: extraVolumeMounts: {{- toYaml .extraVolumeMounts | nindent 6 }} {{- end }} + {{- if .tolerations }} + tolerations: + {{- toYaml .tolerations | nindent 6 }} + {{- end }} + {{- if .affinity }} + affinity: + {{- toYaml .affinity | nindent 6 }} + {{- end }} {{- if and .autoscaling .autoscaling.enabled }} autoscaling: enabled: true @@ -166,6 +202,14 @@ spec: {{- if .tezAm }} tezAm: replicas: {{ .tezAm.replicas | default 1 }} + {{- if .tezAm.affinity }} + affinity: + {{- toYaml .tezAm.affinity | nindent 8 }} + {{- end }} + {{- if .tezAm.tolerations }} + tolerations: + {{- toYaml .tezAm.tolerations | nindent 8 }} + {{- end }} {{- if and .tezAm.autoscaling .tezAm.autoscaling.enabled }} autoscaling: enabled: true @@ -202,6 +246,18 @@ spec: extraVolumeMounts: {{- toYaml .Values.cluster.tezAm.extraVolumeMounts | nindent 6 }} {{- end }} + {{- if .Values.cluster.tezAm.tolerations }} + tolerations: + {{- toYaml .Values.cluster.tezAm.tolerations | nindent 6 }} + {{- end }} + {{- if .Values.cluster.tezAm.affinity }} + affinity: + {{- toYaml .Values.cluster.tezAm.affinity | nindent 6 }} + {{- end }} + {{- if .Values.cluster.tezAm.envVars }} + envVars: + {{- toYaml .Values.cluster.tezAm.envVars | nindent 6 }} + {{- end }} {{- if and .Values.cluster.tezAm.autoscaling .Values.cluster.tezAm.autoscaling.enabled }} autoscaling: enabled: true @@ -251,5 +307,5 @@ spec: {{- end }} {{- end }} - suspend: false + suspend: {{ .Values.cluster.suspend | default false }} {{- end }} diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/HiveOperatorMain.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/HiveOperatorMain.java index a3d23c752c60..319141b1b3e0 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/HiveOperatorMain.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/HiveOperatorMain.java @@ -22,6 +22,7 @@ import io.javaoperatorsdk.operator.Operator; import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; import io.javaoperatorsdk.operator.api.config.ResolvedControllerConfiguration; +import io.javaoperatorsdk.operator.api.config.ControllerConfigurationOverrider; import org.apache.hive.kubernetes.operator.model.HiveCluster; import org.apache.hive.kubernetes.operator.reconciler.HiveClusterReconciler; import org.apache.hive.kubernetes.operator.reconciler.HiveWorkflowSpec; @@ -45,6 +46,13 @@ public static void main(String[] args) { // Get the annotation-derived base config, then inject our programmatic workflow spec. ControllerConfiguration baseConfig = operator.getConfigurationService().getConfigurationFor(reconciler); + + // Watch only our own namespace: cluster-wide, a HiveCluster elsewhere using a field this + // build does not know fails to deserialise in the informer, which stops the process. + baseConfig = ControllerConfigurationOverrider.override(baseConfig) + .watchingOnlyCurrentNamespace().build(); + LOG.info("Watching only this operator's own namespace"); + HiveWorkflowSpec workflowSpec = new HiveWorkflowSpec(); ((ResolvedControllerConfiguration) baseConfig) .setWorkflowSpec(workflowSpec); diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java index 4cb348bfd476..bb031b50ac29 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveDependentResource.java @@ -32,12 +32,12 @@ import io.fabric8.kubernetes.api.model.EnvVar; import io.fabric8.kubernetes.api.model.EnvVarBuilder; import io.fabric8.kubernetes.api.model.HasMetadata; -import io.fabric8.kubernetes.api.model.Quantity; -import io.fabric8.kubernetes.api.model.ResourceRequirements; import io.fabric8.kubernetes.api.model.Probe; import io.fabric8.kubernetes.api.model.ProbeBuilder; import io.fabric8.kubernetes.api.model.IntOrString; -import io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder; +import io.fabric8.kubernetes.api.model.PodSpec; +import io.fabric8.kubernetes.api.model.Affinity; +import io.fabric8.kubernetes.api.model.Toleration; import io.fabric8.kubernetes.api.model.Volume; import io.fabric8.kubernetes.api.model.VolumeBuilder; import io.fabric8.kubernetes.api.model.VolumeMount; @@ -50,7 +50,6 @@ import org.apache.hive.kubernetes.operator.model.HiveCluster; import org.apache.hive.kubernetes.operator.model.spec.AutoscalingSpec; import org.apache.hive.kubernetes.operator.model.spec.DatabaseConfig; -import org.apache.hive.kubernetes.operator.model.spec.ResourceRequirementsSpec; import org.apache.hive.kubernetes.operator.model.spec.SecretKeyRef; import org.apache.hive.kubernetes.operator.model.spec.ProbeSpec; @@ -452,27 +451,6 @@ protected static void buildMetastoreVolumes( HiveConfigMapDependent.Hadoop.resourceName(hiveCluster))); } - /** Builds Kubernetes ResourceRequirements from the operator's spec. */ - protected static ResourceRequirements buildResources(ResourceRequirementsSpec spec) { - if (spec == null) { - return new ResourceRequirements(); - } - ResourceRequirementsBuilder builder = new ResourceRequirementsBuilder(); - if (spec.requestsCpu() != null) { - builder.addToRequests("cpu", new Quantity(spec.requestsCpu())); - } - if (spec.requestsMemory() != null) { - builder.addToRequests("memory", new Quantity(spec.requestsMemory())); - } - if (spec.limitsCpu() != null) { - builder.addToLimits("cpu", new Quantity(spec.limitsCpu())); - } - if (spec.limitsMemory() != null) { - builder.addToLimits("memory", new Quantity(spec.limitsMemory())); - } - return builder.build(); - } - /** * Sets a preferred pod anti-affinity on the pod spec if no affinity is * already defined. This spreads replicas across nodes while allowing @@ -499,6 +477,24 @@ protected static void applySpreadAffinityIfAbsent( .build()); } + /** + * Sets the user-provided affinity override, if any. Must run before + * {@link #applySpreadAffinityIfAbsent}, which only sets its default when + * the pod spec has no affinity yet. + */ + protected static void applyAffinityOverride(PodSpec podSpec, Affinity affinity) { + if (affinity != null) { + podSpec.setAffinity(affinity); + } + } + + /** Sets the given tolerations on the pod spec, if any. */ + protected static void applyTolerations(PodSpec podSpec, List tolerations) { + if (tolerations != null && !tolerations.isEmpty()) { + podSpec.setTolerations(tolerations); + } + } + /** * Builds an init container that downloads external JARs via wget * (for http/https URLs) or hadoop fs (for HDFS/cloud paths). diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveServer2DeploymentDependent.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveServer2DeploymentDependent.java index 28d0250856ee..aab6c8908974 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveServer2DeploymentDependent.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveServer2DeploymentDependent.java @@ -51,7 +51,6 @@ public class HiveServer2DeploymentDependent extends HiveDependentResource { public static final String COMPONENT = ConfigUtils.COMPONENT_HIVESERVER2; - private static final String SCRATCH_MOUNT_PATH = "/opt/hive/scratch"; public HiveServer2DeploymentDependent() { super(Deployment.class); @@ -86,6 +85,7 @@ protected Deployment desired(HiveCluster hiveCluster, if (spec.envVars() != null) { envVars.addAll(spec.envVars()); } + envVars.addAll(hs2.envVars()); // Env vars consumed by the Hive Docker entrypoint.sh to // configure Tez execution mode at container startup. @@ -185,10 +185,10 @@ protected Deployment desired(HiveCluster hiveCluster, if (tezAmEnabled) { volumeMounts.add( new io.fabric8.kubernetes.api.model.VolumeMountBuilder() - .withName("scratch") - .withMountPath(SCRATCH_MOUNT_PATH).build()); + .withName(ScratchPvcDependent.COMPONENT) + .withMountPath(ConfigUtils.SCRATCH_MOUNT_PATH).build()); volumes.add(new io.fabric8.kubernetes.api.model.VolumeBuilder() - .withName("scratch") + .withName(ScratchPvcDependent.COMPONENT) .withNewPersistentVolumeClaim() .withClaimName(ScratchPvcDependent.resourceName(hiveCluster)) .endPersistentVolumeClaim() @@ -262,7 +262,7 @@ protected Deployment desired(HiveCluster hiveCluster, .withPorts(ports) .withReadinessProbe(readinessProbe) .withLivenessProbe(livenessProbe) - .withResources(buildResources(hs2.resources())) + .withResources(hs2.resources()) .withVolumeMounts(volumeMounts) .endContainer() .withVolumes(volumes) @@ -271,8 +271,12 @@ protected Deployment desired(HiveCluster hiveCluster, .endSpec() .build(); + applyAffinityOverride( + deployment.getSpec().getTemplate().getSpec(), hs2.affinity()); applySpreadAffinityIfAbsent( deployment.getSpec().getTemplate().getSpec(), selectorLabels); + applyTolerations( + deployment.getSpec().getTemplate().getSpec(), hs2.tolerations()); // Graceful scale-down: deregister from ZK, then poll JMX Exporter for sessions. if (autoscaling.isEnabled()) { diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java index a144fc121219..a2c10d3688c4 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java @@ -392,14 +392,15 @@ private Deployment doBuildTezAmDeployment(HiveCluster hiveCluster, LlapSpec llap if (spec.envVars() != null) { envVars.addAll(spec.envVars()); } + envVars.addAll(spec.tezAm().envVars()); List volumeMounts = new ArrayList<>(); volumeMounts.add(new io.fabric8.kubernetes.api.model.VolumeMountBuilder() .withName(HIVE_CONFIG_VOLUME) .withMountPath(CONF_MOUNT_PATH).build()); volumeMounts.add(new io.fabric8.kubernetes.api.model.VolumeMountBuilder() - .withName("scratch") - .withMountPath("/opt/hive/scratch").build()); + .withName(ScratchPvcDependent.COMPONENT) + .withMountPath(ConfigUtils.SCRATCH_MOUNT_PATH).build()); List volumes = new ArrayList<>(); // Projected volume: hive-site.xml from HS2 CM, tez-site.xml from per-LLAP CM, core-site.xml from Hadoop CM @@ -408,7 +409,7 @@ private Deployment doBuildTezAmDeployment(HiveCluster hiveCluster, LlapSpec llap String tezAmCmName = tezAmConfigMapName(hiveCluster, llap); volumes.add(buildProjectedConfigVolume(HIVE_CONFIG_VOLUME, hs2CmName, tezAmCmName, hadoopCmName)); volumes.add(new io.fabric8.kubernetes.api.model.VolumeBuilder() - .withName("scratch") + .withName(ScratchPvcDependent.COMPONENT) .withNewPersistentVolumeClaim() .withClaimName(ScratchPvcDependent.resourceName(hiveCluster)) .endPersistentVolumeClaim() @@ -461,7 +462,7 @@ private Deployment doBuildTezAmDeployment(HiveCluster hiveCluster, LlapSpec llap .withImagePullPolicy(spec.imagePullPolicy()) .withEnv(envVars) .withPorts(ports) - .withResources(buildResources(spec.tezAm().resources())) + .withResources(spec.tezAm().resources()) .withVolumeMounts(volumeMounts) .endContainer() .withVolumes(volumes) @@ -470,8 +471,17 @@ private Deployment doBuildTezAmDeployment(HiveCluster hiveCluster, LlapSpec llap .endSpec() .build(); + // Per-cluster affinity, falling back to the global one: spec.tezAm's is a single block + // shared by every cluster's TezAM. + applyAffinityOverride( + deployment.getSpec().getTemplate().getSpec(), + llap.tezAm().affinity() != null ? llap.tezAm().affinity() : spec.tezAm().affinity()); applySpreadAffinityIfAbsent( deployment.getSpec().getTemplate().getSpec(), selectorLabels); + applyTolerations( + deployment.getSpec().getTemplate().getSpec(), + llap.tezAm().tolerations() != null && !llap.tezAm().tolerations().isEmpty() + ? llap.tezAm().tolerations() : spec.tezAm().tolerations()); appendUserVolumes(deployment.getSpec().getTemplate().getSpec(), spec.volumes(), spec.volumeMounts(), @@ -522,6 +532,11 @@ private StatefulSet doBuildStatefulSet(HiveCluster hiveCluster, LlapSpec llap, I if (spec.envVars() != null) { envVars.addAll(spec.envVars()); } + // Component-scoped last, so a per-cluster value wins over the cluster-wide one -- the same + // order HiveServer2/Metastore/TezAm use. Without this LLAP was the one component with no + // scoped env vars, so LLAP_DAEMON_OPTS and LLAP_DAEMON_HEAPSIZE had to be set cluster-wide + // and were then present, unread, on every other pod. + envVars.addAll(llap.envVars()); int managementPort = ConfigUtils.getInt(llap.configOverrides(), ConfigUtils.HIVE_LLAP_MANAGEMENT_RPC_PORT_KEY, null, @@ -562,6 +577,19 @@ private StatefulSet doBuildStatefulSet(HiveCluster hiveCluster, LlapSpec llap, I String hadoopCmName = HiveConfigMapDependent.Hadoop.resourceName(hiveCluster); volumes.add(buildProjectedConfigVolume(LLAP_CONFIG_VOLUME, cmName, hadoopCmName)); + // The scratch PVC only exists when the TezAM does, so mount it on the same condition. + if (spec.tezAm().isEnabled()) { + volumeMounts.add(new io.fabric8.kubernetes.api.model.VolumeMountBuilder() + .withName(ScratchPvcDependent.COMPONENT) + .withMountPath(ConfigUtils.SCRATCH_MOUNT_PATH).build()); + volumes.add(new io.fabric8.kubernetes.api.model.VolumeBuilder() + .withName(ScratchPvcDependent.COMPONENT) + .withNewPersistentVolumeClaim() + .withClaimName(ScratchPvcDependent.resourceName(hiveCluster)) + .endPersistentVolumeClaim() + .build()); + } + List initContainers = new ArrayList<>(); addExternalJars(spec.image(), spec.externalJars(), initContainers, volumeMounts, volumes, envVars); @@ -609,7 +637,7 @@ private StatefulSet doBuildStatefulSet(HiveCluster hiveCluster, LlapSpec llap, I .withEnv(envVars) .withPorts(ports) .withReadinessProbe(readinessProbe) - .withResources(buildResources(llap.resources())) + .withResources(llap.resources()) .withVolumeMounts(volumeMounts) .endContainer() .withVolumes(volumes) @@ -618,8 +646,12 @@ private StatefulSet doBuildStatefulSet(HiveCluster hiveCluster, LlapSpec llap, I .endSpec() .build(); + applyAffinityOverride( + statefulSet.getSpec().getTemplate().getSpec(), llap.affinity()); applySpreadAffinityIfAbsent( statefulSet.getSpec().getTemplate().getSpec(), selectorLabels); + applyTolerations( + statefulSet.getSpec().getTemplate().getSpec(), llap.tolerations()); if (autoscaling.isEnabled()) { String preStopScript = buildDualMetricDrainScript( diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/MetastoreDeploymentDependent.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/MetastoreDeploymentDependent.java index 7ee46dc9187b..73afedd9dbae 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/MetastoreDeploymentDependent.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/MetastoreDeploymentDependent.java @@ -84,6 +84,7 @@ protected Deployment desired(HiveCluster hiveCluster, if (spec.envVars() != null) { envVars.addAll(spec.envVars()); } + envVars.addAll(spec.metastore().envVars()); int thriftPort = ConfigUtils.getInt( spec.metastore().configOverrides(), @@ -172,8 +173,7 @@ protected Deployment desired(HiveCluster hiveCluster, .withPorts(ports) .withReadinessProbe(readinessProbe) .withLivenessProbe(livenessProbe) - .withResources(buildResources( - spec.metastore().resources())) + .withResources(spec.metastore().resources()) .withVolumeMounts(volumeMounts) .endContainer() .withVolumes(volumes) @@ -182,8 +182,12 @@ protected Deployment desired(HiveCluster hiveCluster, .endSpec() .build(); + applyAffinityOverride( + deployment.getSpec().getTemplate().getSpec(), spec.metastore().affinity()); applySpreadAffinityIfAbsent( deployment.getSpec().getTemplate().getSpec(), selectorLabels); + applyTolerations( + deployment.getSpec().getTemplate().getSpec(), spec.metastore().tolerations()); // HMS uses HTTP transport mode — connections are stateless, so no session // drain is needed. The preStop hook simply sends SIGTERM directly to the diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/ScratchPvcDependent.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/ScratchPvcDependent.java index 0f459105ec18..a034ae2e684c 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/ScratchPvcDependent.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/ScratchPvcDependent.java @@ -29,11 +29,12 @@ import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependent; import org.apache.hive.kubernetes.operator.model.HiveCluster; import org.apache.hive.kubernetes.operator.model.spec.TezAmSpec; +import org.apache.hive.kubernetes.operator.util.ConfigUtils; import org.apache.hive.kubernetes.operator.util.Labels; /** - * Manages the shared scratch PersistentVolumeClaim mounted by both - * HiveServer2 and TezAM at /opt/hive/scratch. + * Manages the shared scratch PersistentVolumeClaim mounted by HiveServer2, + * TezAM and LLAP at {@link ConfigUtils#SCRATCH_MOUNT_PATH}. *

* This mirrors the Docker Compose pattern where a named volume * {@code scratch:/opt/hive/scratch} is shared between the hs2 and @@ -50,6 +51,7 @@ public class ScratchPvcDependent extends HiveDependentResource { + /** Component label value, and the name of the pod volume backed by this PVC. */ public static final String COMPONENT = "scratch"; public ScratchPvcDependent() { diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/HiveClusterSpec.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/HiveClusterSpec.java index 3ff8bd4b4b11..04e834017410 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/HiveClusterSpec.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/HiveClusterSpec.java @@ -97,12 +97,12 @@ public record HiveClusterSpec( Objects.requireNonNull(zookeeper, "zookeeper must be provided in the HiveCluster spec"); metastore = metastore != null ? metastore : new MetastoreSpec( - 1, null, null, null, null, null, null, true, null, null, null, null); + 1, null, null, null, null, null, null, null, null, null, true, null, null, null, null); hiveServer2 = hiveServer2 != null ? hiveServer2 : new HiveServer2Spec( - 1, null, null, null, null, null, null, null, null, null); + 1, null, null, null, null, null, null, null, null, null, null, null, null); llapClusters = llapClusters != null ? llapClusters : List.of(); tezAm = tezAm != null ? tezAm : new TezAmSpec( - 1, null, null, null, null, true, null, null, null); + 1, null, null, null, null, null, null, null, true, null, null, null); envVars = envVars != null ? envVars : List.of(); externalJars = externalJars != null ? externalJars : List.of(); volumes = volumes != null ? volumes : List.of(); diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/HiveServer2Spec.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/HiveServer2Spec.java index b4962d7e6f34..27f54eb17a24 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/HiveServer2Spec.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/HiveServer2Spec.java @@ -26,6 +26,10 @@ import io.fabric8.crd.generator.annotation.PreserveUnknownFields; import io.fabric8.crd.generator.annotation.SchemaFrom; import io.fabric8.generator.annotation.Default; +import io.fabric8.kubernetes.api.model.Affinity; +import io.fabric8.kubernetes.api.model.EnvVar; +import io.fabric8.kubernetes.api.model.ResourceRequirements; +import io.fabric8.kubernetes.api.model.Toleration; import io.fabric8.kubernetes.api.model.Volume; import io.fabric8.kubernetes.api.model.VolumeMount; @@ -35,7 +39,9 @@ public record HiveServer2Spec( @Default("1") Integer replicas, @JsonPropertyDescription("Resource requirements for pods") - ResourceRequirementsSpec resources, + @Default("{\"requests\": {\"cpu\": \"500m\", \"memory\": \"1Gi\"}}") + @SchemaFrom(type = Object.class) @PreserveUnknownFields + ResourceRequirements resources, @JsonPropertyDescription("Additional configuration overrides as key-value pairs") Map configOverrides, @JsonPropertyDescription("Additional volumes to attach to the pod (e.g., for keytabs or truststores)") @@ -44,6 +50,15 @@ public record HiveServer2Spec( @JsonPropertyDescription("Additional volume mounts for the container") @SchemaFrom(type = Object[].class) @PreserveUnknownFields List extraVolumeMounts, + @JsonPropertyDescription("Tolerations for scheduling onto tainted nodes") + @SchemaFrom(type = Object[].class) @PreserveUnknownFields + List tolerations, + @JsonPropertyDescription("Affinity override; replaces the default spread anti-affinity when set") + @SchemaFrom(type = Object.class) @PreserveUnknownFields + Affinity affinity, + @JsonPropertyDescription("Component-scoped env vars, appended after the cluster-wide envVars") + @SchemaFrom(type = Object[].class) @PreserveUnknownFields + List envVars, @JsonPropertyDescription("Kubernetes Service type: ClusterIP, LoadBalancer, or NodePort") @Default("ClusterIP") String serviceType, @@ -61,6 +76,8 @@ public record HiveServer2Spec( serviceType = serviceType != null ? serviceType : "ClusterIP"; extraVolumes = extraVolumes != null ? extraVolumes : List.of(); extraVolumeMounts = extraVolumeMounts != null ? extraVolumeMounts : List.of(); + tolerations = tolerations != null ? tolerations : List.of(); + envVars = envVars != null ? envVars : List.of(); externalJars = externalJars != null ? externalJars : List.of(); autoscaling = autoscaling != null ? autoscaling : new AutoscalingSpec( false, 1, 80, 0, 60, 600, 300, 10, 90, 30, null); diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/LlapSpec.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/LlapSpec.java index eb42d8c8c0ee..288e218afece 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/LlapSpec.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/LlapSpec.java @@ -28,6 +28,10 @@ import io.fabric8.crd.generator.annotation.SchemaFrom; import io.fabric8.generator.annotation.Default; import io.fabric8.generator.annotation.Required; +import io.fabric8.kubernetes.api.model.Affinity; +import io.fabric8.kubernetes.api.model.EnvVar; +import io.fabric8.kubernetes.api.model.ResourceRequirements; +import io.fabric8.kubernetes.api.model.Toleration; import io.fabric8.kubernetes.api.model.Volume; import io.fabric8.kubernetes.api.model.VolumeMount; @@ -41,7 +45,9 @@ public record LlapSpec( @Default("1") Integer replicas, @JsonPropertyDescription("Resource requirements for pods") - ResourceRequirementsSpec resources, + @Default("{\"requests\": {\"cpu\": \"500m\", \"memory\": \"1Gi\"}}") + @SchemaFrom(type = Object.class) @PreserveUnknownFields + ResourceRequirements resources, @JsonPropertyDescription("Additional configuration overrides as key-value pairs") Map configOverrides, @JsonPropertyDescription("Additional volumes to attach to the pod (e.g., for keytabs or truststores)") @@ -50,6 +56,15 @@ public record LlapSpec( @JsonPropertyDescription("Additional volume mounts for the container") @SchemaFrom(type = Object[].class) @PreserveUnknownFields List extraVolumeMounts, + @JsonPropertyDescription("Tolerations for scheduling onto tainted nodes") + @SchemaFrom(type = Object[].class) @PreserveUnknownFields + List tolerations, + @JsonPropertyDescription("Affinity override; replaces the default spread anti-affinity when set") + @SchemaFrom(type = Object.class) @PreserveUnknownFields + Affinity affinity, + @JsonPropertyDescription("Component-scoped env vars, appended after the cluster-wide envVars") + @SchemaFrom(type = Object[].class) @PreserveUnknownFields + List envVars, @JsonPropertyDescription("Whether LLAP is enabled") @Default("true") Boolean enabled, @@ -76,7 +91,16 @@ public record LlapTezAmSpec( @Default("1") Integer replicas, @JsonPropertyDescription("Autoscaling configuration for this LLAP cluster's TezAM") - AutoscalingSpec autoscaling) { + AutoscalingSpec autoscaling, + @JsonPropertyDescription("Affinity for this LLAP cluster's TezAM, overriding " + + "spec.tezAm.affinity. Set it with more than one LLAP cluster.") + @SchemaFrom(type = Object.class) + @PreserveUnknownFields + Affinity affinity, + @JsonPropertyDescription("Tolerations for this LLAP cluster's TezAM, overriding spec.tezAm.tolerations") + @SchemaFrom(type = Object[].class) + @PreserveUnknownFields + List tolerations) { public LlapTezAmSpec { replicas = replicas != null ? replicas : 1; @@ -101,9 +125,11 @@ public record LlapTezAmSpec( serviceHosts = serviceHosts != null ? serviceHosts : "@" + name; extraVolumes = extraVolumes != null ? extraVolumes : List.of(); extraVolumeMounts = extraVolumeMounts != null ? extraVolumeMounts : List.of(); + tolerations = tolerations != null ? tolerations : List.of(); + envVars = envVars != null ? envVars : List.of(); autoscaling = autoscaling != null ? autoscaling : new AutoscalingSpec( false, 0, 1, 20, 60, 900, 600, 10, 0, 0, null); - tezAm = tezAm != null ? tezAm : new LlapTezAmSpec(null, null); + tezAm = tezAm != null ? tezAm : new LlapTezAmSpec(null, null, null, null); } public boolean isEnabled() { diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/MetastoreSpec.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/MetastoreSpec.java index 6548d999918a..ba48f3ddf3be 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/MetastoreSpec.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/MetastoreSpec.java @@ -26,6 +26,10 @@ import io.fabric8.crd.generator.annotation.PreserveUnknownFields; import io.fabric8.crd.generator.annotation.SchemaFrom; import io.fabric8.generator.annotation.Default; +import io.fabric8.kubernetes.api.model.Affinity; +import io.fabric8.kubernetes.api.model.EnvVar; +import io.fabric8.kubernetes.api.model.ResourceRequirements; +import io.fabric8.kubernetes.api.model.Toleration; import io.fabric8.kubernetes.api.model.Volume; import io.fabric8.kubernetes.api.model.VolumeMount; @@ -35,7 +39,9 @@ public record MetastoreSpec( @Default("1") Integer replicas, @JsonPropertyDescription("Resource requirements for pods") - ResourceRequirementsSpec resources, + @Default("{\"requests\": {\"cpu\": \"500m\", \"memory\": \"1Gi\"}}") + @SchemaFrom(type = Object.class) @PreserveUnknownFields + ResourceRequirements resources, @JsonPropertyDescription("Additional configuration overrides as key-value pairs") Map configOverrides, @JsonPropertyDescription("Additional volumes to attach to the pod (e.g., for keytabs or truststores)") @@ -44,6 +50,15 @@ public record MetastoreSpec( @JsonPropertyDescription("Additional volume mounts for the container") @SchemaFrom(type = Object[].class) @PreserveUnknownFields List extraVolumeMounts, + @JsonPropertyDescription("Tolerations for scheduling onto tainted nodes") + @SchemaFrom(type = Object[].class) @PreserveUnknownFields + List tolerations, + @JsonPropertyDescription("Affinity override; replaces the default spread anti-affinity when set") + @SchemaFrom(type = Object.class) @PreserveUnknownFields + Affinity affinity, + @JsonPropertyDescription("Component-scoped env vars, appended after the cluster-wide envVars") + @SchemaFrom(type = Object[].class) @PreserveUnknownFields + List envVars, @JsonPropertyDescription("Database connection configuration for the metastore backend") DatabaseConfig database, @JsonPropertyDescription("Warehouse directory path") @@ -69,6 +84,8 @@ public record MetastoreSpec( enabled = enabled != null ? enabled : true; extraVolumes = extraVolumes != null ? extraVolumes : List.of(); extraVolumeMounts = extraVolumeMounts != null ? extraVolumeMounts : List.of(); + tolerations = tolerations != null ? tolerations : List.of(); + envVars = envVars != null ? envVars : List.of(); autoscaling = autoscaling != null ? autoscaling : new AutoscalingSpec( false, 1, 75, 0, 60, 300, 60, 10, 90, 30, null); } diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/ResourceRequirementsSpec.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/ResourceRequirementsSpec.java deleted file mode 100644 index 26c1d81776a5..000000000000 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/ResourceRequirementsSpec.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hive.kubernetes.operator.model.spec; - -import com.fasterxml.jackson.annotation.JsonPropertyDescription; -import io.fabric8.generator.annotation.Default; - -/** Kubernetes resource requirements specification for CPU and memory. */ -public record ResourceRequirementsSpec( - @JsonPropertyDescription("CPU request (e.g. 500m, 1)") - @Default("500m") - String requestsCpu, - @JsonPropertyDescription("Memory request (e.g. 1Gi, 512Mi)") - @Default("1Gi") - String requestsMemory, - @JsonPropertyDescription("CPU limit (e.g. 2, 1000m)") - String limitsCpu, - @JsonPropertyDescription("Memory limit (e.g. 2Gi, 1024Mi)") - String limitsMemory) { - - public ResourceRequirementsSpec { - requestsCpu = requestsCpu != null ? requestsCpu : "500m"; - requestsMemory = requestsMemory != null ? requestsMemory : "1Gi"; - } -} diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/TezAmSpec.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/TezAmSpec.java index 8165b9b829ae..9caa498e5124 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/TezAmSpec.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/TezAmSpec.java @@ -26,6 +26,10 @@ import io.fabric8.crd.generator.annotation.PreserveUnknownFields; import io.fabric8.crd.generator.annotation.SchemaFrom; import io.fabric8.generator.annotation.Default; +import io.fabric8.kubernetes.api.model.Affinity; +import io.fabric8.kubernetes.api.model.EnvVar; +import io.fabric8.kubernetes.api.model.ResourceRequirements; +import io.fabric8.kubernetes.api.model.Toleration; import io.fabric8.kubernetes.api.model.Volume; import io.fabric8.kubernetes.api.model.VolumeMount; @@ -35,7 +39,9 @@ public record TezAmSpec( @Default("1") Integer replicas, @JsonPropertyDescription("Resource requirements for pods") - ResourceRequirementsSpec resources, + @Default("{\"requests\": {\"cpu\": \"500m\", \"memory\": \"1Gi\"}}") + @SchemaFrom(type = Object.class) @PreserveUnknownFields + ResourceRequirements resources, @JsonPropertyDescription("Additional configuration overrides as key-value pairs") Map configOverrides, @JsonPropertyDescription("Additional volumes to attach to the pod (e.g., for keytabs or truststores)") @@ -44,11 +50,20 @@ public record TezAmSpec( @JsonPropertyDescription("Additional volume mounts for the container") @SchemaFrom(type = Object[].class) @PreserveUnknownFields List extraVolumeMounts, + @JsonPropertyDescription("Tolerations for scheduling onto tainted nodes") + @SchemaFrom(type = Object[].class) @PreserveUnknownFields + List tolerations, + @JsonPropertyDescription("Affinity override; replaces the default spread anti-affinity when set") + @SchemaFrom(type = Object.class) @PreserveUnknownFields + Affinity affinity, + @JsonPropertyDescription("Component-scoped env vars, appended after the cluster-wide envVars") + @SchemaFrom(type = Object[].class) @PreserveUnknownFields + List envVars, @JsonPropertyDescription("Whether Tez AM is enabled") @Default("true") Boolean enabled, @JsonPropertyDescription("Storage size for the shared scratch PVC " - + "(ReadWriteMany) mounted on HS2 and TezAM at /opt/hive/scratch") + + "(ReadWriteMany) mounted on HS2, TezAM and LLAP at /opt/hive/scratch") @Default("1Gi") String scratchStorageSize, @JsonPropertyDescription("StorageClass for the shared scratch PVC. " @@ -63,6 +78,8 @@ public record TezAmSpec( scratchStorageSize = scratchStorageSize != null ? scratchStorageSize : "1Gi"; extraVolumes = extraVolumes != null ? extraVolumes : List.of(); extraVolumeMounts = extraVolumeMounts != null ? extraVolumeMounts : List.of(); + tolerations = tolerations != null ? tolerations : List.of(); + envVars = envVars != null ? envVars : List.of(); autoscaling = autoscaling != null ? autoscaling : new AutoscalingSpec( false, 0, 0, 0, 60, 600, 120, 10, 0, 0, null); } diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java index a7da524a3dbd..ec37d6820ad1 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java @@ -160,9 +160,10 @@ public UpdateControl reconcile(HiveCluster resource, Context metastoreReady() { } /** - * Reconcile precondition for HiveServer2: if Metastore is managed, - * wait for it to be ready before reconciling HS2. + * Reconcile precondition for HiveServer2: at least one ready Metastore replica, not every + * replica. An unmet reconcile precondition deletes the dependent, so requiring the full + * count took HiveServer2 down whenever one Metastore pod restarted. */ private static Condition hs2Precondition() { return (dependentResource, primary, context) -> { if (!primary.getSpec().metastore().isEnabled()) { return true; } - int desiredReplicas; - if (primary.getSpec().metastore().autoscaling().isEnabled()) { - desiredReplicas = Math.max(1, primary.getSpec().metastore().autoscaling().minReplicas()); - } else { - desiredReplicas = primary.getSpec().metastore().replicas(); - } return context.getSecondaryResources(Deployment.class).stream() .filter(d -> d.getMetadata().getName().equals( primary.getMetadata().getName() + "-metastore")) .findFirst() .map(deployment -> deployment.getStatus() != null && deployment.getStatus().getReadyReplicas() != null - && deployment.getStatus().getReadyReplicas() >= desiredReplicas) + && deployment.getStatus().getReadyReplicas() >= 1) .orElse(false); }; } diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java index 76dac7a640b9..640b7d3abf75 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java @@ -74,7 +74,12 @@ public static String tezAmComponentKey(String llapName) { public static final String HIVE_USER_INSTALL_DIR_KEY = "hive.user.install.directory"; - public static final String HIVE_LOCAL_SCRATCH_DIR_KEY = "hive.exec.local.scratchdir"; + public static final String HIVE_SCRATCH_DIR_KEY = "hive.exec.scratchdir"; + + public static final String MAPREDUCE_FRAMEWORK_NAME_KEY = "mapreduce.framework.name"; + + /** Mount path of the shared scratch PVC on HS2, TezAM and LLAP. */ + public static final String SCRATCH_MOUNT_PATH = "/opt/hive/scratch"; public static final String HIVE_SERVER2_TEZ_USE_EXTERNAL_SESSIONS_KEY = "hive.server2.tez.use.external.sessions"; diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java index 9c78fa153425..b4957a24d86e 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java @@ -78,11 +78,11 @@ public static Map getHiveServer2HiveSite( props.put(ConfigUtils.HIVE_JAR_DIRECTORY_KEY, "/tmp"); props.put(ConfigUtils.HIVE_USER_INSTALL_DIR_KEY, "/tmp"); if (tezAmEnabled) { - props.put(ConfigUtils.HIVE_LOCAL_SCRATCH_DIR_KEY, - "/opt/hive/scratch"); - } - - if (tezAmEnabled) { + // "yarn-tez" is Tez's own framework name (YarnTezClientProtocolProvider). The default, + // "local", would send staging to the local scratch dir, which the TezAM pod cannot + // read. file:// because a bare path resolves against fs.defaultFS. + props.put(ConfigUtils.MAPREDUCE_FRAMEWORK_NAME_KEY, "yarn-tez"); + props.put(ConfigUtils.HIVE_SCRATCH_DIR_KEY, "file://" + ConfigUtils.SCRATCH_MOUNT_PATH); props.put(ConfigUtils.HIVE_SERVER2_TEZ_USE_EXTERNAL_SESSIONS_KEY, "true"); // Default external sessions namespace points to first LLAP cluster's TezAM. // Client routes to other clusters by overriding both properties in JDBC URL: @@ -114,7 +114,7 @@ public static Map getHiveServer2HiveSite( props.put(ConfigUtils.HIVE_SERVER2_TEZ_USE_EXTERNAL_SESSIONS_KEY, "false"); props.put(ConfigUtils.TEZ_LOCAL_MODE_KEY, "true"); props.put(ConfigUtils.TEZ_AM_FRAMEWORK_MODE_KEY, "LOCAL"); - props.put("mapreduce.framework.name", "local"); + props.put(ConfigUtils.MAPREDUCE_FRAMEWORK_NAME_KEY, "local"); } // Server-side LLAP cluster routing: emit per-cluster definitions and routing rules.