From 7709fe75b6b8c2b6511747afd969da4563f39a58 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 8 Sep 2026 20:32:27 +0000 Subject: [PATCH] docs: deprecate the enableSnappy option in bigtable-beam-import The flag has been a no-op since #4338 (released in 2.14.0), which moved the import job to HBase 2.x / Hadoop 3. Its only effect was swapping in a custom SDK container image and enabling use_runner_v2 so that workers could decompress Snappy; that block was deleted in #4338 and nothing has read getEnableSnappy() since. Snappy compressed snapshots are now read natively. Mark the option deprecated rather than removing it so existing invocations keep working: - @Deprecated on the getter/setter, and the --help description now says the option is ignored. - Log a warning when --enableSnappy=true is passed so it is not silently dropped. - README: collapse the duplicated "Snappy compressed Snapshots" command block (identical to the preferred method except for the flag) into a note that no extra configuration is needed, plus a deprecation warning. - SNAPSHOT_IMPORT_USAGE.md: mark ENABLE_SNAPPY deprecated. - run-snapshot-import.sh: stop passing --enableSnappy; warn if ENABLE_SNAPPY is set in the environment. - EndToEndIT: drop the setEnableSnappy(true) call; the test still imports the snappy snapshot. Change-Id: I7113f112309c7813fba7cc0d7c9b73401029d620 --- .../bigtable-beam-import/README.md | 35 +++++-------------- .../SNAPSHOT_IMPORT_USAGE.md | 2 +- .../bin/run-snapshot-import.sh | 7 ++-- .../ImportJobFromHbaseSnapshot.java | 27 +++++++++++++- .../beam/hbasesnapshots/EndToEndIT.java | 2 +- 5 files changed, 42 insertions(+), 31 deletions(-) diff --git a/bigtable-dataflow-parent/bigtable-beam-import/README.md b/bigtable-dataflow-parent/bigtable-beam-import/README.md index 180bb05352..323340d4cd 100644 --- a/bigtable-dataflow-parent/bigtable-beam-import/README.md +++ b/bigtable-dataflow-parent/bigtable-beam-import/README.md @@ -167,32 +167,15 @@ workers accordingly. ### Snappy compressed Snapshots -1. Set the environment variables. - ``` - PROJECT_ID=your-project-id - INSTANCE_ID=your-instance-id - TABLE_NAME=your-table-name - REGION=us-central1 - - SNAPSHOT_GCS_PATH="$BUCKET_NAME/hbase-migration-snap" - SNAPSHOT_NAME=your-snapshot-name - ``` - -1. Run the import. - ``` - java -jar bigtable-beam-import-2.3.0.jar importsnapshot \ - --runner=DataflowRunner \ - --project=$PROJECT_ID \ - --bigtableInstanceId=$INSTANCE_ID \ - --bigtableTableId=$TABLE_NAME \ - --hbaseSnapshotSourceDir=$SNAPSHOT_GCS_PATH/data \ - --snapshotName=$SNAPSHOT_NAME \ - --stagingLocation=$SNAPSHOT_GCS_PATH/staging \ - --gcpTempLocation=$SNAPSHOT_GCS_PATH/temp \ - --maxNumWorkers=$(expr 3 \* $CLUSTER_NUM_NODES) \ - --region=$REGION \ - --enableSnappy=true - ``` +Snappy compressed snapshots are read natively, so they are imported with the +exact same command as above, with no extra configuration. + +> [!WARNING] +> The `--enableSnappy` flag is **deprecated**. Since +> [#4338](https://github.com/googleapis/java-bigtable-hbase/pull/4338) (released +> in 2.14.0) the import job runs on HBase 2.x / Hadoop 3, which decompresses +> Snappy without the custom worker container the flag used to install. The flag +> is accepted but ignored, and it will be removed in a future release. ### Sequence Files diff --git a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md index 06e36ac999..d3117eb73d 100644 --- a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md +++ b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md @@ -26,7 +26,7 @@ The script relies on the following environment variables. You should set them be | `DISK_SIZE_GB` | *Optional* | Worker disk size in Gigabytes. | `500` (Default) | | `MAX_NUM_WORKERS` | *Optional* | Maximum number of active Dataflow workers per job. | `10` (Default) | | `USE_PUBLIC_IPS` | *Optional* | Whether Dataflow workers should have public IPs. | `false` (Default) | -| `ENABLE_SNAPPY` | *Optional* | Whether to enable Snappy compression for transit files. | `true` (Default) | +| `ENABLE_SNAPPY` | **Deprecated** | Ignored. Snappy compressed snapshots are read natively, so no flag is needed. Setting it only prints a warning. | N/A | ## Understanding Sharding diff --git a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh index b4d8826419..a7bfae513b 100755 --- a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh +++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh @@ -114,7 +114,11 @@ WORKER_MACHINE_TYPE="${WORKER_MACHINE_TYPE:-n1-highmem-4}" DISK_SIZE_GB="${DISK_SIZE_GB:-500}" MAX_NUM_WORKERS="${MAX_NUM_WORKERS:-10}" USE_PUBLIC_IPS="${USE_PUBLIC_IPS:-false}" -ENABLE_SNAPPY="${ENABLE_SNAPPY:-true}" + +# Deprecated: Snappy compressed snapshots are read natively, this variable is ignored. +if [ -n "${ENABLE_SNAPPY:-}" ]; then + echo "⚠️ Warning: ENABLE_SNAPPY is deprecated and ignored. Snappy compressed snapshots are supported natively." +fi # Generate a safe, unique job name prefix to prevent collisions SAFE_TABLE_NAME=$(echo "${TABLE_NAME}" | tr '[:upper:]' '[:lower:]' | tr '_' '-' | tr -cd '[:alnum:]-') @@ -284,7 +288,6 @@ for (( i=START_SHARD; i<=END_SHARD; i++ )); do --region="${REGION}" \ "${SERVICE_ACCOUNT_ARGS[@]}" \ --usePublicIps="${USE_PUBLIC_IPS}" \ - --enableSnappy="${ENABLE_SNAPPY}" \ --skipRestoreStep="${SKIP_RESTORE}" \ --deleteRestoredSnapshots=false \ --restorePath="${RESTORE_DIR}" \ diff --git a/bigtable-dataflow-parent/bigtable-beam-import/src/main/java/com/google/cloud/bigtable/beam/hbasesnapshots/ImportJobFromHbaseSnapshot.java b/bigtable-dataflow-parent/bigtable-beam-import/src/main/java/com/google/cloud/bigtable/beam/hbasesnapshots/ImportJobFromHbaseSnapshot.java index 40747862b7..aad6a1568f 100644 --- a/bigtable-dataflow-parent/bigtable-beam-import/src/main/java/com/google/cloud/bigtable/beam/hbasesnapshots/ImportJobFromHbaseSnapshot.java +++ b/bigtable-dataflow-parent/bigtable-beam-import/src/main/java/com/google/cloud/bigtable/beam/hbasesnapshots/ImportJobFromHbaseSnapshot.java @@ -120,10 +120,23 @@ public interface ImportOptions extends ImportJob.ImportOptions { @SuppressWarnings("unused") void setSnapshotName(String snapshotName); - @Description("Is importing Snappy compressed snapshot.") + /** + * @deprecated This option is a no-op since the job moved to HBase 2.x / Hadoop 3. Snappy + * compressed snapshots are read natively, so the flag no longer needs to be set and is + * ignored. It will be removed in a future release. + */ + @Deprecated + @Description( + "Deprecated: this option is ignored. Snappy compressed snapshots are supported natively" + + " and no longer require a flag.") @Default.Boolean(false) Boolean getEnableSnappy(); + /** + * @deprecated This option is a no-op since the job moved to HBase 2.x / Hadoop 3. See {@link + * #getEnableSnappy()}. + */ + @Deprecated @SuppressWarnings("unused") void setEnableSnappy(Boolean enableSnappy); @@ -260,6 +273,8 @@ public static void main(String[] args) throws Exception { // To determine the Google Cloud Storage file scheme (gs://) FileSystems.setDefaultPipelineOptions(options); + warnIfSnappyFlagSet(options); + LOG.info("Building Pipeline"); Pipeline pipeline = null; ImportConfig importConfig = null; @@ -286,6 +301,16 @@ public static void main(String[] args) throws Exception { } } + @SuppressWarnings("deprecation") + private static void warnIfSnappyFlagSet(ImportOptions options) { + if (Boolean.TRUE.equals(options.getEnableSnappy())) { + LOG.warn( + "--enableSnappy is deprecated and ignored. Snappy compressed snapshots are read natively" + + " since the job moved to HBase 2.x / Hadoop 3; the flag will be removed in a future" + + " release."); + } + } + @VisibleForTesting static ImportConfig buildImportConfigFromConfigFile(String configFilePath) throws Exception { Gson gson = new GsonBuilder().create(); diff --git a/bigtable-dataflow-parent/bigtable-beam-import/src/test/java/com/google/cloud/bigtable/beam/hbasesnapshots/EndToEndIT.java b/bigtable-dataflow-parent/bigtable-beam-import/src/test/java/com/google/cloud/bigtable/beam/hbasesnapshots/EndToEndIT.java index acbbd1edf2..2295a7d247 100644 --- a/bigtable-dataflow-parent/bigtable-beam-import/src/test/java/com/google/cloud/bigtable/beam/hbasesnapshots/EndToEndIT.java +++ b/bigtable-dataflow-parent/bigtable-beam-import/src/test/java/com/google/cloud/bigtable/beam/hbasesnapshots/EndToEndIT.java @@ -419,8 +419,8 @@ public void testHBaseSnapshotImportWithCorruptions() throws Exception { @Test public void testSnappyCompressedHBaseSnapshotImport() throws Exception { // Start import + // Snappy compressed snapshots are read natively, no flag needed. ImportOptions importOpts = createImportOptions(); - importOpts.setEnableSnappy(true); importOpts.setSnapshotName(TEST_SNAPPY_SNAPSHOT_NAME); // run pipeline