Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 9 additions & 26 deletions bigtable-dataflow-parent/bigtable-beam-import/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -167,32 +167,15 @@ workers accordingly.

### Snappy compressed Snapshots

1. Set the environment variables.
```
PROJECT_ID=your-project-id
INSTANCE_ID=your-instance-id
TABLE_NAME=your-table-name
REGION=us-central1

SNAPSHOT_GCS_PATH="$BUCKET_NAME/hbase-migration-snap"
SNAPSHOT_NAME=your-snapshot-name
```

1. Run the import.
```
java -jar bigtable-beam-import-2.3.0.jar importsnapshot \
--runner=DataflowRunner \
--project=$PROJECT_ID \
--bigtableInstanceId=$INSTANCE_ID \
--bigtableTableId=$TABLE_NAME \
--hbaseSnapshotSourceDir=$SNAPSHOT_GCS_PATH/data \
--snapshotName=$SNAPSHOT_NAME \
--stagingLocation=$SNAPSHOT_GCS_PATH/staging \
--gcpTempLocation=$SNAPSHOT_GCS_PATH/temp \
--maxNumWorkers=$(expr 3 \* $CLUSTER_NUM_NODES) \
--region=$REGION \
--enableSnappy=true
```
Snappy compressed snapshots are read natively, so they are imported with the
exact same command as above, with no extra configuration.

> [!WARNING]
> The `--enableSnappy` flag is **deprecated**. Since
> [#4338](https://github.com/googleapis/java-bigtable-hbase/pull/4338) (released
> in 2.14.0) the import job runs on HBase 2.x / Hadoop 3, which decompresses
> Snappy without the custom worker container the flag used to install. The flag
> is accepted but ignored, and it will be removed in a future release.

### Sequence Files

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ The script relies on the following environment variables. You should set them be
| `DISK_SIZE_GB` | *Optional* | Worker disk size in Gigabytes. | `500` (Default) |
| `MAX_NUM_WORKERS` | *Optional* | Maximum number of active Dataflow workers per job. | `10` (Default) |
| `USE_PUBLIC_IPS` | *Optional* | Whether Dataflow workers should have public IPs. | `false` (Default) |
| `ENABLE_SNAPPY` | *Optional* | Whether to enable Snappy compression for transit files. | `true` (Default) |
| `ENABLE_SNAPPY` | **Deprecated** | Ignored. Snappy compressed snapshots are read natively, so no flag is needed. Setting it only prints a warning. | N/A |

## Understanding Sharding

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,11 @@ WORKER_MACHINE_TYPE="${WORKER_MACHINE_TYPE:-n1-highmem-4}"
DISK_SIZE_GB="${DISK_SIZE_GB:-500}"
MAX_NUM_WORKERS="${MAX_NUM_WORKERS:-10}"
USE_PUBLIC_IPS="${USE_PUBLIC_IPS:-false}"
ENABLE_SNAPPY="${ENABLE_SNAPPY:-true}"

# Deprecated: Snappy compressed snapshots are read natively, this variable is ignored.
if [ -n "${ENABLE_SNAPPY:-}" ]; then
echo "⚠️ Warning: ENABLE_SNAPPY is deprecated and ignored. Snappy compressed snapshots are supported natively."
fi

# Generate a safe, unique job name prefix to prevent collisions
SAFE_TABLE_NAME=$(echo "${TABLE_NAME}" | tr '[:upper:]' '[:lower:]' | tr '_' '-' | tr -cd '[:alnum:]-')
Expand Down Expand Up @@ -284,7 +288,6 @@ for (( i=START_SHARD; i<=END_SHARD; i++ )); do
--region="${REGION}" \
"${SERVICE_ACCOUNT_ARGS[@]}" \
--usePublicIps="${USE_PUBLIC_IPS}" \
--enableSnappy="${ENABLE_SNAPPY}" \
--skipRestoreStep="${SKIP_RESTORE}" \
--deleteRestoredSnapshots=false \
--restorePath="${RESTORE_DIR}" \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,10 +120,23 @@ public interface ImportOptions extends ImportJob.ImportOptions {
@SuppressWarnings("unused")
void setSnapshotName(String snapshotName);

@Description("Is importing Snappy compressed snapshot.")
/**
* @deprecated This option is a no-op since the job moved to HBase 2.x / Hadoop 3. Snappy
* compressed snapshots are read natively, so the flag no longer needs to be set and is
* ignored. It will be removed in a future release.
*/
@Deprecated
@Description(
"Deprecated: this option is ignored. Snappy compressed snapshots are supported natively"
+ " and no longer require a flag.")
@Default.Boolean(false)
Boolean getEnableSnappy();

/**
* @deprecated This option is a no-op since the job moved to HBase 2.x / Hadoop 3. See {@link
* #getEnableSnappy()}.
*/
@Deprecated
@SuppressWarnings("unused")
void setEnableSnappy(Boolean enableSnappy);

Expand Down Expand Up @@ -260,6 +273,8 @@ public static void main(String[] args) throws Exception {
// To determine the Google Cloud Storage file scheme (gs://)
FileSystems.setDefaultPipelineOptions(options);

warnIfSnappyFlagSet(options);

LOG.info("Building Pipeline");
Pipeline pipeline = null;
ImportConfig importConfig = null;
Expand All @@ -286,6 +301,16 @@ public static void main(String[] args) throws Exception {
}
}

@SuppressWarnings("deprecation")
private static void warnIfSnappyFlagSet(ImportOptions options) {
if (Boolean.TRUE.equals(options.getEnableSnappy())) {
LOG.warn(
"--enableSnappy is deprecated and ignored. Snappy compressed snapshots are read natively"
+ " since the job moved to HBase 2.x / Hadoop 3; the flag will be removed in a future"
+ " release.");
}
}

@VisibleForTesting
static ImportConfig buildImportConfigFromConfigFile(String configFilePath) throws Exception {
Gson gson = new GsonBuilder().create();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -419,8 +419,8 @@ public void testHBaseSnapshotImportWithCorruptions() throws Exception {
@Test
public void testSnappyCompressedHBaseSnapshotImport() throws Exception {
// Start import
// Snappy compressed snapshots are read natively, no flag needed.
ImportOptions importOpts = createImportOptions();
importOpts.setEnableSnappy(true);
importOpts.setSnapshotName(TEST_SNAPPY_SNAPSHOT_NAME);

// run pipeline
Expand Down
Loading