From 485571d262b9b26c703fa05b8d83e2fec9c80aa7 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 28 Oct 2026 16:58:36 +0000 Subject: [PATCH 01/21] feat(people-and-planet-ai): modernize wildlife image classification with Gemini and Apache Beam RunInference Problem: The legacy people-and-planet-ai/image-classification sample relied on deprecated AutoML Vision APIs (train_model.py, deploy_model.py, predict.py), pointed to a defunct Azure Blob Storage URL (HTTP 403) for the LILA WCS Camera Traps dataset, required copying gigabytes of camera trap images into a user bucket, excluded rare species with fewer than 50 training images, and required hours of model training plus an always-on endpoint. Solution: - Update create_images_metadata_table.py (region tag dataflow_wildlife_ingest_metadata) to ingest and clean WCS camera trap annotations from LILA Science's public Cloud Storage archive (https://storage.googleapis.com/public-datasets-lila/wcs/wcs_camera_traps.json.zip) into BigQuery. - Add classify_wildlife_image.py (region tag genai_wildlife_classify_image) using Gemini 3.8 Flash (gemini-3.8-flash) on the global endpoint with Pydantic Structured Outputs (WildlifeObservation) and habitat geo-fencing (habitat_region) to classify camera trap images in-place from gs://public-datasets-lila/ into species, taxonomic family, common name, animal count, juvenile presence, and empty/misfire flags. - Add embed_wildlife_image.py (region tag genai_wildlife_embed_image) using Gemini Embedding 2 (gemini-embedding-2) to generate 768-dimensional multimodal image embeddings for 1-shot rare species similarity search with BigQuery VECTOR_SEARCH. - Add run_wildlife_inference.py (region tag dataflow_wildlife_run_inference) using Apache Beam Sample.FixedSizePerKey and RunInference (GeminiWildlifeModelHandler) to run class-balanced batch inference directly against gs://public-datasets-lila/wcs-unzipped/ and write structured observations to BigQuery. - Remove deprecated train_model.py, deploy_model.py, and predict.py. - Update e2e_test.py, requirements.txt, requirements-test.txt, README.ipynb, README.md, and people-and-planet-ai/README.md for DevSite style compliance and end-to-end test coverage. --- people-and-planet-ai/README.md | 20 +- .../image-classification/README.ipynb | 582 ++++++++---------- .../image-classification/README.md | 29 +- .../classify_wildlife_image.py | 136 ++++ .../create_images_metadata_table.py | 168 ++--- .../image-classification/deploy_model.py | 128 ---- .../image-classification/e2e_test.py | 313 ++++------ .../embed_wildlife_image.py | 91 +++ .../image-classification/predict.py | 109 ---- .../requirements-test.txt | 1 - .../image-classification/requirements.txt | 6 +- .../run_wildlife_inference.py | 235 +++++++ .../image-classification/train_model.py | 388 ------------ 13 files changed, 962 insertions(+), 1244 deletions(-) create mode 100644 people-and-planet-ai/image-classification/classify_wildlife_image.py delete mode 100644 people-and-planet-ai/image-classification/deploy_model.py create mode 100644 people-and-planet-ai/image-classification/embed_wildlife_image.py delete mode 100644 people-and-planet-ai/image-classification/predict.py create mode 100644 people-and-planet-ai/image-classification/run_wildlife_inference.py delete mode 100644 people-and-planet-ai/image-classification/train_model.py diff --git a/people-and-planet-ai/README.md b/people-and-planet-ai/README.md index f9a477686b7..6ed48a37096 100644 --- a/people-and-planet-ai/README.md +++ b/people-and-planet-ai/README.md @@ -4,21 +4,23 @@ > [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/GoogleCloudPlatform/python-docs-samples/blob/main/people-and-planet-ai/image-classification/README.ipynb) > -> [Watch the video in YouTube
![thumbnail](http://img.youtube.com/vi/hUzODH3uGg0/0.jpg)](https://youtu.be/hUzODH3uGg0) +> [Watch the video on YouTube ![thumbnail](https://img.youtube.com/vi/hUzODH3uGg0/0.jpg)](https://youtu.be/hUzODH3uGg0) -This model is trained to recognize animal species from +Classify wildlife species and extract ecological recovery metrics from [camera trap](https://en.wikipedia.org/wiki/Camera_trap) -pictures. +pictures in the public [LILA Science](https://lila.science/datasets/wcscameratraps) Cloud Storage bucket (`gs://public-datasets-lila/`). -* **Model**: [AutoML] Vision -* **Creating datasets**: [Apache Beam] in [Dataflow] -* **Training the model**: [AutoML] in [Vertex AI] -* **Getting predictions**: [Vertex AI] +* **Multimodal classification and ecological extraction**: [Gemini] (`gemini-3.8-flash`) via the [Google Gen AI SDK] +* **Rare-species visual similarity and vector search**: [Gemini Embedding 2] (`gemini-embedding-2`) +* **Class-balanced batch inference pipeline**: [Apache Beam] `RunInference` in [Dataflow] +* **Metadata and structured observation storage**: [BigQuery] [Apache Beam]: https://beam.apache.org +[BigQuery]: https://cloud.google.com/bigquery [Dataflow]: https://cloud.google.com/dataflow -[AutoML]: https://cloud.google.com/vertex-ai/docs/beginner/beginners-guide -[Vertex AI]: https://cloud.google.com/vertex-ai +[Gemini]: https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/3-8-flash +[Gemini Embedding 2]: https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/embedding-2 +[Google Gen AI SDK]: https://googleapis.github.io/python-genai/ --- diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index cd141cd8e78..fa445f21e0e 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -20,24 +20,21 @@ "cell_type": "code", "execution_count": null, "source": [ - "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n", - "\n", - "# Licensed to the Apache Software Foundation (ASF) under one\n", - "# or more contributor license agreements. See the NOTICE file\n", - "# distributed with this work for additional information\n", - "# regarding copyright ownership. The ASF licenses this file\n", - "# to you under the Apache License, Version 2.0 (the\n", - "# \"License\"); you may not use this file except in compliance\n", - "# with the License. You may obtain a copy of the License at\n", + "# @title ###### Copyright 2026 Google LLC. Licensed under the Apache License, Version 2.0 (the \"License\")\n", + "\n", + "# Copyright 2026 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", "#\n", - "# Unless required by applicable law or agreed to in writing,\n", - "# software distributed under the License is distributed on an\n", - "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n", - "# KIND, either express or implied. See the License for the\n", - "# specific language governing permissions and limitations\n", - "# under the License." + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." ], "outputs": [], "metadata": { @@ -52,17 +49,16 @@ "\n", "๐Ÿฆœ๐Ÿฆง ๐Ÿ˜๐Ÿฆฉ๐Ÿฆ‹ ๐Ÿฆ‡ ๐Ÿฆ€ ๐Ÿฆ” ๐Ÿ‡ ๐Ÿฆซ ๐Ÿฆ ๐Ÿ† ๐Ÿ— ๐Ÿฆ™ ๐Ÿฆฅ ๐ŸฆŽ ๐Ÿฆƒ\n", "\n", - "- **Time estimate**: 2 hours 15 minutes\n", - "- **Cost estimate**: less than \\$30.00 (_free_ if you use \\$300 [Cloud credits](https://cloud.google.com/free/docs/gcp-free-tier))\n", + "- **Time estimate**: 10 minutes\n", + "- **Cost estimate**: less than \\$1.00 (_free_ if you use \\$300 [Cloud credits](https://cloud.google.com/free/docs/gcp-free-tier))\n", "\n", - "> [Watch the video in YouTube
![thumbnail](http://img.youtube.com/vi/hUzODH3uGg0/0.jpg)](https://youtu.be/hUzODH3uGg0)\n", + "> [Watch the video on YouTube ![thumbnail](https://img.youtube.com/vi/hUzODH3uGg0/0.jpg)](https://youtu.be/hUzODH3uGg0)\n", ">\n", - "> ๐Ÿ“œ Read more about this sample in the [blog post](https://cloud.google.com/blog/topics/developers-practitioners/recovering-global-wildlife-populations-using-ml).\n", + "> ๐Ÿ“œ For more information about the background of this solution, see [Recovering global wildlife populations using ML](https://cloud.google.com/blog/topics/developers-practitioners/recovering-global-wildlife-populations-using-ml).\n", "\n", - "This is an interactive notebook that contains all of the code necessary to train an ML model for image classification.\n", - "This model is trained to recognize animal species from\n", + "Classify animal species and extract ecological recovery signals from\n", "[camera trap](https://en.wikipedia.org/wiki/Camera_trap)\n", - "pictures." + "pictures using **Gemini 3.8 Flash** (`gemini-3.8-flash`), **Gemini Embedding 2** (`gemini-embedding-2`), and **Apache Beam `RunInference`** on **Dataflow**." ], "metadata": { "id": "1LUt29D0MSeu" @@ -71,16 +67,15 @@ { "cell_type": "markdown", "source": [ - "## ๐Ÿ™ˆ Using this interactive notebook\n", + "## ๐Ÿ™ˆ Use this interactive notebook\n", "\n", - "Click the **run** icons โ–ถ๏ธ of each section within this notebook. \n", + "Click the **run** icons โ–ถ๏ธ of each section within this notebook.\n", "\n", - "This notebook code lets you train and deploy an ML model, as well as test its accuracy, from end-to-end. When you run a code cell, the code runs in the notebook's runtime, so you're not making any changes to your personal computer.\n", + "Ingest camera trap metadata into BigQuery, run class-balanced multimodal inference with Apache Beam and Gemini, and explore zero-shot classification, habitat geo-fencing, and multimodal image embeddings end-to-end. When you run a code cell, the code runs in the notebook runtime, so you don't make any changes to your personal computer.\n", "\n", - "> ๐Ÿ›Ž๏ธ **To avoid any errors**, wait for each section to finish in their order before clicking the next โ€œrunโ€ icon.\n", + "> ๐Ÿ›Ž๏ธ **To avoid errors**, wait for each section to finish in order before clicking the next **run** icon.\n", "\n", - "This sample must be connected to a Google Cloud project, but nothing else is needed other than your Google Cloud project.\n", - "You can use an existing project and the cost will be less than \\$30. Alternatively, you can create a new Cloud project with cloud credits for free" + "This notebook requires a Google Cloud project. You can use an existing project (cost is less than \\$1.00 with zero always-on endpoint charges) or create a new Google Cloud project with free trial credits." ], "metadata": { "id": "jXDRG8QK2BZD" @@ -91,34 +86,25 @@ "source": [ "## ๐Ÿšดโ€โ™€๏ธ Steps included\n", "\n", - "The notebook leverages camera trap images dataset from the\n", - "[WCS Camera Traps dataset](http://lila.science/datasets/wcscameratraps)\n", - "from the _Labeled Information Library of Alexandria: Biology and Conservation_ (LILA) organization.\n", - " \n", - "Here's a quick summary of what youโ€™ll go through:\n", + "The notebook uses public camera trap datasets hosted in Cloud Storage (`gs://public-datasets-lila/`) by the\n", + "[Labeled Information Library of Alexandria: Biology and Conservation (LILA)](https://lila.science/datasets/wcscameratraps),\n", + "including the **Wildlife Conservation Society (WCS) Camera Traps** dataset and the **Felidae Conservation Fund** San Francisco Bay Area dataset.\n", "\n", - "1. **Create a database of image metadata** _(~5 minutes to complete, costs a few cents)_:\n", - " in a\n", - " [BigQuery](https://cloud.google.com/bigquery)\n", - " table with all the image file names along with their respective category. \n", + "In this notebook, you complete the following steps:\n", "\n", - "1. **Train an image classifier** _(~2 hours to complete, costs ~$25.00)_:\n", - " With some desired criteria, a\n", - " [Dataflow](https://cloud.google.com/dataflow)\n", - " pipeline creates a balanced dataset from the metadata database.\n", - " It then physically downloads the necessary images from LILA into\n", - " [Cloud Storage](https://cloud.google.com/storage),\n", - " imports the data into\n", - " [AI Platform Unified](https://cloud.google.com/ai-platform-unified/docs/start/introduction-unified-platform),\n", - " and starts the model training job.\n", + "1. **Create a table of image metadata** _(~5 minutes to complete, costs a few cents)_:\n", + " Run [`create_images_metadata_table.py`](create_images_metadata_table.py) on [Dataflow](https://cloud.google.com/dataflow) to ingest and clean the WCS Camera Traps metadata JSON archive directly from `gs://public-datasets-lila/wcs/wcs_camera_traps.json.zip` into a [BigQuery](https://cloud.google.com/bigquery) table.\n", "\n", - "1. **Deploy an ML model** _(costs **$1.25 for every hour** the model is deployed onlineโ€ฆ so complete the โ€œdelete modelโ€ step soon)_:\n", - " After the model finishes training, we look at the results and deploy it into a Cloud endpoint in AI Platform Unified.\n", + "2. **Run class-balanced batch inference in Dataflow** _(~5 minutes to complete, costs a few cents)_:\n", + " Run [`run_wildlife_inference.py`](run_wildlife_inference.py) using Apache Beam's `Sample.FixedSizePerKey` and `RunInference` with **Gemini 3.8 Flash** (`gemini-3.8-flash`). Unlike legacy custom model trainingโ€”which took hours, required copying images into your own bucket, and discarded rare species with fewer than 50 imagesโ€”Gemini classifies images **in-place** from `gs://public-datasets-lila/wcs-unzipped/` and writes structured ecological observations (`species_scientific_name`, `taxon_family`, `common_name`, `animal_count`, `juvenile_present`, `is_empty_or_misfire`, `confidence`) to BigQuery.\n", "\n", - "1. **Classify images with the model**:\n", - " We send some images to the model and get back the model's classification predictions.\n", + "3. **Classify and geo-fence camera trap images interactively**:\n", + " Use [`classify_wildlife_image.py`](classify_wildlife_image.py) to inspect camera trap photos, see how taxonomic family (`taxon_family`) resolves look-alike species, and pass `habitat_region` context to geo-fence predictions (for example, San Francisco Bay Area oak woodlands in `gs://public-datasets-lila/felidae-conservation-fund/`).\n", "\n", - "1. (Optional) **Delete the project** to avoid ongoing costs." + "4. **Generate multimodal embeddings for rare-species search**:\n", + " Use [`embed_wildlife_image.py`](embed_wildlife_image.py) and **Gemini Embedding 2** (`gemini-embedding-2`) to turn a single reference photo of an endangered species into a 768-dimensional vector embedding for similarity search in BigQuery.\n", + "\n", + "5. **Optional: Clean up resources** to avoid ongoing storage costs." ], "metadata": { "id": "Kb9doxCpUcpF" @@ -127,33 +113,31 @@ { "cell_type": "markdown", "source": [ - "## โœจ Before you begin, you need toโ€ฆ\n", + "## โœจ Before you begin\n", "\n", - "1. Decide on creating a new\n", - " [free project](https://cloud.google.com/free/docs/gcp-free-tier)\n", - " _(recommended)_ or using an existing one.\n", - " Then copy the project ID and paste it in the `google_cloud_project` field in the \"Entering project detailsโ€ section below.\n", + "1. Create a new\n", + " [Google Cloud project](https://cloud.google.com/free/docs/gcp-free-tier)\n", + " _(recommended)_ or select an existing project.\n", + " Copy the project ID and paste it in the `google_cloud_project` field in the \"Enter project details\" section.\n", "\n", - " > ๐Ÿ’ก If you _don't plan to keep the resources_ that you create via this sample, we recommend creating a new project instead of selecting an existing project.\n", - " > After you finish these steps, you can delete the project, removing all the resources associated in bulk.\n", + " > ๐Ÿ’ก If you don't plan to keep the resources that you create in this notebook, Google recommends creating a new project instead of selecting an existing project.\n", + " > After you finish these steps, you can delete the project to remove all associated resources.\n", "\n", - "1. [_Click here_](https://console.cloud.google.com/flows/enableapi?apiid=dataflow,aiplatform.googleapis.com)\n", - " to **enable the following APIs** in your Google Cloud project:\n", - " _Dataflow_ and _AI Platform_.\n", + "1. [Enable the Dataflow, Gemini Enterprise Agent Platform, and BigQuery APIs](https://console.cloud.google.com/flows/enableapi?apiid=dataflow,aiplatform.googleapis.com,bigquery.googleapis.com)\n", + " in your Google Cloud project.\n", "\n", - "1. Make sure that **billing is enabled** for your Google Cloud project,\n", - " [_click here_](https://cloud.google.com/billing/docs/how-to/modify-project)\n", - " to learn how to confirm that billing is enabled.\n", + "1. Make sure that billing is enabled for your Google Cloud project. For more information, see\n", + " [Verify the billing status of your projects](https://cloud.google.com/billing/docs/how-to/modify-project).\n", "\n", - "1. [_Click here_](https://console.cloud.google.com/storage/create-bucket)\n", - " to create a Cloud Storage bucket.\n", - " Then copy the bucketโ€™s name and paste it in the `cloud_storage_bucket` field in the โ€œEntering project detailsโ€ section below.\n", + "1. [Create a Cloud Storage bucket](https://console.cloud.google.com/storage/create-bucket)\n", + " for Dataflow temporary staging files.\n", + " Copy the bucket name and paste it in the `cloud_storage_bucket` field in the \"Enter project details\" section.\n", "\n", - " > ๐Ÿ›Ž๏ธ Make sure it's a _regional_ bucket in a location where\n", - " > [AI Platform is available](https://cloud.google.com/ai-platform-unified/docs/general/locations#available_regions).\n", + " > ๐Ÿ›Ž๏ธ Create a _regional_ bucket in a location where\n", + " > [Gemini Enterprise Agent Platform and Dataflow are available](https://docs.cloud.google.com/gemini-enterprise-agent-platform/resources/locations).\n", "\n", "1. [Create a BigQuery dataset](https://cloud.google.com/bigquery/docs/datasets#create-dataset).\n", - " Then copy the dataset ID and paste it in the `bigquery_dataset` field in the โ€œEntering project detailsโ€ section below.\n", + " Copy the dataset ID and paste it in the `bigquery_dataset` field in the \"Enter project details\" section.\n", "\n", " > ๐Ÿ’ก You can use the name `samples` for your dataset." ], @@ -164,12 +148,12 @@ { "cell_type": "markdown", "source": [ - "### โ›๏ธ Preparing the project environment\n", + "### โ›๏ธ Prepare the project environment\n", "\n", - "Click run โ–ถ๏ธ for the following cells to download and install the necessary libraries and resources for this solution.\n", + "Click **run** โ–ถ๏ธ on the following cells to download and install the required libraries and resources.\n", "\n", - "> ๐Ÿ’ก You can _optionally_ view the entire\n", - "> [code in GitHub](https://github.com/GoogleCloudPlatform/python-docs-samples/tree/main/people-and-planet-ai/image-classification)." + "> ๐Ÿ’ก You can also view the complete\n", + "> [sample code on GitHub](https://github.com/GoogleCloudPlatform/python-docs-samples/tree/main/people-and-planet-ai/image-classification)." ], "metadata": { "id": "5S5_eLKE-M2o" @@ -179,10 +163,10 @@ "cell_type": "code", "execution_count": null, "source": [ - "# We need libffi-dev to launch the Dataflow pipeline.\n", + "# Install libffi-dev for the Dataflow pipeline.\n", "!apt-get -qq install libffi-dev\n", "\n", - "# Clone the python-docs-samples respository.\n", + "# Clone the python-docs-samples repository.\n", "!git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git\n", "\n", "# Navigate to the sample code directory.\n", @@ -200,13 +184,11 @@ { "cell_type": "markdown", "source": [ - "> ๐Ÿ›Ž๏ธ **\\[DONโ€™T PANIC\\]** Itโ€™s safe to _ignore the warnings_.\n", - "> When we `pip install` the requirements, there might be some warnings about conflicting dependency versions.\n", - "> For the scope of this sample, thatโ€™s ok.\n", + "> ๐Ÿ›Ž๏ธ **Note**: You can ignore dependency version warnings printed by `pip install` in the notebook environment.\n", "\n", - "> โš ๏ธ **Restart the runtime**: Running the previous cell just updated some libraries and requires to restart the runtime to load those libraries correctly.\n", + "> โš ๏ธ **Restart the runtime**: After installing the libraries in the previous cell, restart the runtime so the updated packages load properly.\n", ">\n", - "> In the top-left menu, click _**\"Runtime\" > \"Restart runtime\"**_." + "> In the top menu, click **Runtime** > **Restart session** (or **Restart runtime**)." ], "metadata": { "id": "j_Us7ESa90WN" @@ -227,9 +209,9 @@ { "cell_type": "markdown", "source": [ - "### โœ๏ธ Entering project details\n", + "### โœ๏ธ Enter project details\n", "\n", - "Fill these in with the Google Cloud project resources you just created." + "Enter your Google Cloud project ID, Cloud Storage bucket name, and BigQuery dataset ID in the fields below." ], "metadata": { "id": "cipdLUp92BZF" @@ -245,9 +227,9 @@ "\n", "# Validate inputs.\n", "if not google_cloud_project:\n", - " raise ValueError(\"Please provide your google_cloud_project\")\n", + " raise ValueError(\"Set google_cloud_project to your Google Cloud project ID.\")\n", "if not cloud_storage_bucket:\n", - " raise ValueError(\"Please provide your cloud_storage_bucket\")\n", + " raise ValueError(\"Set cloud_storage_bucket to your Cloud Storage bucket name.\")\n", "\n", "# Authenticate to use the Google Cloud resources.\n", "try:\n", @@ -258,9 +240,10 @@ "except ModuleNotFoundError:\n", " import os\n", "\n", - " if os.environ.get(\"GOOGLE_APPLICATION_CREDENTIALS\") is None:\n", + " service_account_file = os.environ.get(\"GOOGLE_APPLICATION_CREDENTIALS\")\n", + " if service_account_file is None:\n", " raise ValueError(\n", - " \"Please set your GOOGLE_APPLICATION_CREDENTIALS environment variable to your service account JSON file path.\"\n", + " \"Set the GOOGLE_APPLICATION_CREDENTIALS environment variable to your service account JSON file path.\"\n", " )\n", " print(f\"GOOGLE_APPLICATION_CREDENTIALS: {service_account_file}\")\n", "\n", @@ -275,10 +258,10 @@ { "cell_type": "markdown", "source": [ - "1. Click the run button โ–ถ๏ธ for the cells above.\n", - "1. Paste the \"verification code\" that is presented, and press **\\[ENTER\\]** to authenticate.\n", + "1. Click **run** โ–ถ๏ธ on the cell above.\n", + "1. Follow the authentication prompt to sign in to your Google Cloud account.\n", "\n", - "Run the following cell, you can leave them with these values or change them if you prefer." + "Run the following cell to set the default configuration values, or modify them before running." ], "metadata": { "id": "sWFY2a_P2BZF" @@ -290,7 +273,7 @@ "source": [ "cloud_storage_directory = \"samples/wildlife-insights\" # @param {type:\"string\"}\n", "bigquery_table = \"wildlife_images_metadata\" # @param {type:\"string\"}\n", - "ai_platform_name_prefix = \"wildlife_classifier\" # @param {type:\"string\"}\n", + "predictions_table = \"wildlife_predictions\" # @param {type:\"string\"}\n", "region = \"us-central1\" # @param {type:\"string\"}" ], "outputs": [], @@ -311,20 +294,19 @@ { "cell_type": "markdown", "source": [ - "## ๐Ÿป Creating the database of images metadata\n", + "## ๐Ÿป Create the image metadata table\n", + "\n", + "Before running batch wildlife inference, create a BigQuery metadata table containing the species annotations and image file paths from the WCS Camera Traps archive. This is a _one-time_ setup step.\n", "\n", - "To build our model we need data, so we create a database table from the metadata of all the images.\n", - "This is a _one-time only_ process.\n", + "Click **run** โ–ถ๏ธ on the following cell to launch the Dataflow pipeline, which extracts two fields for each camera trap image from the WCS Camera Traps JSON archive:\n", "\n", - "Click run on the first Dataflow pipeline to create the images metadata table in BigQuery, which contains 2 fields of information from each image from the JSON file in WCS Camera Traps database:\n", + "- `category`: The ground-truth species label.\n", + "- `file_name`: The relative file path of the image in `gs://public-datasets-lila/wcs-unzipped/`.\n", "\n", - " - `category`: The _species_ we want to predict, this is our _label_.\n", - " - `file_name`: The _file path_ where the image file is located.\n", - " \n", - "We do some very basic _data cleaning_ like discarding rows with categories that are note useful like `#ref!`, `empty`, `unidentifiable`, `unidentified`, `unknown`.\n", + "The pipeline filters out rows with categories that are not useful species labels, such as `#ref!`, `empty`, `unidentifiable`, `unidentified`, and `unknown`.\n", "\n", - "> ๐Ÿ’ก As you collect new images from the field via your own camera traps, you continue to store the new imageโ€™s category and file name into the BigQuery table.\n", - "> You could extend this sample to collect more pertinent data for reporting purposes such as location, date, etc." + "> ๐Ÿ’ก As you collect new images from the field from your own camera traps, you can insert the new image categories and file paths into the BigQuery table.\n", + "> You can also extend this table schema to store additional ecological metadata, such as camera location and capture timestamp." ], "metadata": { "id": "GHs2UDDnT1YG" @@ -360,11 +342,9 @@ "\n", "\n", "\n", - "> ๐Ÿ›Ž๏ธ We need at least `n1-standard-2` [worker machines](https://cloud.google.com/compute/docs/machine-types#n1_machine_types) due to large RAM usage when parsing the metadata JSON file.\n", + "> ๐Ÿ›Ž๏ธ The pipeline uses the `n1-standard-2` [machine type](https://cloud.google.com/compute/docs/machine-types#n1_machine_types) to provide sufficient memory when parsing the full WCS metadata JSON archive.\n", "\n", - "You can look up the job details in the Dataflow jobs page:\n", - "\n", - "- [console.cloud.google.com/dataflow/jobs](https://console.cloud.google.com/dataflow/jobs)" + "You can view the job progress on the [Dataflow Jobs page in the Google Cloud console](https://console.cloud.google.com/dataflow/jobs)." ], "metadata": { "id": "p8rDiRzZcWvK" @@ -373,49 +353,26 @@ { "cell_type": "markdown", "source": [ - "# ๐Ÿ‘Ÿ Training the image classifier\n", + "## ๐Ÿ‘Ÿ Run class-balanced wildlife inference with Apache Beam and Gemini\n", "\n", - "One of the challenges of a camera trap dataset is that it usually contains a _different number of images per species_, making it very _unbalanced_.\n", + "One of the classic challenges of a camera trap dataset is that it contains a _heavily skewed distribution of images per species_.\n", "\n", - "For example, there are tens of thousands of pictures of some species like [tayassu pecari](https://www.google.com/search?q=tayassu+pecari&tbm=isch), while only a handful of pictures of other species like [tolypeutes matacus](https://www.google.com/search?q=tolypeutes+matacus&tbm=isch).\n", + "For example, there are tens of thousands of pictures of common species like [tayassu pecari](https://www.google.com/search?q=tayassu+pecari&tbm=isch) (white-lipped peccary), while only a handful of pictures of endangered or elusive species like [tolypeutes matacus](https://www.google.com/search?q=tolypeutes+matacus&tbm=isch) (southern three-banded armadillo) or [dicerorhinus sumatrensis](https://www.google.com/search?q=dicerorhinus+sumatrensis&tbm=isch) (Sumatran rhinoceros).\n", "\n", - "In _emojis_, the data looks _skewed_ like this: \n", + "In _emojis_, the data looks _skewed_ like this:\n", "\n", "- ๐Ÿฆœ๐Ÿฆœ๐Ÿฆœ๐Ÿฆœ๐Ÿฆœ (5 Green Macaws)\n", "- ๐Ÿฆง (1 Bornean orangutan)\n", - "- ๐Ÿ˜๐Ÿ˜๐Ÿ˜ (3 Sumatran elephant)\n", + "- ๐Ÿ˜๐Ÿ˜๐Ÿ˜ (3 Sumatran elephants)\n", "- ๐Ÿ—๐Ÿ—๐Ÿ—๐Ÿ—๐Ÿ—๐Ÿ—๐Ÿ—๐Ÿ—๐Ÿ—๐Ÿ—๐Ÿ—๐Ÿ— (12 wild boars)\n", "\n", - "This could introduce a\n", - "[_bias_](https://developers.google.com/machine-learning/crash-course/fairness/types-of-bias)\n", - "into your model which can lead to it predicting a species just because it's more common rather than identifying it by itโ€™s actual features.\n", - "So instead of using the entire database of images, we leverage Dataflow to create a balanced dataset by specifying the _minimum_ and _maximum_ number of images we want for every category.\n", - "For categories with too many images, it chooses at most `max_images_per_class` randomly selected images.\n", - "And categories with less than `min_images_per_class` are discarded as _\"too little information to teach our model to classify this species\"_.\n", - "\n", - "> ๐Ÿ’ก For this sample, we defaulted to using _between 50 and 100 images_ per species/class to keep the training dataset small. This reduces the training time and the _potential cost_ of prediction accuracy.\n", - "> Feel free to play around with other numbers.\n", - "\n", - "Once Dataflow chooses what images meet our criteria by counting the number of species per categories in BigQuery, it\n", - "[lazily](https://en.wikipedia.org/wiki/Lazy_evaluation)\n", - "downloads the actual JPEG images into Cloud Storage from the Lila Science database using itโ€™s file name.\n", - "This means we are only preprocessing the images we need for training rather than processing the entire dataset of JPEG images.\n", - " \n", - "When retraining in the future, if an image already exists in Cloud Storage, there's nothing else to do for that image, as we have created a _cache_ for images that have already been processed in our prior run.\n", - "\n", - "After all the images are stored in Cloud Storage, Dataflow creates a\n", - "[CSV file for AI Platform](https://cloud.google.com/ai-platform-unified/docs/datasets/prepare-image#csv).\n", - "Each row includes the Cloud Storage path of an image, alongside with the category, our label.\n", - "\n", - "Next, Dataflow tells AI Platform to import the CSV file and create a dataset from it. This is where the Dataflow job stops and AI Platform begins to train the model.\n", - " \n", - "We let AI Platform do the _data splitting_ automatically for us. By default, it uses _80% of the data for training_, _10% for validation_, and _10% for testing_. See [About data splits for AI Platform models](https://cloud.google.com/ai-platform-unified/docs/general/ml-use) for more information.\n", - "\n", - "> ๐Ÿ›Ž๏ธ For simplicity, in this sample we are training a `CLOUD` model.\n", - "> This allows us to deploy it to an HTTP endpoint and get predictions via the browser.\n", - ">\n", - "> See [Train an Edge model](https://cloud.google.com/ai-platform-unified/docs/training/automl-edge-console)\n", - "> for information on how to train a model for a microcontroller." + "In traditional supervised ML (such as legacy AutoML Vision), species with fewer than `min_images_per_class = 50` had to be **discarded** because there was not enough training dataโ€”meaning the rarest, most endangered animals were excluded from the model.\n", + "\n", + "With **Gemini 3.8 Flash** (`gemini-3.8-flash`) inside Apache Beam's `RunInference` transform:\n", + "1. **No rare species are left behind**: You can set `min_images_per_class = 1` while still using `Sample.FixedSizePerKey(max_images_per_class)` so common species don't dominate your batch inference budget.\n", + "2. **Zero image duplication**: Instead of downloading and copying gigabytes of JPEGs into your own bucket, Gemini reads `gs://public-datasets-lila/wcs-unzipped/{file_name}` directly in place.\n", + "3. **Richer ecological recovery signals in one pass**: Using Structured Outputs (`WildlifeObservation`), Gemini returns not just `species_scientific_name` and `confidence`, but also `taxon_family`, `common_name`, `animal_count`, `juvenile_present` (to track breeding and population recovery), and `is_empty_or_misfire`.\n", + "4. **Zero idle endpoint cost**: Because `RunInference` calls the serverless Gemini API on Gemini Enterprise Agent Platform directly from Dataflow workers, there is no always-on model endpoint to deploy or pay for by the hour." ], "metadata": { "id": "I9sD-nSOmbK6" @@ -425,8 +382,8 @@ "cell_type": "code", "execution_count": null, "source": [ - "min_images_per_class = 50 # @param {type:\"integer\"}\n", - "max_images_per_class = 100 # @param {type:\"integer\"}" + "min_images_per_class = 1 # @param {type:\"integer\"}\n", + "max_images_per_class = 5 # @param {type:\"integer\"}" ], "outputs": [], "metadata": { @@ -438,17 +395,16 @@ "cell_type": "code", "execution_count": null, "source": [ - "# Create a balanced dataset and signal AI Platform to train a model.\n", - "!python train_model.py \\\n", - " --cloud-storage-path \"gs://{cloud_storage_bucket}/{cloud_storage_directory}\" \\\n", + "# Run class-balanced sampling and Gemini 3.8 Flash RunInference into BigQuery.\n", + "!python run_wildlife_inference.py \\\n", + " --project \"{google_cloud_project}\" \\\n", " --bigquery-dataset \"{bigquery_dataset}\" \\\n", " --bigquery-table \"{bigquery_table}\" \\\n", - " --ai-platform-name-prefix \"{ai_platform_name_prefix}\" \\\n", + " --predictions-table \"{predictions_table}\" \\\n", " --min-images-per-class \"{min_images_per_class}\" \\\n", " --max-images-per-class \"{max_images_per_class}\" \\\n", " --runner \"DataflowRunner\" \\\n", - " --job_name \"wildlife-train-model-`date +%Y%m%d-%H%M%S`\" \\\n", - " --project \"{google_cloud_project}\" \\\n", + " --job_name \"wildlife-run-inference-`date +%Y%m%d-%H%M%S`\" \\\n", " --temp_location \"gs://{cloud_storage_bucket}/{cloud_storage_directory}/temp\" \\\n", " --requirements_file \"requirements.txt\" \\\n", " --region \"{region}\"" @@ -464,21 +420,11 @@ "\n", "\n", - "> ๐Ÿ›Ž๏ธ It can take several minutes for the job to show up in the\n", - "> [Dataflow jobs page](https://console.cloud.google.com/dataflow/jobs).\n", - "> See [[ARROW-8983]](https://issues.apache.org/jira/browse/ARROW-8983) for more information.\n", - "\n", - "Training the AI Platform model can take a while, depending on the dataset size and the training budget you allow.\n", - "\n", - "> ๐Ÿ’ก You can adjust the training budget using the `--budget-milli-node-hours` flag. We default to `8000` which is the minimum.\n", - ">\n", - "> See [the pricing page](https://cloud.google.com/vision/automl/pricing) and\n", - "> [`train_budget_milli_node_hours`](https://cloud.google.com/automl/docs/reference/rpc/google.cloud.automl.v1#imageclassificationmodelmetadata)\n", - "> for more information." + "You can monitor the pipeline progress in the [Dataflow jobs page](https://console.cloud.google.com/dataflow/jobs) and query the structured predictions table in [BigQuery](https://console.cloud.google.com/bigquery)." ], "metadata": { "id": "h_xtp4SrwLZN" @@ -487,20 +433,11 @@ { "cell_type": "markdown", "source": [ - "# ๐Ÿฃ Deploying the model\n", - "\n", - "> ๐Ÿ›Ž Make sure your model has finished training.\n", - "> You can look at the training progress on the\n", - "> [AI Platform training jobs page](https://console.cloud.google.com/ai/platform/training/training-pipelines).\n", - "\n", - "> ๐Ÿ’ก If you were disconnected from the session due to inactivity, please make sure to re-run the cells at the _\"Before you begin\"_ section at the beginning of the notebook.\n", - "\n", - "We need to deploy the trained model into an endpoint to get predictions from it.\n", - "Since we don't have a micro controller readily available, letโ€™s deploy it in the cloud as an HTTP endpoint.\n", + "## ๐Ÿ“บ Visualize camera trap classification with Gemini 3.8 Flash\n", "\n", - "You can deploy it through the Cloud Console: [console.cloud.google.com/ai/platform/models](https://console.cloud.google.com/ai/platform/models)\n", + "Because Gemini 3.8 Flash is a pre-trained multimodal foundation model on Gemini Enterprise Agent Platform (`location=\"global\"`), you don't need to wait hours to train or deploy a custom endpoint before classifying camera trap images.\n", "\n", - "Or alternatively, you can deploy it through the API by running the following cells:" + "Define helper functions to render images directly from the public LILA Science Cloud Storage bucket (`https://storage.googleapis.com/public-datasets-lila/wcs-unzipped`) and invoke [`classify_wildlife_image.py`](classify_wildlife_image.py):" ], "metadata": { "id": "WgHVM2Nwxw9r" @@ -510,17 +447,34 @@ "cell_type": "code", "execution_count": null, "source": [ - "# First we need the model path, we can get it with gcloud.\n", - "# ๐Ÿ’ก If you get an error here, please make sure your model has finished training.\n", - "cmd_output = !gcloud beta ai models list \\\n", - " --project {google_cloud_project} \\\n", - " --region {region} \\\n", - " --filter \"display_name:{ai_platform_name_prefix}*\" \\\n", - " --format \"table[no-heading](display_name,name)\" 2>/dev/null\n", - "models = sorted([line.split() for line in cmd_output])\n", - "model_path = models[0][1]\n", + "import io\n", + "import requests\n", + "from PIL import Image\n", + "from IPython.display import display\n", "\n", - "print(f\"model_path: {model_path}\")" + "import classify_wildlife_image\n", + "\n", + "LILA_WCS_HTTP_BASE = \"https://storage.googleapis.com/public-datasets-lila/wcs-unzipped\"\n", + "LILA_WCS_GCS_BASE = \"gs://public-datasets-lila/wcs-unzipped\"\n", + "\n", + "\n", + "def display_image(image_file: str, base_url: str = LILA_WCS_HTTP_BASE, width: int = 400) -> None:\n", + " response = requests.get(f\"{base_url}/{image_file}\", timeout=60)\n", + " response.raise_for_status()\n", + " image = Image.open(io.BytesIO(response.content))\n", + " display(image.resize((int(width), int(width / image.size[0] * image.size[1]))))\n", + "\n", + "\n", + "def predict(\n", + " image_file: str,\n", + " habitat_region: str = \"Southeast Asia tropical rainforest\",\n", + ") -> None:\n", + " display_image(image_file)\n", + " classify_wildlife_image.classify_wildlife_image(\n", + " project_id=google_cloud_project,\n", + " image_gcs_uri=f\"{LILA_WCS_GCS_BASE}/{image_file}\",\n", + " habitat_region=habitat_region,\n", + " )" ], "outputs": [], "metadata": { @@ -528,86 +482,54 @@ } }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "source": [ - "# Create an endpoint and deploy the model to it.\n", - "!python deploy_model.py \\\n", - " --project {google_cloud_project} \\\n", - " --region {region} \\\n", - " --model-path {model_path} \\\n", - " --model-endpoint-name {ai_platform_name_prefix}" + "\n", + "\n", + "Now, run the following cells to see how `gemini-3.8-flash` classifies each camera trap photo into structured ecological metadata (`Species`, `Taxon Family`, `Common Name`, `Animal Count`, `Juvenile Present`, `Empty or Misfire`, and `Confidence`)." ], - "outputs": [], "metadata": { "id": "lRRg39FcYILC" } }, { - "cell_type": "markdown", + "cell_type": "code", "source": [ - "" + "# Species: dicerorhinus sumatrensis (Sumatran rhinoceros)\n", + "predict(\"animals/0325/1529.jpg\")" ], "metadata": { "id": "VQPz8h9z2BZK" - } + }, + "execution_count": null, + "outputs": [] }, { - "cell_type": "markdown", + "cell_type": "code", "source": [ - "# ๐Ÿ“บ Visualizing classification of images online\n", - "\n", - "Now that we have a deployed model, we can classify images using it.\n", - " \n", - "First, lets define some functions to help us visualize and navigate the images, click run:" + "# Species: didelphis imperfecta (Guianan white-eared opossum)\n", + "predict(\n", + " \"animals/0667/1214.jpg\",\n", + " habitat_region=\"Central and South America tropical rainforest\",\n", + ")" ], "metadata": { "id": "1jDpAELnxzf7" - } + }, + "execution_count": null, + "outputs": [] }, { "cell_type": "code", "execution_count": null, "source": [ - "import io\n", - "import requests\n", - "from PIL import Image\n", - "from IPython.display import display\n", - "\n", - "from google.cloud import bigquery\n", - "\n", - "\n", - "def display_image(image_file, width=400):\n", - " base_url = \"https://lilablobssc.blob.core.windows.net/wcs-unzipped\"\n", - " image_bytes = requests.get(f\"{base_url}/{image_file}\").content\n", - " if b\"\" in image_bytes:\n", - " raise ValueError(\n", - " f\"Error requesting image: {base_url}/{image_file}\\n{image_bytes.decode('utf-8')}\"\n", - " )\n", - " image = Image.open(io.BytesIO(image_bytes))\n", - " display(image.resize((int(width), int(width / image.size[0] * image.size[1]))))\n", - "\n", - "\n", - "def display_samples_for_category(category, num_samples=3, width=400):\n", - " client = bigquery.Client()\n", - " query_job = client.query(\n", - " f\"\"\"\n", - " SELECT file_name\n", - " FROM `{google_cloud_project}.{bigquery_dataset}.{bigquery_table}`\n", - " WHERE category = '{category}'\n", - " LIMIT {num_samples}\n", - " \"\"\"\n", - " )\n", - "\n", - " for row in query_job:\n", - " image_file = row[\"file_name\"]\n", - " print(f\"{category}: {image_file}\")\n", - " display_image(image_file, width)" + "# Species: tapirus indicus (Malayan tapir)\n", + "predict(\"animals/0036/0072.jpg\")" ], "outputs": [], "metadata": { @@ -615,27 +537,26 @@ } }, { - "cell_type": "markdown", + "cell_type": "code", "source": [ - "Now run the following cell to get the model endpoint ID, which we need to make predictions." + "# Species: leopardus wiedii (Margay)\n", + "predict(\n", + " \"animals/0000/1705.jpg\",\n", + " habitat_region=\"Central and South America tropical rainforest\",\n", + ")" ], "metadata": { "id": "AFCtx6I195DA" - } + }, + "execution_count": null, + "outputs": [] }, { "cell_type": "code", "execution_count": null, "source": [ - "# First we need the endpoint ID, we can get it with gcloud.\n", - "stdout = !gcloud beta ai endpoints list \\\n", - " --project {google_cloud_project} \\\n", - " --region {region} \\\n", - " --filter \"display_name={ai_platform_name_prefix}\" \\\n", - " --format \"table[no-heading](ENDPOINT_ID)\" 2>/dev/null\n", - "model_endpoint_id = stdout[0]\n", - "\n", - "print(f\"model_endpoint_id: {model_endpoint_id}\")" + "# Species: hemigalus derbyanus (Banded palm civet)\n", + "predict(\"animals/0036/0566.jpg\")" ], "outputs": [], "metadata": { @@ -646,14 +567,11 @@ "cell_type": "code", "execution_count": null, "source": [ - "def predict(image_file):\n", - " # First, display the image, and then we run our prediction script.\n", - " display_image(image_file)\n", - " !python predict.py \\\n", - " --project \"{google_cloud_project}\" \\\n", - " --region \"{region}\" \\\n", - " --model-endpoint-id \"{model_endpoint_id}\" \\\n", - " --image-file \"{image_file}\"" + "# Species: dasypus novemcinctus (Nine-banded armadillo)\n", + "predict(\n", + " \"animals/0000/0425.jpg\",\n", + " habitat_region=\"Central and South America tropical rainforest\",\n", + ")" ], "outputs": [], "metadata": { @@ -663,15 +581,15 @@ { "cell_type": "markdown", "source": [ - "\n", + "With Gemini's structured output schema (`WildlifeObservation`), you can address this in two ways without training a separate model:\n", + "1. **Taxonomic hierarchy (`taxon_family`)**: Every prediction returns both `species_scientific_name` and `taxon_family` (for example, `Felidae`), giving ecologists reliable family-level rollups even on blurry or partially occluded frames.\n", + "2. **Habitat geo-fencing (`habitat_region`)**: Because camera traps are stationary, passing the trap's geographic region in the prompt constrains predictions to species native to that ecosystem.\n", "\n", - "Now, run the following cells to see what our model thinks about each of the following images.\n", - "The model gives us a list of species it predicts it is, alongside with the confidence score for each prediction." + "Run the following cell to test habitat geo-fencing on a camera trap photo from the **Felidae Conservation Fund** dataset (`gs://public-datasets-lila/felidae-conservation-fund/`), captured in the **San Francisco Bay Area, California**:" ], "metadata": { "id": "nScTC8Si2BZL" @@ -681,8 +599,17 @@ "cell_type": "code", "execution_count": null, "source": [ - "# Species: dicerorhinus sumatrensis\n", - "predict(\"animals/0325/1529.jpg\")" + "# San Francisco Bay Area mountain lion (Puma concolor) from Felidae Conservation Fund (LILA Science)\n", + "sf_bay_image = \"1/2025-03/53c186d3a51ede0a8a002d10dc9af6a985fbc70d04a17a335847e7179ce1a812.jpg\"\n", + "display_image(\n", + " sf_bay_image,\n", + " base_url=\"https://storage.googleapis.com/public-datasets-lila/felidae-conservation-fund\",\n", + ")\n", + "classify_wildlife_image.classify_wildlife_image(\n", + " project_id=google_cloud_project,\n", + " image_gcs_uri=f\"gs://public-datasets-lila/felidae-conservation-fund/{sf_bay_image}\",\n", + " habitat_region=\"San Francisco Bay Area, California, USA\",\n", + ")" ], "outputs": [], "metadata": { @@ -690,13 +617,21 @@ } }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "source": [ - "# Species: didelphis imperfecta\n", - "predict(\"animals/0667/1214.jpg\")" + "## ๐Ÿงฌ Discover rare species with Gemini Embedding 2\n", + "\n", + "When rangers capture a single sighting of a critically endangered animalโ€”such as the **Sumatran rhinoceros** (`Dicerorhinus sumatrensis`, with fewer than 50 individuals remaining in the wild)โ€”they often need to search millions of historical camera trap frames for visually similar animals.\n", + "\n", + "Using [`embed_wildlife_image.py`](embed_wildlife_image.py) and **Gemini Embedding 2** (`gemini-embedding-2`), you can generate a 768-dimensional multimodal vector embedding for any camera trap image:\n", + "\n", + "" ], - "outputs": [], "metadata": { "id": "VF-S9qpnEqjD" } @@ -705,67 +640,53 @@ "cell_type": "code", "execution_count": null, "source": [ - "# Species: tapirus indicus\n", - "predict(\"animals/0036/0072.jpg\")" + "import embed_wildlife_image\n", + "\n", + "# Generate a 768-dimensional multimodal embedding for a Sumatran rhinoceros camera trap frame\n", + "embed_wildlife_image.embed_wildlife_image(\n", + " project_id=google_cloud_project,\n", + " image_gcs_uri=\"gs://public-datasets-lila/wcs-unzipped/animals/0325/1529.jpg\",\n", + ")" ], "outputs": [], "metadata": { "id": "LneZo04sF_Bm" } }, - { - "cell_type": "code", - "execution_count": null, - "source": [ - "# Species: leopardus wiedii\n", - "predict(\"animals/0000/1705.jpg\")" - ], - "outputs": [], - "metadata": { - "id": "qJp96gNqHUwn" - } - }, - { - "cell_type": "code", - "execution_count": null, - "source": [ - "# Species: hemigalus derbyanus\n", - "predict(\"animals/0036/0566.jpg\")" - ], - "outputs": [], - "metadata": { - "id": "yl523QeUGXmR" - } - }, - { - "cell_type": "code", - "execution_count": null, - "source": [ - "# Species: dasypus novemcinctus\n", - "predict(\"animals/0000/0425.jpg\")" - ], - "outputs": [], - "metadata": { - "id": "xlgjmCEQRUsS" - } - }, { "cell_type": "markdown", "source": [ - "๐Ÿ•ต๏ธ While analyzing the modelโ€™s evaluation, you may notice that many species of the same family are very similar and it may be confusing the model. This means it might be worth trying to classify the image to a family instead of a specific species, or get more images from different angles/proximity to detect it better." + "Once image embeddings are stored in an `ARRAY` column in BigQuery, conservationists can run `VECTOR_SEARCH` to find the closest matching frames across an entire camera trap archive:\n", + "\n", + "```sql\n", + "SELECT\n", + " base.image_gcs_uri,\n", + " base.species_scientific_name,\n", + " distance\n", + "FROM\n", + " VECTOR_SEARCH(\n", + " TABLE `samples.wildlife_image_embeddings`,\n", + " 'embedding',\n", + " (\n", + " SELECT embedding\n", + " FROM `samples.wildlife_image_embeddings`\n", + " WHERE image_gcs_uri = 'gs://public-datasets-lila/wcs-unzipped/animals/0325/1529.jpg'\n", + " ),\n", + " top_k => 10,\n", + " distance_type => 'COSINE'\n", + " );\n", + "```" ], "metadata": { - "id": "r7INPuD2I2vV" + "id": "qJp96gNqHUwn" } }, { "cell_type": "markdown", "source": [ - "# ๐Ÿงน Cleaning up to avoid additional costs\n", + "## ๐Ÿงน Clean up\n", "\n", - "To avoid incurring charges to your Google Cloud account for the resources used in this tutorial, either delete the project that contains the resources, or keep the project and delete the individual resources.\n", - " \n", - "If you wish to delete the project you created for this tutorial (itโ€™s the easiest way to eliminate billing), here are the steps:" + "To avoid incurring charges to your Google Cloud account for the resources used in this notebook, either delete the project that contains the resources, or keep the project and delete the individual resources." ], "metadata": { "id": "MGl_1UyiGVl6" @@ -774,20 +695,18 @@ { "cell_type": "markdown", "source": [ - "## Deleting the project\n", - "\n", - "The easiest way to eliminate billing is to delete the project that you created for the tutorial.\n", + "### Delete the project\n", "\n", - "To delete the project:\n", + "To delete the Google Cloud project that you created for this notebook:\n", "\n", "> โš ๏ธ **Caution**: Deleting a project has the following effects:\n", ">\n", - "> - **Everything in the project is deleted.** If you used an existing project for this tutorial, when you delete it, you also delete any other work you've done in the project.\n", + "> - **Everything in the project is deleted.** If you used an existing project for this notebook, when you delete it, you also delete any other work you've done in the project.\n", "> - **Custom project IDs are lost.** When you created this project, you might have created a custom project ID that you want to use in the future. To preserve the URLs that use the project ID, such as an `appspot.com` URL, delete selected resources inside the project instead of deleting the whole project.\n", ">\n", "> If you plan to explore multiple tutorials and quickstarts, reusing projects can help you avoid exceeding project quota limits.\n", "\n", - "1. In the Cloud Console, go to the **Manage resources** page.\n", + "1. In the Google Cloud console, go to the **Manage resources** page.\n", "\n", " \n", + "# Delete the temporary Dataflow staging files in Cloud Storage.\n", + "!gcloud storage rm -r \"gs://{cloud_storage_bucket}/{cloud_storage_directory}\"" + ], + "metadata": { + "id": "8418262e" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "### Optional: Delete a dedicated project\n", "\n", - "1. In the project list, select the project that you want to delete, and then click **Delete**.\n", + "If you created a dedicated Google Cloud project solely for this notebook and want to delete the entire project, uncomment and run the command in the following cell:\n", "\n", - "1. In the dialog, type the project ID, and then click **Shut down** to delete the project." + "> โš ๏ธ **Caution**: Deleting a project permanently deletes everything in the project and releases the project ID. Only delete the project if you created it exclusively for this notebook." ], "metadata": { "id": "nv7phC7WGZuT" } }, + { + "cell_type": "code", + "source": [ + "# Uncomment and run the following line ONLY if you created a dedicated project exclusively for this notebook:\n", + "# !gcloud projects delete \"{google_cloud_project}\" --quiet" + ], + "metadata": { + "id": "a64bdcae" + }, + "execution_count": null, + "outputs": [] + }, { "cell_type": "markdown", "source": [ From 3a7ece3232afbcc14e86e5f5b1eea71696de8cc9 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 28 Oct 2026 20:10:06 +0000 Subject: [PATCH 03/21] docs(people-and-planet-ai): align README.ipynb with vertex-ai-samples notebook_template.ipynb Problem: README.ipynb used custom legacy headings and project configuration variables instead of the standard GoogleCloudPlatform/vertex-ai-samples notebook_template.ipynb layout. Solution: - Adopt the vertex-ai-samples notebook_template.ipynb header and section hierarchy (# Title + 4-way launch table, ## Overview, ## Get started, ### Install required packages, ### Import libraries, ### Authenticate your notebook environment (Colab only), ### Set Google Cloud project information, ### Initialize Gen AI SDK, tutorial sections, and ## Cleaning up). - Standardize notebook configuration parameters on PROJECT_ID, LOCATION, BUCKET_NAME, CLOUD_STORAGE_DIRECTORY, BIGQUERY_DATASET, BIGQUERY_TABLE, and PREDICTIONS_TABLE across all pipeline, interactive inference, embedding, and cleanup cells. --- .../image-classification/README.ipynb | 301 +++++++++--------- 1 file changed, 151 insertions(+), 150 deletions(-) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index bf67e114b42..f9a53e26e2f 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -20,24 +20,19 @@ "cell_type": "code", "execution_count": null, "source": [ - "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n", - "\n", - "# Licensed to the Apache Software Foundation (ASF) under one\n", - "# or more contributor license agreements. See the NOTICE file\n", - "# distributed with this work for additional information\n", - "# regarding copyright ownership. The ASF licenses this file\n", - "# to you under the Apache License, Version 2.0 (the\n", - "# \"License\"); you may not use this file except in compliance\n", - "# with the License. You may obtain a copy of the License at\n", + "# @title Copyright 2026 Google LLC\n", "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", "#\n", - "# Unless required by applicable law or agreed to in writing,\n", - "# software distributed under the License is distributed on an\n", - "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n", - "# KIND, either express or implied. See the License for the\n", - "# specific language governing permissions and limitations\n", - "# under the License." + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." ], "outputs": [], "metadata": { @@ -50,18 +45,28 @@ "source": [ "# Monitor wildlife health via image classification\n", "\n", - "๐Ÿฆœ๐Ÿฆง ๐Ÿ˜๐Ÿฆฉ๐Ÿฆ‹ ๐Ÿฆ‡ ๐Ÿฆ€ ๐Ÿฆ” ๐Ÿ‡ ๐Ÿฆซ ๐Ÿฆ ๐Ÿ† ๐Ÿ— ๐Ÿฆ™ ๐Ÿฆฅ ๐ŸฆŽ ๐Ÿฆƒ\n", - "\n", - "- **Time estimate**: ~25โ€“30 minutes total (~10 minutes of Dataflow pipeline execution time, plus ~15โ€“20 minutes for project setup and interactive exploration)\n", - "- **Cost estimate**: less than \\$1.00\n", - "\n", - "> [Watch the video on YouTube ![thumbnail](https://img.youtube.com/vi/hUzODH3uGg0/0.jpg)](https://youtu.be/hUzODH3uGg0)\n", - ">\n", - "> ๐Ÿ“œ For more information about the background of this solution, see [Recovering global wildlife populations using ML](https://cloud.google.com/blog/topics/developers-practitioners/recovering-global-wildlife-populations-using-ml).\n", - "\n", - "Classify animal species and extract ecological recovery signals from\n", - "[camera trap](https://en.wikipedia.org/wiki/Camera_trap)\n", - "pictures using **Gemini 3.8 Flash** (`gemini-3.8-flash`), **Gemini Embedding 2** (`gemini-embedding-2`), and **Apache Beam `RunInference`** on **Dataflow**." + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \"Google
Open in Colab\n", + " \n", + " 		\n", + " \n", + " \"Google
Open in Colab Enterprise\n", + " \n", + " 		\n", + " \n", + " \"Vertex
Open in Workbench\n", + " \n", + " 		\n", + " \n", + " \"GitHub
View on GitHub\n", + " \n", + "
" ], "metadata": { "id": "1LUt29D0MSeu" @@ -70,78 +75,55 @@ { "cell_type": "markdown", "source": [ - "## ๐Ÿ™ˆ Use this interactive notebook\n", + "## Overview\n", "\n", - "Click the **run** icons โ–ถ๏ธ of each section within this notebook.\n", + "๐Ÿฆœ๐Ÿฆง ๐Ÿ˜๐Ÿฆฉ๐Ÿฆ‹ ๐Ÿฆ‡ ๐Ÿฆ€ ๐Ÿฆ” ๐Ÿ‡ ๐Ÿฆซ ๐Ÿฆ ๐Ÿ† ๐Ÿ— ๐Ÿฆ™ ๐Ÿฆฅ ๐ŸฆŽ ๐Ÿฆƒ\n", "\n", - "Ingest camera trap metadata into BigQuery, run class-balanced multimodal inference with Apache Beam and Gemini, and explore zero-shot classification, habitat geo-fencing, and multimodal image embeddings end-to-end. When you run a code cell, the code runs in the notebook runtime, so you don't make any changes to your personal computer.\n", + "Classify animal species and extract ecological recovery signals from [camera trap](https://en.wikipedia.org/wiki/Camera_trap) pictures using **Gemini 3.8 Flash** (`gemini-3.8-flash`), **Gemini Embedding 2** (`gemini-embedding-2`), and **Apache Beam `RunInference`** on **Dataflow**.\n", "\n", - "> ๐Ÿ›Ž๏ธ **To avoid errors**, wait for each section to finish in order before clicking the next **run** icon.\n", + "- **Time estimate**: ~25โ€“30 minutes total (~10 minutes of Dataflow pipeline execution time, plus ~15โ€“20 minutes for project setup and interactive exploration)\n", + "- **Cost estimate**: less than \\$1.00 (with zero always-on endpoint charges)\n", "\n", - "This notebook requires a Google Cloud project with billing enabled (estimated cost is less than \\$1.00 with zero always-on endpoint charges)." - ], - "metadata": { - "id": "jXDRG8QK2BZD" - } - }, - { - "cell_type": "markdown", - "source": [ - "## ๐Ÿšดโ€โ™€๏ธ Steps included\n", + "> [Watch the video on YouTube ![thumbnail](https://img.youtube.com/vi/hUzODH3uGg0/0.jpg)](https://youtu.be/hUzODH3uGg0)\n", + ">\n", + "> ๐Ÿ“œ For more information about the background of this solution, see [Recovering global wildlife populations using ML](https://cloud.google.com/blog/topics/developers-practitioners/recovering-global-wildlife-populations-using-ml).\n", "\n", - "The notebook uses public camera trap datasets hosted in Cloud Storage (`gs://public-datasets-lila/`) by the\n", - "[Labeled Information Library of Alexandria: Biology and Conservation (LILA)](https://lila.science/datasets/wcscameratraps),\n", - "including the **Wildlife Conservation Society (WCS) Camera Traps** dataset and the **Felidae Conservation Fund** San Francisco Bay Area dataset.\n", + "This notebook uses public camera trap datasets hosted in Cloud Storage (`gs://public-datasets-lila/`) by the [Labeled Information Library of Alexandria: Biology and Conservation (LILA)](https://lila.science/datasets/wcscameratraps), including the **Wildlife Conservation Society (WCS) Camera Traps** dataset and the **Felidae Conservation Fund** San Francisco Bay Area dataset.\n", "\n", "In this notebook, you complete the following steps:\n", "\n", "1. **Create a table of image metadata** _(~5 minutes to complete, costs a few cents)_:\n", " Run [`create_images_metadata_table.py`](create_images_metadata_table.py) on [Dataflow](https://cloud.google.com/dataflow) to ingest and clean the WCS Camera Traps metadata JSON archive directly from `gs://public-datasets-lila/wcs/wcs_camera_traps.json.zip` into a [BigQuery](https://cloud.google.com/bigquery) table.\n", - "\n", "2. **Run class-balanced batch inference in Dataflow** _(~5 minutes to complete, costs a few cents)_:\n", - " Run [`run_wildlife_inference.py`](run_wildlife_inference.py) using Apache Beam's `Sample.FixedSizePerKey` and `RunInference` with **Gemini 3.8 Flash** (`gemini-3.8-flash`). Unlike legacy custom model trainingโ€”which took hours, required copying images into your own bucket, and discarded rare species with fewer than 50 imagesโ€”Gemini classifies images **in-place** from `gs://public-datasets-lila/wcs-unzipped/` and writes structured ecological observations (`species_scientific_name`, `taxon_family`, `common_name`, `animal_count`, `juvenile_present`, `is_empty_or_misfire`, `confidence`) to BigQuery.\n", - "\n", + " Run [`run_wildlife_inference.py`](run_wildlife_inference.py) using Apache Beam's `Sample.FixedSizePerKey` and `RunInference` with **Gemini 3.8 Flash** (`gemini-3.8-flash`) to classify images in-place from `gs://public-datasets-lila/wcs-unzipped/` and write structured ecological observations (`species_scientific_name`, `taxon_family`, `common_name`, `animal_count`, `juvenile_present`, `is_empty_or_misfire`, `confidence`) to BigQuery.\n", "3. **Classify and geo-fence camera trap images interactively**:\n", - " Use [`classify_wildlife_image.py`](classify_wildlife_image.py) to inspect camera trap photos, see how taxonomic family (`taxon_family`) resolves look-alike species, and pass `habitat_region` context to geo-fence predictions (for example, San Francisco Bay Area oak woodlands in `gs://public-datasets-lila/felidae-conservation-fund/`).\n", - "\n", + " Use [`classify_wildlife_image.py`](classify_wildlife_image.py) to inspect camera trap photos, see how taxonomic family (`taxon_family`) resolves look-alike species, and pass `habitat_region` context to geo-fence predictions.\n", "4. **Generate multimodal embeddings for rare-species search**:\n", " Use [`embed_wildlife_image.py`](embed_wildlife_image.py) and **Gemini Embedding 2** (`gemini-embedding-2`) to turn a single reference photo of an endangered species into a 768-dimensional vector embedding for similarity search in BigQuery.\n", - "\n", - "5. **Optional: Clean up resources** to avoid ongoing storage costs." + "5. **Clean up resources** to avoid ongoing storage costs." ], "metadata": { - "id": "Kb9doxCpUcpF" + "id": "jXDRG8QK2BZD" } }, { "cell_type": "markdown", "source": [ - "## โœจ Before you begin\n", - "\n", - "1. Select or create a Google Cloud project with billing enabled, and note your project ID for the `google_cloud_project` field in the \"Enter project details\" section below.\n", - "\n", - " > ๐Ÿ’ก If you don't plan to keep the resources that you create in this notebook, Google recommends creating a new project instead of selecting an existing project so that you can delete the project when you finish.\n", - "\n", - "2. Choose a globally unique name for your Cloud Storage staging bucket (for example, `YOUR_PROJECT_ID-wildlife-staging`) to enter in the `cloud_storage_bucket` field below.\n", - "\n", - "3. The setup cells in the \"Enter project details\" section below automatically enable the required Google Cloud APIs (`dataflow.googleapis.com`, `aiplatform.googleapis.com`, and `bigquery.googleapis.com`), create the regional Cloud Storage bucket, and create the BigQuery dataset directly from this notebook." + "## Get started" ], "metadata": { - "id": "VRZjMwyf2BZE" + "id": "Kb9doxCpUcpF" } }, { "cell_type": "markdown", "source": [ - "### โ›๏ธ Prepare the project environment\n", - "\n", - "Click **run** โ–ถ๏ธ on the following cells to download and install the required libraries and resources.\n", + "### Install required packages\n", "\n", - "> ๐Ÿ’ก You can also view the complete\n", - "> [sample code on GitHub](https://github.com/GoogleCloudPlatform/python-docs-samples/tree/main/people-and-planet-ai/image-classification)." + "Download and install the required libraries and sample scripts for this notebook." ], "metadata": { - "id": "5S5_eLKE-M2o" + "id": "VRZjMwyf2BZE" } }, { @@ -158,25 +140,23 @@ "%cd python-docs-samples/people-and-planet-ai/image-classification\n", "\n", "# Install the sample requirements.\n", - "!pip install --quiet -U pip\n", - "!pip install -r requirements.txt" + "!pip3 install --upgrade --quiet pip\n", + "!pip3 install --upgrade --quiet -r requirements.txt" ], "outputs": [], "metadata": { - "id": "zdiQJibg9iEn" + "id": "5S5_eLKE-M2o" } }, { "cell_type": "markdown", "source": [ - "> ๐Ÿ›Ž๏ธ **Note**: You can ignore dependency version warnings printed by `pip install` in the notebook environment.\n", - "\n", "> โš ๏ธ **Restart the runtime**: After installing the libraries in the previous cell, restart the runtime so the updated packages load properly.\n", ">\n", - "> In the top menu, click **Runtime** > **Restart session** (or **Restart runtime**)." + "> In the top menu, click **Runtime** > **Restart session** (or **Restart runtime**), and then run the following cell." ], "metadata": { - "id": "j_Us7ESa90WN" + "id": "zdiQJibg9iEn" } }, { @@ -188,91 +168,101 @@ ], "outputs": [], "metadata": { - "id": "LevEkQNG94Lx" + "id": "j_Us7ESa90WN" } }, { "cell_type": "markdown", "source": [ - "### โœ๏ธ Enter project details\n", - "\n", - "Enter your Google Cloud project ID, Cloud Storage bucket name, and BigQuery dataset ID in the fields below." + "### Import libraries" ], "metadata": { - "id": "cipdLUp92BZF" + "id": "LevEkQNG94Lx" } }, { "cell_type": "code", "execution_count": null, "source": [ - "google_cloud_project = \"\" # @param {type:\"string\"}\n", - "cloud_storage_bucket = \"\" # @param {type:\"string\"}\n", - "bigquery_dataset = \"samples\" # @param {type:\"string\"}\n", - "\n", - "# Validate inputs.\n", - "if not google_cloud_project:\n", - " raise ValueError(\"Set google_cloud_project to your Google Cloud project ID.\")\n", - "if not cloud_storage_bucket:\n", - " raise ValueError(\"Set cloud_storage_bucket to your Cloud Storage bucket name.\")\n", - "\n", - "# Authenticate to use the Google Cloud resources.\n", - "try:\n", - " from google.colab import auth\n", - "\n", - " auth.authenticate_user()\n", - " print(\"Authenticated\")\n", - "except ModuleNotFoundError:\n", - " import os\n", + "import io\n", + "import os\n", + "import sys\n", "\n", - " service_account_file = os.environ.get(\"GOOGLE_APPLICATION_CREDENTIALS\")\n", - " if service_account_file is None:\n", - " raise ValueError(\n", - " \"Set the GOOGLE_APPLICATION_CREDENTIALS environment variable to your service account JSON file path.\"\n", - " )\n", - " print(f\"GOOGLE_APPLICATION_CREDENTIALS: {service_account_file}\")\n", + "from google import genai\n", + "from IPython.display import display\n", + "from PIL import Image\n", + "import requests\n", "\n", - "%env GOOGLE_CLOUD_PROJECT={google_cloud_project}" + "import classify_wildlife_image\n", + "import embed_wildlife_image" ], "outputs": [], "metadata": { - "id": "lWfoIueZMO9P", - "cellView": "form" + "id": "cipdLUp92BZF" } }, { "cell_type": "markdown", "source": [ - "1. Click **run** โ–ถ๏ธ on the cell above.\n", - "1. Follow the authentication prompt to sign in to your Google Cloud account.\n", + "### Authenticate your notebook environment (Colab only)\n", "\n", - "Run the following cell to set the default configuration values, or modify them before running." + "Authenticate your environment on Google Colab." ], "metadata": { - "id": "sWFY2a_P2BZF" + "id": "lWfoIueZMO9P" } }, { "cell_type": "code", "execution_count": null, "source": [ - "cloud_storage_directory = \"samples/wildlife-insights\" # @param {type:\"string\"}\n", - "bigquery_table = \"wildlife_images_metadata\" # @param {type:\"string\"}\n", - "predictions_table = \"wildlife_predictions\" # @param {type:\"string\"}\n", - "region = \"us-central1\" # @param {type:\"string\"}" + "if \"google.colab\" in sys.modules:\n", + " from google.colab import auth\n", + "\n", + " auth.authenticate_user()" ], "outputs": [], "metadata": { - "cellView": "form", - "id": "arH5wwVq2BZG" + "id": "sWFY2a_P2BZF" } }, { "cell_type": "markdown", "source": [ - "Run the following cell to enable the required APIs (`Dataflow`, `Gemini Enterprise Agent Platform`, and `BigQuery`), create your regional Cloud Storage bucket if it doesn't already exist, and create your BigQuery dataset if it doesn't already exist:" + "### Set Google Cloud project information\n", + "\n", + "To get started using Gemini Enterprise Agent Platform, Dataflow, and BigQuery, you must have an existing Google Cloud project with billing enabled. Set your project ID, Cloud Storage bucket name, region, and BigQuery dataset information below, and then run the setup cell to enable the required APIs and create the Cloud Storage bucket and BigQuery dataset." + ], + "metadata": { + "id": "arH5wwVq2BZG" + } + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "PROJECT_ID = \"[your-project-id]\" # @param {type: \"string\", placeholder: \"[your-project-id]\", isTemplate: true}\n", + "if not PROJECT_ID or PROJECT_ID == \"[your-project-id]\":\n", + " PROJECT_ID = str(os.environ.get(\"GOOGLE_CLOUD_PROJECT\", \"\"))\n", + "\n", + "LOCATION = \"us-central1\" # @param {type:\"string\"}\n", + "BUCKET_NAME = \"[your-bucket-name]\" # @param {type: \"string\", placeholder: \"[your-bucket-name]\", isTemplate: true}\n", + "if not BUCKET_NAME or BUCKET_NAME == \"[your-bucket-name]\":\n", + " BUCKET_NAME = f\"{PROJECT_ID}-wildlife-staging\"\n", + "\n", + "CLOUD_STORAGE_DIRECTORY = \"samples/wildlife-insights\" # @param {type:\"string\"}\n", + "BIGQUERY_DATASET = \"samples\" # @param {type:\"string\"}\n", + "BIGQUERY_TABLE = \"wildlife_images_metadata\" # @param {type:\"string\"}\n", + "PREDICTIONS_TABLE = \"wildlife_predictions\" # @param {type:\"string\"}\n", + "\n", + "if not PROJECT_ID:\n", + " raise ValueError(\"Set PROJECT_ID to your Google Cloud project ID.\")\n", + "\n", + "%env GOOGLE_CLOUD_PROJECT={PROJECT_ID}" ], + "outputs": [], "metadata": { + "cellView": "form", "id": "wc5yFA9W2BZG" } }, @@ -284,14 +274,14 @@ " dataflow.googleapis.com \\\n", " aiplatform.googleapis.com \\\n", " bigquery.googleapis.com \\\n", - " --project \"{google_cloud_project}\"\n", + " --project \"{PROJECT_ID}\"\n", "\n", "# Create the regional Cloud Storage staging bucket if it doesn't already exist.\n", - "!gcloud storage buckets describe \"gs://{cloud_storage_bucket}\" --project=\"{google_cloud_project}\" >/dev/null 2>&1 || \\\n", - " gcloud storage buckets create \"gs://{cloud_storage_bucket}\" --project=\"{google_cloud_project}\" --location=\"{region}\"\n", + "!gcloud storage buckets describe \"gs://{BUCKET_NAME}\" --project=\"{PROJECT_ID}\" >/dev/null 2>&1 || \\\n", + " gcloud storage buckets create \"gs://{BUCKET_NAME}\" --project=\"{PROJECT_ID}\" --location=\"{LOCATION}\"\n", "\n", "# Create the BigQuery dataset if it doesn't already exist.\n", - "!bq --project_id=\"{google_cloud_project}\" --location=\"{region}\" mk --dataset \"{google_cloud_project}:{bigquery_dataset}\" 2>/dev/null || true" + "!bq --project_id=\"{PROJECT_ID}\" --location=\"{LOCATION}\" mk --dataset \"{PROJECT_ID}:{BIGQUERY_DATASET}\" 2>/dev/null || true" ], "metadata": { "id": "2e39c724" @@ -299,6 +289,26 @@ "execution_count": null, "outputs": [] }, + { + "cell_type": "markdown", + "source": [ + "### Initialize Gen AI SDK" + ], + "metadata": { + "id": "34a58e2f" + } + }, + { + "cell_type": "code", + "source": [ + "client = genai.Client(vertexai=True, project=PROJECT_ID, location=\"global\")" + ], + "metadata": { + "id": "45bdd3ac" + }, + "execution_count": null, + "outputs": [] + }, { "cell_type": "markdown", "source": [ @@ -326,13 +336,13 @@ "source": [ "# [One time only] Create the images metadata table.\n", "!python create_images_metadata_table.py \\\n", - " --bigquery-dataset \"{bigquery_dataset}\" \\\n", - " --bigquery-table \"{bigquery_table}\" \\\n", + " --bigquery-dataset \"{BIGQUERY_DATASET}\" \\\n", + " --bigquery-table \"{BIGQUERY_TABLE}\" \\\n", " --runner \"DataflowRunner\" \\\n", " --job_name \"wildlife-images-metadata-`date +%Y%m%d-%H%M%S`\" \\\n", - " --project \"{google_cloud_project}\" \\\n", - " --temp_location \"gs://{cloud_storage_bucket}/{cloud_storage_directory}/temp\" \\\n", - " --region \"{region}\" \\\n", + " --project \"{PROJECT_ID}\" \\\n", + " --temp_location \"gs://{BUCKET_NAME}/{CLOUD_STORAGE_DIRECTORY}/temp\" \\\n", + " --region \"{LOCATION}\" \\\n", " --worker_machine_type \"n1-standard-2\"" ], "outputs": [], @@ -405,17 +415,17 @@ "source": [ "# Run class-balanced sampling and Gemini 3.8 Flash RunInference into BigQuery.\n", "!python run_wildlife_inference.py \\\n", - " --project \"{google_cloud_project}\" \\\n", - " --bigquery-dataset \"{bigquery_dataset}\" \\\n", - " --bigquery-table \"{bigquery_table}\" \\\n", - " --predictions-table \"{predictions_table}\" \\\n", + " --project \"{PROJECT_ID}\" \\\n", + " --bigquery-dataset \"{BIGQUERY_DATASET}\" \\\n", + " --bigquery-table \"{BIGQUERY_TABLE}\" \\\n", + " --predictions-table \"{PREDICTIONS_TABLE}\" \\\n", " --min-images-per-class \"{min_images_per_class}\" \\\n", " --max-images-per-class \"{max_images_per_class}\" \\\n", " --runner \"DataflowRunner\" \\\n", " --job_name \"wildlife-run-inference-`date +%Y%m%d-%H%M%S`\" \\\n", - " --temp_location \"gs://{cloud_storage_bucket}/{cloud_storage_directory}/temp\" \\\n", + " --temp_location \"gs://{BUCKET_NAME}/{CLOUD_STORAGE_DIRECTORY}/temp\" \\\n", " --requirements_file \"requirements.txt\" \\\n", - " --region \"{region}\"" + " --region \"{LOCATION}\"" ], "outputs": [], "metadata": { @@ -455,13 +465,6 @@ "cell_type": "code", "execution_count": null, "source": [ - "import io\n", - "import requests\n", - "from PIL import Image\n", - "from IPython.display import display\n", - "\n", - "import classify_wildlife_image\n", - "\n", "LILA_WCS_HTTP_BASE = \"https://storage.googleapis.com/public-datasets-lila/wcs-unzipped\"\n", "LILA_WCS_GCS_BASE = \"gs://public-datasets-lila/wcs-unzipped\"\n", "\n", @@ -479,7 +482,7 @@ ") -> None:\n", " display_image(image_file)\n", " classify_wildlife_image.classify_wildlife_image(\n", - " project_id=google_cloud_project,\n", + " project_id=PROJECT_ID,\n", " image_gcs_uri=f\"{LILA_WCS_GCS_BASE}/{image_file}\",\n", " habitat_region=habitat_region,\n", " )" @@ -614,7 +617,7 @@ " base_url=\"https://storage.googleapis.com/public-datasets-lila/felidae-conservation-fund\",\n", ")\n", "classify_wildlife_image.classify_wildlife_image(\n", - " project_id=google_cloud_project,\n", + " project_id=PROJECT_ID,\n", " image_gcs_uri=f\"gs://public-datasets-lila/felidae-conservation-fund/{sf_bay_image}\",\n", " habitat_region=\"San Francisco Bay Area, California, USA\",\n", ")" @@ -648,11 +651,9 @@ "cell_type": "code", "execution_count": null, "source": [ - "import embed_wildlife_image\n", - "\n", "# Generate a 768-dimensional multimodal embedding for a Sumatran rhinoceros camera trap frame\n", "embed_wildlife_image.embed_wildlife_image(\n", - " project_id=google_cloud_project,\n", + " project_id=PROJECT_ID,\n", " image_gcs_uri=\"gs://public-datasets-lila/wcs-unzipped/animals/0325/1529.jpg\",\n", ")" ], @@ -692,7 +693,7 @@ { "cell_type": "markdown", "source": [ - "## ๐Ÿงน Clean up\n", + "## Cleaning up\n", "\n", "To avoid incurring ongoing storage charges for the resources created in this notebook, run the following cell to delete the BigQuery dataset and the temporary Dataflow staging files in Cloud Storage:" ], @@ -704,10 +705,10 @@ "cell_type": "code", "source": [ "# Delete the BigQuery dataset (including the metadata and predictions tables).\n", - "!bq rm -r -f -d \"{google_cloud_project}:{bigquery_dataset}\"\n", + "!bq rm -r -f -d \"{PROJECT_ID}:{BIGQUERY_DATASET}\"\n", "\n", "# Delete the temporary Dataflow staging files in Cloud Storage.\n", - "!gcloud storage rm -r \"gs://{cloud_storage_bucket}/{cloud_storage_directory}\"" + "!gcloud storage rm -r \"gs://{BUCKET_NAME}/{CLOUD_STORAGE_DIRECTORY}\"" ], "metadata": { "id": "8418262e" @@ -732,7 +733,7 @@ "cell_type": "code", "source": [ "# Uncomment and run the following line ONLY if you created a dedicated project exclusively for this notebook:\n", - "# !gcloud projects delete \"{google_cloud_project}\" --quiet" + "# !gcloud projects delete \"{PROJECT_ID}\" --quiet" ], "metadata": { "id": "a64bdcae" From 6cb06b56239f5489e3920cff427e7233c900534c Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 28 Oct 2026 20:16:44 +0000 Subject: [PATCH 04/21] docs(people-and-planet-ai): restore original ASF license notice in Cell 0 of README.ipynb Problem: Aligning README.ipynb with notebook_template.ipynb replaced the original Apache Software Foundation (ASF) license notice in Cell 0. Solution: Restore the original Licensed to the Apache Software Foundation (ASF) notice in Cell 0 of README.ipynb. --- .../image-classification/README.ipynb | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index f9a53e26e2f..8c4f0fc5896 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -20,19 +20,24 @@ "cell_type": "code", "execution_count": null, "source": [ - "# @title Copyright 2026 Google LLC\n", + "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n", + "\n", + "# Licensed to the Apache Software Foundation (ASF) under one\n", + "# or more contributor license agreements. See the NOTICE file\n", + "# distributed with this work for additional information\n", + "# regarding copyright ownership. The ASF licenses this file\n", + "# to you under the Apache License, Version 2.0 (the\n", + "# \"License\"); you may not use this file except in compliance\n", + "# with the License. You may obtain a copy of the License at\n", "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." + "# Unless required by applicable law or agreed to in writing,\n", + "# software distributed under the License is distributed on an\n", + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n", + "# KIND, either express or implied. See the License for the\n", + "# specific language governing permissions and limitations\n", + "# under the License." ], "outputs": [], "metadata": { From 951f3ee371e29416f39c6bb68460bb1d65162cb0 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 28 Oct 2026 20:37:56 +0000 Subject: [PATCH 05/21] docs(people-and-planet-ai): auto-derive bucket and dataset from PROJECT_ID and add Colab Enterprise VPC note Problem: Cell 13 rendered @param form fields asking the user to paste in a bucket name and dataset name, and users opening Colab Enterprise in organizations with constraints/compute.skipDefaultNetworkCreation encountered a blank Network dropdown and read-only preview mode. Solution: - Keep only PROJECT_ID and LOCATION in the Set Google Cloud project information @param cell. - Add a dedicated Create a Cloud Storage bucket and BigQuery dataset subsection that automatically derives BUCKET_NAME = f'{PROJECT_ID}-wildlife-insights' and BIGQUERY_DATASET = 'samples', enables the required APIs, and creates the bucket and dataset inline. - Add a collapsible note under the launch buttons explaining how to create the default VPC network and regional subnet if the Network dropdown is blank when importing into Colab Enterprise. --- .../image-classification/README.ipynb | 48 ++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index 8c4f0fc5896..d47584db945 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -71,7 +71,21 @@ " \"GitHub
View on GitHub\n", " \n", " \n", - "" + "\n", + "\n", + "
\n", + "\n", + "
\n", + "Using Colab Enterprise? (Click if the Network dropdown is blank when importing)\n", + "\n", + "Colab Enterprise runs inside your project's VPC network. In organizations that enforce `constraints/compute.skipDefaultNetworkCreation`, new projects do not have a `default` network, so the **Network** dropdown is blank and the notebook opens in read-only preview mode until a network exists. To create a `default` network and `us-central1` subnet in Cloud Shell:\n", + "\n", + "```bash\n", + "gcloud compute networks create default --project=YOUR_PROJECT_ID --subnet-mode=custom\n", + "gcloud compute networks subnets create default --project=YOUR_PROJECT_ID --network=default --region=us-central1 --range=10.128.0.0/20 --enable-private-ip-google-access\n", + "```\n", + "\n", + "
" ], "metadata": { "id": "1LUt29D0MSeu" @@ -236,7 +250,7 @@ "source": [ "### Set Google Cloud project information\n", "\n", - "To get started using Gemini Enterprise Agent Platform, Dataflow, and BigQuery, you must have an existing Google Cloud project with billing enabled. Set your project ID, Cloud Storage bucket name, region, and BigQuery dataset information below, and then run the setup cell to enable the required APIs and create the Cloud Storage bucket and BigQuery dataset." + "To get started using Gemini Enterprise Agent Platform, Dataflow, and BigQuery, you must have an existing Google Cloud project with billing enabled. Enter your `PROJECT_ID` (and optional `LOCATION`) below:" ], "metadata": { "id": "arH5wwVq2BZG" @@ -251,14 +265,6 @@ " PROJECT_ID = str(os.environ.get(\"GOOGLE_CLOUD_PROJECT\", \"\"))\n", "\n", "LOCATION = \"us-central1\" # @param {type:\"string\"}\n", - "BUCKET_NAME = \"[your-bucket-name]\" # @param {type: \"string\", placeholder: \"[your-bucket-name]\", isTemplate: true}\n", - "if not BUCKET_NAME or BUCKET_NAME == \"[your-bucket-name]\":\n", - " BUCKET_NAME = f\"{PROJECT_ID}-wildlife-staging\"\n", - "\n", - "CLOUD_STORAGE_DIRECTORY = \"samples/wildlife-insights\" # @param {type:\"string\"}\n", - "BIGQUERY_DATASET = \"samples\" # @param {type:\"string\"}\n", - "BIGQUERY_TABLE = \"wildlife_images_metadata\" # @param {type:\"string\"}\n", - "PREDICTIONS_TABLE = \"wildlife_predictions\" # @param {type:\"string\"}\n", "\n", "if not PROJECT_ID:\n", " raise ValueError(\"Set PROJECT_ID to your Google Cloud project ID.\")\n", @@ -271,9 +277,26 @@ "id": "wc5yFA9W2BZG" } }, + { + "cell_type": "markdown", + "source": [ + "### Create a Cloud Storage bucket and BigQuery dataset\n", + "\n", + "Run the following cell to automatically derive your staging bucket (`{PROJECT_ID}-wildlife-insights`) and BigQuery dataset (`samples`), enable the required APIs, and create the bucket and dataset if they don't already exist:" + ], + "metadata": { + "id": "5f1a08ef" + } + }, { "cell_type": "code", "source": [ + "BUCKET_NAME = f\"{PROJECT_ID}-wildlife-insights\"\n", + "CLOUD_STORAGE_DIRECTORY = \"samples/wildlife-insights\"\n", + "BIGQUERY_DATASET = \"samples\"\n", + "BIGQUERY_TABLE = \"wildlife_images_metadata\"\n", + "PREDICTIONS_TABLE = \"wildlife_predictions\"\n", + "\n", "# Enable the required Google Cloud APIs.\n", "!gcloud services enable \\\n", " dataflow.googleapis.com \\\n", @@ -286,7 +309,10 @@ " gcloud storage buckets create \"gs://{BUCKET_NAME}\" --project=\"{PROJECT_ID}\" --location=\"{LOCATION}\"\n", "\n", "# Create the BigQuery dataset if it doesn't already exist.\n", - "!bq --project_id=\"{PROJECT_ID}\" --location=\"{LOCATION}\" mk --dataset \"{PROJECT_ID}:{BIGQUERY_DATASET}\" 2>/dev/null || true" + "!bq --project_id=\"{PROJECT_ID}\" --location=\"{LOCATION}\" mk --dataset \"{PROJECT_ID}:{BIGQUERY_DATASET}\" 2>/dev/null || true\n", + "\n", + "print(f\"Cloud Storage bucket: gs://{BUCKET_NAME}\")\n", + "print(f\"BigQuery dataset: {PROJECT_ID}.{BIGQUERY_DATASET}\")" ], "metadata": { "id": "2e39c724" From 7cafdf2ed7a6fadfadedc948d9ea239efff5ca7b Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 28 Oct 2026 21:08:40 +0000 Subject: [PATCH 06/21] chore(ppai): point notebook clone and launch links to branch for testing --- people-and-planet-ai/README.md | 2 +- .../image-classification/README.ipynb | 21 +++++++++++-------- .../image-classification/README.md | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/people-and-planet-ai/README.md b/people-and-planet-ai/README.md index 6ed48a37096..c2d280f853e 100644 --- a/people-and-planet-ai/README.md +++ b/people-and-planet-ai/README.md @@ -2,7 +2,7 @@ ## ๐Ÿฆ [Wildlife Insights -- _image-classification_](image-classification) -> [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/GoogleCloudPlatform/python-docs-samples/blob/main/people-and-planet-ai/image-classification/README.ipynb) +> [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/GoogleCloudPlatform/python-docs-samples/blob/feat/modernize-ppai-image-classification/people-and-planet-ai/image-classification/README.ipynb) > > [Watch the video on YouTube ![thumbnail](https://img.youtube.com/vi/hUzODH3uGg0/0.jpg)](https://youtu.be/hUzODH3uGg0) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index d47584db945..659e33a3e55 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -52,22 +52,22 @@ "\n", "\n", " \n", " \n", " \n", " \n", @@ -149,14 +149,17 @@ "cell_type": "code", "execution_count": null, "source": [ + "import os\n", + "\n", "# Install libffi-dev for the Dataflow pipeline.\n", "!apt-get -qq install libffi-dev\n", "\n", - "# Clone the python-docs-samples repository.\n", - "!git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git\n", - "\n", - "# Navigate to the sample code directory.\n", - "%cd python-docs-samples/people-and-planet-ai/image-classification\n", + "# Clone the python-docs-samples repository (or update if already cloned) and navigate to the sample directory.\n", + "if not os.getcwd().endswith(\"people-and-planet-ai/image-classification\"):\n", + " if not os.path.exists(\"python-docs-samples\"):\n", + " !git clone --depth 1 -b feat/modernize-ppai-image-classification https://github.com/GoogleCloudPlatform/python-docs-samples.git\n", + " %cd python-docs-samples/people-and-planet-ai/image-classification\n", + "!git pull --ff-only origin feat/modernize-ppai-image-classification\n", "\n", "# Install the sample requirements.\n", "!pip3 install --upgrade --quiet pip\n", diff --git a/people-and-planet-ai/image-classification/README.md b/people-and-planet-ai/image-classification/README.md index 86d36d98324..1396ee6a766 100644 --- a/people-and-planet-ai/image-classification/README.md +++ b/people-and-planet-ai/image-classification/README.md @@ -1,6 +1,6 @@ # ๐Ÿฆ Wildlife Insights -- _image-classification_ -[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/GoogleCloudPlatform/python-docs-samples/blob/main/people-and-planet-ai/image-classification/README.ipynb) +[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/GoogleCloudPlatform/python-docs-samples/blob/feat/modernize-ppai-image-classification/people-and-planet-ai/image-classification/README.ipynb) > [Watch the video on YouTube ![thumbnail](https://img.youtube.com/vi/hUzODH3uGg0/0.jpg)](https://youtu.be/hUzODH3uGg0) From 37f4de8a1f37370e8270e09ba1c046b4edc2a972 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 28 Oct 2026 23:48:33 +0000 Subject: [PATCH 07/21] feat(ppai): use %pip install and remove kernel restart requirement --- .../image-classification/README.ipynb | 35 +++++-------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index 659e33a3e55..059fe803575 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -149,7 +149,9 @@ "cell_type": "code", "execution_count": null, "source": [ + "import importlib\n", "import os\n", + "import sys\n", "\n", "# Install libffi-dev for the Dataflow pipeline.\n", "!apt-get -qq install libffi-dev\n", @@ -161,38 +163,19 @@ " %cd python-docs-samples/people-and-planet-ai/image-classification\n", "!git pull --ff-only origin feat/modernize-ppai-image-classification\n", "\n", - "# Install the sample requirements.\n", - "!pip3 install --upgrade --quiet pip\n", - "!pip3 install --upgrade --quiet -r requirements.txt" + "if os.getcwd() not in sys.path:\n", + " sys.path.insert(0, os.getcwd())\n", + "\n", + "# Install the sample requirements in-place (no kernel restart required).\n", + "%pip install --upgrade --quiet pip\n", + "%pip install --upgrade --quiet -r requirements.txt\n", + "importlib.invalidate_caches()" ], "outputs": [], "metadata": { "id": "5S5_eLKE-M2o" } }, - { - "cell_type": "markdown", - "source": [ - "> โš ๏ธ **Restart the runtime**: After installing the libraries in the previous cell, restart the runtime so the updated packages load properly.\n", - ">\n", - "> In the top menu, click **Runtime** > **Restart session** (or **Restart runtime**), and then run the following cell." - ], - "metadata": { - "id": "zdiQJibg9iEn" - } - }, - { - "cell_type": "code", - "execution_count": null, - "source": [ - "# After restarting the runtime, navigate back to the sample code directory.\n", - "%cd python-docs-samples/people-and-planet-ai/image-classification" - ], - "outputs": [], - "metadata": { - "id": "j_Us7ESa90WN" - } - }, { "cell_type": "markdown", "source": [ From 15c8dad3ba7c9d4691b263662b9fe634a322de4e Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Thu, 29 Oct 2026 00:14:15 +0000 Subject: [PATCH 08/21] fix(ppai): use git fetch and checkout FETCH_HEAD for idempotent branch sync --- people-and-planet-ai/image-classification/README.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index 059fe803575..8eb24710b70 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -161,7 +161,8 @@ " if not os.path.exists(\"python-docs-samples\"):\n", " !git clone --depth 1 -b feat/modernize-ppai-image-classification https://github.com/GoogleCloudPlatform/python-docs-samples.git\n", " %cd python-docs-samples/people-and-planet-ai/image-classification\n", - "!git pull --ff-only origin feat/modernize-ppai-image-classification\n", + "!git fetch --depth 1 origin feat/modernize-ppai-image-classification\n", + "!git checkout -f FETCH_HEAD\n", "\n", "if os.getcwd() not in sys.path:\n", " sys.path.insert(0, os.getcwd())\n", From 5130974f6f295970c3779c5c9741f725a8287cf4 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Thu, 29 Oct 2026 00:18:11 +0000 Subject: [PATCH 09/21] feat(ppai): use git sparse-checkout in isolated sample directory --- .../image-classification/README.ipynb | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index 8eb24710b70..74059dc719a 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -156,13 +156,22 @@ "# Install libffi-dev for the Dataflow pipeline.\n", "!apt-get -qq install libffi-dev\n", "\n", - "# Clone the python-docs-samples repository (or update if already cloned) and navigate to the sample directory.\n", - "if not os.getcwd().endswith(\"people-and-planet-ai/image-classification\"):\n", - " if not os.path.exists(\"python-docs-samples\"):\n", - " !git clone --depth 1 -b feat/modernize-ppai-image-classification https://github.com/GoogleCloudPlatform/python-docs-samples.git\n", - " %cd python-docs-samples/people-and-planet-ai/image-classification\n", - "!git fetch --depth 1 origin feat/modernize-ppai-image-classification\n", - "!git checkout -f FETCH_HEAD\n", + "# Sparse-checkout only people-and-planet-ai/image-classification into an isolated directory.\n", + "REPO_DIR = \"ppai-image-classification\"\n", + "SAMPLE_SUBPATH = \"people-and-planet-ai/image-classification\"\n", + "BRANCH = \"feat/modernize-ppai-image-classification\"\n", + "\n", + "if not os.getcwd().endswith(SAMPLE_SUBPATH):\n", + " if not os.path.exists(REPO_DIR):\n", + " !git clone --filter=blob:none --no-checkout --depth 1 --sparse -b {BRANCH} https://github.com/GoogleCloudPlatform/python-docs-samples.git {REPO_DIR}\n", + " !git -C {REPO_DIR} sparse-checkout set {SAMPLE_SUBPATH}\n", + " %cd {REPO_DIR}\n", + " !git fetch --depth 1 origin {BRANCH}\n", + " !git checkout -f FETCH_HEAD\n", + " %cd {SAMPLE_SUBPATH}\n", + "else:\n", + " !git fetch --depth 1 origin {BRANCH}\n", + " !git checkout -f FETCH_HEAD\n", "\n", "if os.getcwd() not in sys.path:\n", " sys.path.insert(0, os.getcwd())\n", From 921dba367d83fde0dd3e672c1d813c63d6a2fcbf Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Thu, 29 Oct 2026 00:20:29 +0000 Subject: [PATCH 10/21] fix(ppai): disable soft-delete on Dataflow temp bucket and silence detachedHead notice --- .../image-classification/README.ipynb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index 74059dc719a..580fddc4dd3 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -167,11 +167,11 @@ " !git -C {REPO_DIR} sparse-checkout set {SAMPLE_SUBPATH}\n", " %cd {REPO_DIR}\n", " !git fetch --depth 1 origin {BRANCH}\n", - " !git checkout -f FETCH_HEAD\n", + " !git -c advice.detachedHead=false checkout -f FETCH_HEAD\n", " %cd {SAMPLE_SUBPATH}\n", "else:\n", " !git fetch --depth 1 origin {BRANCH}\n", - " !git checkout -f FETCH_HEAD\n", + " !git -c advice.detachedHead=false checkout -f FETCH_HEAD\n", "\n", "if os.getcwd() not in sys.path:\n", " sys.path.insert(0, os.getcwd())\n", @@ -300,14 +300,15 @@ " bigquery.googleapis.com \\\n", " --project \"{PROJECT_ID}\"\n", "\n", - "# Create the regional Cloud Storage staging bucket if it doesn't already exist.\n", + "# Create the regional Cloud Storage staging bucket with soft-delete disabled (avoids Dataflow temp storage warnings and charges).\n", "!gcloud storage buckets describe \"gs://{BUCKET_NAME}\" --project=\"{PROJECT_ID}\" >/dev/null 2>&1 || \\\n", - " gcloud storage buckets create \"gs://{BUCKET_NAME}\" --project=\"{PROJECT_ID}\" --location=\"{LOCATION}\"\n", + " gcloud storage buckets create \"gs://{BUCKET_NAME}\" --project=\"{PROJECT_ID}\" --location=\"{LOCATION}\" --soft-delete-duration=0s\n", + "!gcloud storage buckets update \"gs://{BUCKET_NAME}\" --project=\"{PROJECT_ID}\" --clear-soft-delete >/dev/null 2>&1 || true\n", "\n", "# Create the BigQuery dataset if it doesn't already exist.\n", "!bq --project_id=\"{PROJECT_ID}\" --location=\"{LOCATION}\" mk --dataset \"{PROJECT_ID}:{BIGQUERY_DATASET}\" 2>/dev/null || true\n", "\n", - "print(f\"Cloud Storage bucket: gs://{BUCKET_NAME}\")\n", + "print(f\"Cloud Storage bucket: gs://{BUCKET_NAME} (soft-delete disabled)\")\n", "print(f\"BigQuery dataset: {PROJECT_ID}.{BIGQUERY_DATASET}\")" ], "metadata": { From 1598cf63dde2d2332c40450476956bac5aedfae2 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Thu, 29 Oct 2026 00:26:29 +0000 Subject: [PATCH 11/21] docs(ppai): add expected runtime duration comments to notebook cells --- people-and-planet-ai/image-classification/README.ipynb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index 580fddc4dd3..2e4249f0449 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -139,7 +139,7 @@ "source": [ "### Install required packages\n", "\n", - "Download and install the required libraries and sample scripts for this notebook." + "Download and install the required libraries and sample scripts for this notebook _(~1 minute)_." ], "metadata": { "id": "VRZjMwyf2BZE" @@ -149,6 +149,7 @@ "cell_type": "code", "execution_count": null, "source": [ + "# Takes ~1 minute to fetch the sample scripts and install required packages.\n", "import importlib\n", "import os\n", "import sys\n", @@ -278,7 +279,7 @@ "source": [ "### Create a Cloud Storage bucket and BigQuery dataset\n", "\n", - "Run the following cell to automatically derive your staging bucket (`{PROJECT_ID}-wildlife-insights`) and BigQuery dataset (`samples`), enable the required APIs, and create the bucket and dataset if they don't already exist:" + "Run the following cell _(~30 seconds)_ to automatically derive your staging bucket (`{PROJECT_ID}-wildlife-insights`) and BigQuery dataset (`samples`), enable the required APIs, and create the bucket and dataset if they don't already exist:" ], "metadata": { "id": "5f1a08ef" @@ -287,6 +288,7 @@ { "cell_type": "code", "source": [ + "# Takes ~15-30 seconds to enable APIs and create the staging bucket and dataset.\n", "BUCKET_NAME = f\"{PROJECT_ID}-wildlife-insights\"\n", "CLOUD_STORAGE_DIRECTORY = \"samples/wildlife-insights\"\n", "BIGQUERY_DATASET = \"samples\"\n", @@ -342,7 +344,7 @@ "source": [ "## ๐Ÿป Create the image metadata table\n", "\n", - "Before running batch wildlife inference, create a BigQuery metadata table containing the species annotations and image file paths from the WCS Camera Traps archive. This is a _one-time_ setup step.\n", + "Before running batch wildlife inference, create a BigQuery metadata table containing the species annotations and image file paths from the WCS Camera Traps archive. This is a _one-time_ setup step _(~6โ€“8 minutes total: ~3โ€“4 minutes for Dataflow worker VM startup + ~3โ€“4 minutes to stream 736,952 annotations into BigQuery)_:\n", "\n", "Click **run** โ–ถ๏ธ on the following cell to launch the Dataflow pipeline, which extracts two fields for each camera trap image from the WCS Camera Traps JSON archive:\n", "\n", @@ -363,6 +365,7 @@ "execution_count": null, "source": [ "# [One time only] Create the images metadata table.\n", + "# Takes ~6-8 minutes on Dataflow (~3-4 min worker VM startup + ~3-4 min to ingest 736,952 rows).\n", "!python create_images_metadata_table.py \\\n", " --bigquery-dataset \"{BIGQUERY_DATASET}\" \\\n", " --bigquery-table \"{BIGQUERY_TABLE}\" \\\n", @@ -442,6 +445,7 @@ "execution_count": null, "source": [ "# Run class-balanced sampling and Gemini 3.8 Flash RunInference into BigQuery.\n", + "# Takes ~6-10 minutes on Dataflow (~3-4 min worker VM startup + batch Gemini inference).\n", "!python run_wildlife_inference.py \\\n", " --project \"{PROJECT_ID}\" \\\n", " --bigquery-dataset \"{BIGQUERY_DATASET}\" \\\n", From c2265b697d0f923be6b62946240270048875cda0 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Thu, 29 Oct 2026 00:28:29 +0000 Subject: [PATCH 12/21] fix(ppai): enable INFO logging for Dataflow progress and merge param cell --- .../image-classification/README.ipynb | 17 ++++------------- .../create_images_metadata_table.py | 3 +++ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index 2e4249f0449..e9a723fe81a 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -432,18 +432,8 @@ "execution_count": null, "source": [ "min_images_per_class = 1 # @param {type:\"integer\"}\n", - "max_images_per_class = 5 # @param {type:\"integer\"}" - ], - "outputs": [], - "metadata": { - "id": "V5Y4byrzxVMC", - "cellView": "form" - } - }, - { - "cell_type": "code", - "execution_count": null, - "source": [ + "max_images_per_class = 5 # @param {type:\"integer\"}\n", + "\n", "# Run class-balanced sampling and Gemini 3.8 Flash RunInference into BigQuery.\n", "# Takes ~6-10 minutes on Dataflow (~3-4 min worker VM startup + batch Gemini inference).\n", "!python run_wildlife_inference.py \\\n", @@ -461,7 +451,8 @@ ], "outputs": [], "metadata": { - "id": "BgBiq2CzQ9Cp" + "id": "V5Y4byrzxVMC", + "cellView": "form" } }, { diff --git a/people-and-planet-ai/image-classification/create_images_metadata_table.py b/people-and-planet-ai/image-classification/create_images_metadata_table.py index 08006581a5d..f54ebfdab66 100644 --- a/people-and-planet-ai/image-classification/create_images_metadata_table.py +++ b/people-and-planet-ai/image-classification/create_images_metadata_table.py @@ -135,6 +135,9 @@ def create_images_metadata_table( if __name__ == "__main__": import argparse + import logging + + logging.getLogger().setLevel(logging.INFO) parser = argparse.ArgumentParser() parser.add_argument( From 06d1588271d17a4219229460939671e01601439d Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Thu, 29 Oct 2026 00:30:11 +0000 Subject: [PATCH 13/21] fix(ppai): enable INFO logging in run_wildlife_inference.py --- .../image-classification/run_wildlife_inference.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/people-and-planet-ai/image-classification/run_wildlife_inference.py b/people-and-planet-ai/image-classification/run_wildlife_inference.py index 84e32864624..48a02e76515 100644 --- a/people-and-planet-ai/image-classification/run_wildlife_inference.py +++ b/people-and-planet-ai/image-classification/run_wildlife_inference.py @@ -184,8 +184,11 @@ def run_wildlife_inference( if __name__ == "__main__": import argparse + import logging import os + logging.getLogger().setLevel(logging.INFO) + parser = argparse.ArgumentParser() parser.add_argument( "--project", From 36a806b40594df66f671226b1e077d701df3f695 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Thu, 29 Oct 2026 00:40:52 +0000 Subject: [PATCH 14/21] feat(ppai): make BigQuery VECTOR_SEARCH section runnable in notebook --- .../image-classification/README.ipynb | 86 ++++++++++++++----- 1 file changed, 66 insertions(+), 20 deletions(-) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index e9a723fe81a..8522ebab3e6 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -688,31 +688,77 @@ { "cell_type": "markdown", "source": [ - "Once image embeddings are stored in an `ARRAY` column in BigQuery, conservationists can run `VECTOR_SEARCH` to find the closest matching frames across an entire camera trap archive:\n", - "\n", - "```sql\n", - "SELECT\n", - " base.image_gcs_uri,\n", - " base.species_scientific_name,\n", - " distance\n", - "FROM\n", - " VECTOR_SEARCH(\n", - " TABLE `samples.wildlife_image_embeddings`,\n", - " 'embedding',\n", - " (\n", - " SELECT embedding\n", - " FROM `samples.wildlife_image_embeddings`\n", - " WHERE image_gcs_uri = 'gs://public-datasets-lila/wcs-unzipped/animals/0325/1529.jpg'\n", - " ),\n", - " top_k => 10,\n", - " distance_type => 'COSINE'\n", - " );\n", - "```" + "Run the following cell _(~10 seconds)_ to embed the sample camera trap images into a BigQuery table (`samples.wildlife_image_embeddings`) with an `ARRAY` column and execute a live `VECTOR_SEARCH` query to rank the closest matching frames by cosine distance:" ], "metadata": { "id": "qJp96gNqHUwn" } }, + { + "cell_type": "code", + "source": [ + "# Takes ~10 seconds to embed 6 camera trap frames, load into BigQuery, and run VECTOR_SEARCH.\n", + "import json\n", + "from google.genai import types\n", + "\n", + "EMBEDDINGS_TABLE = \"wildlife_image_embeddings\"\n", + "gallery = [\n", + " (\"gs://public-datasets-lila/wcs-unzipped/animals/0325/1529.jpg\", \"Dicerorhinus sumatrensis (Sumatran rhinoceros)\"),\n", + " (\"gs://public-datasets-lila/wcs-unzipped/animals/0036/0072.jpg\", \"Tapirus indicus (Malayan tapir)\"),\n", + " (\"gs://public-datasets-lila/wcs-unzipped/animals/0667/1214.jpg\", \"Didelphis imperfecta (Guianan white-eared opossum)\"),\n", + " (\"gs://public-datasets-lila/wcs-unzipped/animals/0000/1705.jpg\", \"Leopardus wiedii (Margay)\"),\n", + " (\"gs://public-datasets-lila/wcs-unzipped/animals/0036/0566.jpg\", \"Hemigalus derbyanus (Banded palm civet)\"),\n", + " (\"gs://public-datasets-lila/wcs-unzipped/animals/0000/0425.jpg\", \"Dasypus novemcinctus (Nine-banded armadillo)\"),\n", + "]\n", + "\n", + "with open(\"embeddings.jsonl\", \"w\") as f:\n", + " for gcs_uri, species_label in gallery:\n", + " resp = client.models.embed_content(\n", + " model=\"gemini-embedding-2\",\n", + " contents=[types.Part.from_uri(file_uri=gcs_uri, mime_type=\"image/jpeg\")],\n", + " config=types.EmbedContentConfig(output_dimensionality=768),\n", + " )\n", + " f.write(\n", + " json.dumps(\n", + " {\n", + " \"image_gcs_uri\": gcs_uri,\n", + " \"species_scientific_name\": species_label,\n", + " \"embedding\": list(resp.embeddings[0].values),\n", + " }\n", + " )\n", + " + \"\\n\"\n", + " )\n", + "\n", + "with open(\"embeddings_schema.json\", \"w\") as f:\n", + " json.dump(\n", + " [\n", + " {\"name\": \"image_gcs_uri\", \"type\": \"STRING\", \"mode\": \"NULLABLE\"},\n", + " {\"name\": \"species_scientific_name\", \"type\": \"STRING\", \"mode\": \"NULLABLE\"},\n", + " {\"name\": \"embedding\", \"type\": \"FLOAT64\", \"mode\": \"REPEATED\"},\n", + " ],\n", + " f,\n", + " )\n", + "\n", + "!bq --project_id=\"{PROJECT_ID}\" load --source_format=NEWLINE_DELIMITED_JSON --replace \\\n", + " \"{PROJECT_ID}:{BIGQUERY_DATASET}.{EMBEDDINGS_TABLE}\" embeddings.jsonl embeddings_schema.json\n", + "\n", + "!bq --project_id=\"{PROJECT_ID}\" query --use_legacy_sql=false \\\n", + " \"SELECT base.species_scientific_name, base.image_gcs_uri, ROUND(distance, 4) AS cosine_distance \\\n", + " FROM VECTOR_SEARCH( \\\n", + " TABLE \\`{PROJECT_ID}.{BIGQUERY_DATASET}.{EMBEDDINGS_TABLE}\\`, \\\n", + " 'embedding', \\\n", + " (SELECT embedding FROM \\`{PROJECT_ID}.{BIGQUERY_DATASET}.{EMBEDDINGS_TABLE}\\` \\\n", + " WHERE image_gcs_uri = 'gs://public-datasets-lila/wcs-unzipped/animals/0325/1529.jpg'), \\\n", + " top_k => 6, \\\n", + " distance_type => 'COSINE' \\\n", + " ) ORDER BY cosine_distance ASC\"" + ], + "metadata": { + "id": "f9a99c63" + }, + "execution_count": null, + "outputs": [] + }, { "cell_type": "markdown", "source": [ From a3bfc86338c0cdd8c54c7138e38b02c71643ca8c Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Thu, 29 Oct 2026 21:30:31 +0000 Subject: [PATCH 15/21] chore(ppai): align README.ipynb with cloud_sample_notebook rubric checks --- .../image-classification/README.ipynb | 120 +++++++++++------- 1 file changed, 75 insertions(+), 45 deletions(-) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index 8522ebab3e6..9d231720fbb 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -81,8 +81,8 @@ "Colab Enterprise runs inside your project's VPC network. In organizations that enforce `constraints/compute.skipDefaultNetworkCreation`, new projects do not have a `default` network, so the **Network** dropdown is blank and the notebook opens in read-only preview mode until a network exists. To create a `default` network and `us-central1` subnet in Cloud Shell:\n", "\n", "```bash\n", - "gcloud compute networks create default --project=YOUR_PROJECT_ID --subnet-mode=custom\n", - "gcloud compute networks subnets create default --project=YOUR_PROJECT_ID --network=default --region=us-central1 --range=10.128.0.0/20 --enable-private-ip-google-access\n", + "gcloud compute networks create default --project=example-project-id --subnet-mode=custom\n", + "gcloud compute networks subnets create default --project=example-project-id --network=default --region=us-central1 --range=10.128.0.0/20 --enable-private-ip-google-access\n", "```\n", "\n", "" @@ -101,7 +101,7 @@ "Classify animal species and extract ecological recovery signals from [camera trap](https://en.wikipedia.org/wiki/Camera_trap) pictures using **Gemini 3.8 Flash** (`gemini-3.8-flash`), **Gemini Embedding 2** (`gemini-embedding-2`), and **Apache Beam `RunInference`** on **Dataflow**.\n", "\n", "- **Time estimate**: ~25โ€“30 minutes total (~10 minutes of Dataflow pipeline execution time, plus ~15โ€“20 minutes for project setup and interactive exploration)\n", - "- **Cost estimate**: less than \\$1.00 (with zero always-on endpoint charges)\n", + "- **Cost estimate**: less than \\$1.00 (with zero always-on endpoint charges; see the [Google Cloud Pricing Calculator](https://cloud.google.com/products/calculator))\n", "\n", "> [Watch the video on YouTube ![thumbnail](https://img.youtube.com/vi/hUzODH3uGg0/0.jpg)](https://youtu.be/hUzODH3uGg0)\n", ">\n", @@ -257,14 +257,13 @@ "cell_type": "code", "execution_count": null, "source": [ - "PROJECT_ID = \"[your-project-id]\" # @param {type: \"string\", placeholder: \"[your-project-id]\", isTemplate: true}\n", - "if not PROJECT_ID or PROJECT_ID == \"[your-project-id]\":\n", - " PROJECT_ID = str(os.environ.get(\"GOOGLE_CLOUD_PROJECT\", \"\"))\n", + "PROJECT_ID = \"example-project-id\" # @param {type: \"string\", placeholder: \"example-project-id\", isTemplate: true}\n", + "if not PROJECT_ID or PROJECT_ID == \"example-project-id\":\n", + " PROJECT_ID = os.environ.get(\"GOOGLE_CLOUD_PROJECT\", \"example-project-id\")\n", "\n", - "LOCATION = \"us-central1\" # @param {type:\"string\"}\n", - "\n", - "if not PROJECT_ID:\n", - " raise ValueError(\"Set PROJECT_ID to your Google Cloud project ID.\")\n", + "LOCATION = \"us-central1\" # @param {type: \"string\"}\n", + "if not LOCATION:\n", + " LOCATION = os.environ.get(\"GOOGLE_CLOUD_LOCATION\", \"us-central1\")\n", "\n", "%env GOOGLE_CLOUD_PROJECT={PROJECT_ID}" ], @@ -525,7 +524,9 @@ "\n", "\n", "\n", - "Now, run the following cells to see how `gemini-3.8-flash` classifies each camera trap photo into structured ecological metadata (`Species`, `Taxon Family`, `Common Name`, `Animal Count`, `Juvenile Present`, `Empty or Misfire`, and `Confidence`)." + "Now, run the following cells to see how `gemini-3.8-flash` classifies each camera trap photo into structured ecological metadata (`Species`, `Taxon Family`, `Common Name`, `Animal Count`, `Juvenile Present`, `Empty or Misfire`, and `Confidence`).\n", + "\n", + "First, classify a **Sumatran rhinoceros** (`Dicerorhinus sumatrensis`) camera trap frame:" ], "metadata": { "id": "lRRg39FcYILC" @@ -543,6 +544,15 @@ "execution_count": null, "outputs": [] }, + { + "cell_type": "markdown", + "source": [ + "Classify a **Guianan white-eared opossum** (`Didelphis imperfecta`) camera trap frame:" + ], + "metadata": { + "id": "bf03ee3d" + } + }, { "cell_type": "code", "source": [ @@ -558,6 +568,15 @@ "execution_count": null, "outputs": [] }, + { + "cell_type": "markdown", + "source": [ + "Classify a **Malayan tapir** (`Tapirus indicus`) camera trap frame:" + ], + "metadata": { + "id": "9386b435" + } + }, { "cell_type": "code", "execution_count": null, @@ -570,6 +589,15 @@ "id": "6-fuHNjhHh2d" } }, + { + "cell_type": "markdown", + "source": [ + "Classify a **margay** (`Leopardus wiedii`) camera trap frame:" + ], + "metadata": { + "id": "d021f39a" + } + }, { "cell_type": "code", "source": [ @@ -585,6 +613,15 @@ "execution_count": null, "outputs": [] }, + { + "cell_type": "markdown", + "source": [ + "Classify a **banded palm civet** (`Hemigalus derbyanus`) camera trap frame:" + ], + "metadata": { + "id": "dac341c7" + } + }, { "cell_type": "code", "execution_count": null, @@ -597,6 +634,15 @@ "id": "M9jn39iv6EDu" } }, + { + "cell_type": "markdown", + "source": [ + "Classify a **nine-banded armadillo** (`Dasypus novemcinctus`) camera trap frame:" + ], + "metadata": { + "id": "5a0d7818" + } + }, { "cell_type": "code", "execution_count": null, @@ -710,48 +756,29 @@ " (\"gs://public-datasets-lila/wcs-unzipped/animals/0036/0566.jpg\", \"Hemigalus derbyanus (Banded palm civet)\"),\n", " (\"gs://public-datasets-lila/wcs-unzipped/animals/0000/0425.jpg\", \"Dasypus novemcinctus (Nine-banded armadillo)\"),\n", "]\n", - "\n", "with open(\"embeddings.jsonl\", \"w\") as f:\n", " for gcs_uri, species_label in gallery:\n", + " part = types.Part.from_uri(file_uri=gcs_uri, mime_type=\"image/jpeg\")\n", " resp = client.models.embed_content(\n", " model=\"gemini-embedding-2\",\n", - " contents=[types.Part.from_uri(file_uri=gcs_uri, mime_type=\"image/jpeg\")],\n", + " contents=[part],\n", " config=types.EmbedContentConfig(output_dimensionality=768),\n", " )\n", - " f.write(\n", - " json.dumps(\n", - " {\n", - " \"image_gcs_uri\": gcs_uri,\n", - " \"species_scientific_name\": species_label,\n", - " \"embedding\": list(resp.embeddings[0].values),\n", - " }\n", - " )\n", - " + \"\\n\"\n", - " )\n", + " row = {\"image_gcs_uri\": gcs_uri, \"species_scientific_name\": species_label, \"embedding\": list(resp.embeddings[0].values)}\n", + " f.write(json.dumps(row) + \"\\n\")\n", "\n", + "schema = [\n", + " {\"name\": \"image_gcs_uri\", \"type\": \"STRING\", \"mode\": \"NULLABLE\"},\n", + " {\"name\": \"species_scientific_name\", \"type\": \"STRING\", \"mode\": \"NULLABLE\"},\n", + " {\"name\": \"embedding\", \"type\": \"FLOAT64\", \"mode\": \"REPEATED\"},\n", + "]\n", "with open(\"embeddings_schema.json\", \"w\") as f:\n", - " json.dump(\n", - " [\n", - " {\"name\": \"image_gcs_uri\", \"type\": \"STRING\", \"mode\": \"NULLABLE\"},\n", - " {\"name\": \"species_scientific_name\", \"type\": \"STRING\", \"mode\": \"NULLABLE\"},\n", - " {\"name\": \"embedding\", \"type\": \"FLOAT64\", \"mode\": \"REPEATED\"},\n", - " ],\n", - " f,\n", - " )\n", + " json.dump(schema, f)\n", "\n", "!bq --project_id=\"{PROJECT_ID}\" load --source_format=NEWLINE_DELIMITED_JSON --replace \\\n", " \"{PROJECT_ID}:{BIGQUERY_DATASET}.{EMBEDDINGS_TABLE}\" embeddings.jsonl embeddings_schema.json\n", - "\n", "!bq --project_id=\"{PROJECT_ID}\" query --use_legacy_sql=false \\\n", - " \"SELECT base.species_scientific_name, base.image_gcs_uri, ROUND(distance, 4) AS cosine_distance \\\n", - " FROM VECTOR_SEARCH( \\\n", - " TABLE \\`{PROJECT_ID}.{BIGQUERY_DATASET}.{EMBEDDINGS_TABLE}\\`, \\\n", - " 'embedding', \\\n", - " (SELECT embedding FROM \\`{PROJECT_ID}.{BIGQUERY_DATASET}.{EMBEDDINGS_TABLE}\\` \\\n", - " WHERE image_gcs_uri = 'gs://public-datasets-lila/wcs-unzipped/animals/0325/1529.jpg'), \\\n", - " top_k => 6, \\\n", - " distance_type => 'COSINE' \\\n", - " ) ORDER BY cosine_distance ASC\"" + " \"SELECT base.species_scientific_name, base.image_gcs_uri, ROUND(distance, 4) AS cosine_distance FROM VECTOR_SEARCH(TABLE \\`{PROJECT_ID}.{BIGQUERY_DATASET}.{EMBEDDINGS_TABLE}\\`, 'embedding', (SELECT embedding FROM \\`{PROJECT_ID}.{BIGQUERY_DATASET}.{EMBEDDINGS_TABLE}\\` WHERE image_gcs_uri = 'gs://public-datasets-lila/wcs-unzipped/animals/0325/1529.jpg'), top_k => 6, distance_type => 'COSINE') ORDER BY cosine_distance ASC\"" ], "metadata": { "id": "f9a99c63" @@ -773,11 +800,14 @@ { "cell_type": "code", "source": [ - "# Delete the BigQuery dataset (including the metadata and predictions tables).\n", - "!bq rm -r -f -d \"{PROJECT_ID}:{BIGQUERY_DATASET}\"\n", + "delete_resources = True # @param {type:\"boolean\"}\n", + "\n", + "if delete_resources or os.getenv(\"IS_TESTING\"):\n", + " # Delete the BigQuery dataset (including the metadata, predictions, and embeddings tables).\n", + " !bq rm -r -f -d \"{PROJECT_ID}:{BIGQUERY_DATASET}\"\n", "\n", - "# Delete the temporary Dataflow staging files in Cloud Storage.\n", - "!gcloud storage rm -r \"gs://{BUCKET_NAME}/{CLOUD_STORAGE_DIRECTORY}\"" + " # Delete the temporary Dataflow staging files in Cloud Storage.\n", + " !gcloud storage rm -r \"gs://{BUCKET_NAME}/{CLOUD_STORAGE_DIRECTORY}\"" ], "metadata": { "id": "8418262e" From dc69138827d0b404d59032515e44a287612e856b Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Thu, 29 Oct 2026 23:04:41 +0000 Subject: [PATCH 16/21] fix(ppai): add HTTP retry and specific pipeline exception handling per review --- .../create_images_metadata_table.py | 16 ++++++++-- .../run_wildlife_inference.py | 30 ++++++++++++++----- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/people-and-planet-ai/image-classification/create_images_metadata_table.py b/people-and-planet-ai/image-classification/create_images_metadata_table.py index f54ebfdab66..3657b675395 100644 --- a/people-and-planet-ai/image-classification/create_images_metadata_table.py +++ b/people-and-planet-ai/image-classification/create_images_metadata_table.py @@ -22,6 +22,7 @@ import json import os import sys +import time import zipfile import apache_beam as beam @@ -50,7 +51,16 @@ class _ExtractMetadataFn(beam.DoFn): """Downloads and parses camera trap image annotations from LILA Science.""" def process(self, metadata_url: str) -> Iterable[dict[str, str]]: - content = requests.get(metadata_url, timeout=120).content + content = b"" + for attempt in range(3): + try: + content = requests.get(metadata_url, timeout=120).content + break + except requests.RequestException: + if attempt == 2: + raise + time.sleep(2**attempt) + with zipfile.ZipFile(io.BytesIO(content)) as archive: filename = os.path.splitext(os.path.basename(metadata_url))[0] with archive.open(filename) as metadata_file: @@ -119,10 +129,10 @@ def create_images_metadata_table( create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED, ) ) - except exceptions.GoogleAPICallError as err: + except (exceptions.GoogleAPICallError, RuntimeError) as err: print( f"error: failed to populate BigQuery table '{table_spec}' for " - f"project '{project_id}': {err.message}", + f"project '{project_id}': {err}", file=sys.stderr, ) else: diff --git a/people-and-planet-ai/image-classification/run_wildlife_inference.py b/people-and-planet-ai/image-classification/run_wildlife_inference.py index 48a02e76515..966a8ff1fc8 100644 --- a/people-and-planet-ai/image-classification/run_wildlife_inference.py +++ b/people-and-planet-ai/image-classification/run_wildlife_inference.py @@ -30,6 +30,7 @@ from google.genai import errors from google.genai import types from pydantic import BaseModel +from pydantic import ValidationError LILA_WCS_GCS_PREFIX = "gs://public-datasets-lila/wcs-unzipped" @@ -82,12 +83,21 @@ def run_inference( file_uri=image_gcs_uri, mime_type="image/jpeg", ) - response = model.models.generate_content( - model="gemini-3.8-flash", - contents=[image_part, prompt], - config=config, - ) - obs = WildlifeObservation.model_validate_json(response.text) + try: + response = model.models.generate_content( + model="gemini-3.8-flash", + contents=[image_part, prompt], + config=config, + ) + obs = WildlifeObservation.model_validate_json(response.text) + except (errors.APIError, ValidationError) as err: + print( + f"error: failed to classify camera trap image " + f"'{image_gcs_uri}': {err}", + file=sys.stderr, + ) + continue + yield { "image_gcs_uri": image_gcs_uri, "ground_truth_category": item["category"], @@ -168,10 +178,14 @@ def run_wildlife_inference( create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED, ) ) - except (exceptions.GoogleAPICallError, errors.APIError) as err: + except ( + exceptions.GoogleAPICallError, + errors.APIError, + RuntimeError, + ) as err: print( f"error: failed to run wildlife inference pipeline writing to " - f"'{dest_spec}' for project '{project_id}': {err.message}", + f"'{dest_spec}' for project '{project_id}': {err}", file=sys.stderr, ) else: From 9517341d2bd48a27679c8c8fb8bc4b386e503746 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Thu, 29 Oct 2026 23:16:41 +0000 Subject: [PATCH 17/21] fix(ppai): use googlegenaisdk_ region tag prefix for Gen AI SDK snippets --- people-and-planet-ai/image-classification/README.md | 4 ++-- .../image-classification/classify_wildlife_image.py | 4 ++-- .../image-classification/embed_wildlife_image.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/people-and-planet-ai/image-classification/README.md b/people-and-planet-ai/image-classification/README.md index 1396ee6a766..c67c00c6ea2 100644 --- a/people-and-planet-ai/image-classification/README.md +++ b/people-and-planet-ai/image-classification/README.md @@ -16,8 +16,8 @@ pictures in the public [LILA Science](https://lila.science/datasets/wcscameratra ## Snippet files * [`create_images_metadata_table.py`](create_images_metadata_table.py) (`dataflow_wildlife_ingest_metadata`): Ingests and cleans LILA WCS camera trap annotations into BigQuery. -* [`classify_wildlife_image.py`](classify_wildlife_image.py) (`genai_wildlife_classify_image`): Classifies a camera trap image into structured ecological metadata (`species_scientific_name`, `taxon_family`, `common_name`, `animal_count`, `juvenile_present`, `is_empty_or_misfire`, `confidence`) with optional habitat geo-fencing (`habitat_region`). -* [`embed_wildlife_image.py`](embed_wildlife_image.py) (`genai_wildlife_embed_image`): Generates a 768-dimensional multimodal vector embedding for a camera trap image using `gemini-embedding-2`. +* [`classify_wildlife_image.py`](classify_wildlife_image.py) (`googlegenaisdk_wildlife_classify_image`): Classifies a camera trap image into structured ecological metadata (`species_scientific_name`, `taxon_family`, `common_name`, `animal_count`, `juvenile_present`, `is_empty_or_misfire`, `confidence`) with optional habitat geo-fencing (`habitat_region`). +* [`embed_wildlife_image.py`](embed_wildlife_image.py) (`googlegenaisdk_wildlife_embed_image`): Generates a 768-dimensional multimodal vector embedding for a camera trap image using `gemini-embedding-2`. * [`run_wildlife_inference.py`](run_wildlife_inference.py) (`dataflow_wildlife_run_inference`): Runs class-balanced sampling (`Sample.FixedSizePerKey`) and batch Gemini inference (`RunInference`) directly against `gs://public-datasets-lila/wcs-unzipped/` and writes results to BigQuery. [Apache Beam]: https://beam.apache.org diff --git a/people-and-planet-ai/image-classification/classify_wildlife_image.py b/people-and-planet-ai/image-classification/classify_wildlife_image.py index 1885a606890..6a2e93cf070 100644 --- a/people-and-planet-ai/image-classification/classify_wildlife_image.py +++ b/people-and-planet-ai/image-classification/classify_wildlife_image.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# [START genai_wildlife_classify_image] +# [START googlegenaisdk_wildlife_classify_image] import sys from google import genai @@ -102,7 +102,7 @@ def classify_wildlife_image( print(f"Juvenile Present: {observation.juvenile_present}") print(f"Species: {observation.species_scientific_name}") print(f"Taxon Family: {observation.taxon_family}") -# [END genai_wildlife_classify_image] +# [END googlegenaisdk_wildlife_classify_image] if __name__ == "__main__": diff --git a/people-and-planet-ai/image-classification/embed_wildlife_image.py b/people-and-planet-ai/image-classification/embed_wildlife_image.py index 83492f78ba8..c664b2e3b5d 100644 --- a/people-and-planet-ai/image-classification/embed_wildlife_image.py +++ b/people-and-planet-ai/image-classification/embed_wildlife_image.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# [START genai_wildlife_embed_image] +# [START googlegenaisdk_wildlife_embed_image] import sys from google import genai @@ -63,7 +63,7 @@ def embed_wildlife_image(project_id: str, image_gcs_uri: str) -> None: print(f"Embedding Dimensions: {len(values)}") print("Model: gemini-embedding-2") print(f"Sample Vector Values: {preview}") -# [END genai_wildlife_embed_image] +# [END googlegenaisdk_wildlife_embed_image] if __name__ == "__main__": From 10b32273f445f10bd9e24a06261feeecd6cd5d0b Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Thu, 29 Oct 2026 23:23:32 +0000 Subject: [PATCH 18/21] chore(ppai): remove outdated Google Form feedback link from notebook --- people-and-planet-ai/image-classification/README.ipynb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index 9d231720fbb..1d182006f2d 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -845,11 +845,7 @@ "source": [ "## ๐ŸŽ‰ What's next\n", "\n", - "Explore the complete sample code on\n", - "[GitHub](https://github.com/GoogleCloudPlatform/python-docs-samples/tree/main/people-and-planet-ai/image-classification).\n", - "\n", - "> ๐Ÿ›Ž๏ธ To share feedback on this notebook,\n", - "> [complete the two-question feedback survey](https://docs.google.com/forms/d/e/1FAIpQLSeClJev12m2CGE5uwJAvfJRZe9LYUZR4nimsDjrRjvc5xBifA/viewform)." + "Explore the complete sample code on [GitHub](https://github.com/GoogleCloudPlatform/python-docs-samples/tree/main/people-and-planet-ai/image-classification)." ], "metadata": { "id": "Em4E_RNv2BZM" From 68d761272ee2d639303d9e745d2364a77dc1f94e Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 4 Nov 2026 04:50:40 +0000 Subject: [PATCH 19/21] chore(people-and-planet-ai): remove Workbench badge and switch branch references to main Problem: - Vertex AI Workbench (rebranded to Agent Platform Workbench) deprecated the standalone /deploy-notebook flow; the download_url link now only shows an informational banner on the VM instances page requiring a pre-existing running instance. - The notebook launch badges and sparse-checkout BRANCH constant pointed to feat/modernize-ppai-image-classification during testing. Solution: - Remove the visible Open in Workbench badge from people-and-planet-ai/image-classification/README.ipynb while keeping the Colab, Colab Enterprise, and GitHub badges. - Switch all branch references in people-and-planet-ai/image-classification/README.ipynb and README.md back to main so PR #14701 is ready for merge. --- .../image-classification/README.ipynb | 14 +++++--------- .../image-classification/README.md | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index 1d182006f2d..f26bf713c59 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -52,26 +52,22 @@ "\n", "
\n", - " \n", + " \n", " \"Google
Open in Colab\n", " \n", " 		\n", - " \n", + " \n", " \"Google
Open in Colab Enterprise\n", " \n", " 		\n", - " \n", + " \n", " \"Vertex
Open in Workbench\n", " \n", " 		\n", - " \n", + " \n", " \"GitHub
View on GitHub\n", " \n", "
\n", " \n", " \n", " \n", - " \n", "
\n", - " \n", + " \n", " \"Google
Open in Colab\n", " \n", " 		\n", - " \n", + " \n", " \"Google
Open in Colab Enterprise\n", " \n", " 		\n", - " \n", - " \"Vertex
Open in Workbench\n", - " \n", - " 		\n", - " \n", + " \n", " \"GitHub
View on GitHub\n", " \n", "
\n", + "\n", "\n", "
\n", "\n", @@ -160,7 +156,7 @@ "# Sparse-checkout only people-and-planet-ai/image-classification into an isolated directory.\n", "REPO_DIR = \"ppai-image-classification\"\n", "SAMPLE_SUBPATH = \"people-and-planet-ai/image-classification\"\n", - "BRANCH = \"feat/modernize-ppai-image-classification\"\n", + "BRANCH = \"main\"\n", "\n", "if not os.getcwd().endswith(SAMPLE_SUBPATH):\n", " if not os.path.exists(REPO_DIR):\n", diff --git a/people-and-planet-ai/image-classification/README.md b/people-and-planet-ai/image-classification/README.md index c67c00c6ea2..a8a62651008 100644 --- a/people-and-planet-ai/image-classification/README.md +++ b/people-and-planet-ai/image-classification/README.md @@ -1,6 +1,6 @@ # ๐Ÿฆ Wildlife Insights -- _image-classification_ -[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/GoogleCloudPlatform/python-docs-samples/blob/feat/modernize-ppai-image-classification/people-and-planet-ai/image-classification/README.ipynb) +[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/GoogleCloudPlatform/python-docs-samples/blob/main/people-and-planet-ai/image-classification/README.ipynb) > [Watch the video on YouTube ![thumbnail](https://img.youtube.com/vi/hUzODH3uGg0/0.jpg)](https://youtu.be/hUzODH3uGg0) From 1fe7645b07e636b6b54d7925d64fcd3efc684495 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 4 Nov 2026 05:48:17 +0000 Subject: [PATCH 20/21] fix(people-and-planet-ai): address PR #14701 reviewer feedback Problem: - The Colab Enterprise badge logo URL on lh3.googleusercontent.com fails to load when running Jupyter on localhost. - The notebook setup cell executed apt-get and git sparse-checkout commands unconditionally, failing or resetting working trees when run locally outside Google Colab. - e2e_test.py raised a KeyError during pytest collection if GOOGLE_CLOUD_PROJECT was not set. - Image MIME types were hardcoded to image/jpeg when constructing Part.from_uri, which would fail if users adapted the scripts to PNG or WebP images. - The Dataflow batch inference runtime estimate in README.ipynb did not reflect full 639-species runs (~1.5-3.5 hours). Solution: - Update the Colab Enterprise logo URL to https://www.gstatic.com/pantheon/images/bigquery/welcome_page/colab-enterprise-logo.png. - Guard the ephemeral Colab apt-get and git sparse-checkout setup behind if "google.colab" in sys.modules. - Use os.environ.get("GOOGLE_CLOUD_PROJECT") with pytest.skip(..., allow_module_level=True) in e2e_test.py. - Use mimetypes.guess_type(image_gcs_uri) with an image/jpeg fallback in run_wildlife_inference.py, classify_wildlife_image.py, and embed_wildlife_image.py, and add beam.Reshuffle() before RunInference. - Document the ~1.5-3.5 hour full-dataset Dataflow runtime in README.ipynb while noting that interactive sections below can run immediately. --- .../image-classification/README.ipynb | 46 +++++++++---------- .../classify_wildlife_image.py | 4 +- .../image-classification/e2e_test.py | 7 ++- .../embed_wildlife_image.py | 7 ++- .../run_wildlife_inference.py | 5 +- 5 files changed, 40 insertions(+), 29 deletions(-) diff --git a/people-and-planet-ai/image-classification/README.ipynb b/people-and-planet-ai/image-classification/README.ipynb index f26bf713c59..54c3c3958cc 100644 --- a/people-and-planet-ai/image-classification/README.ipynb +++ b/people-and-planet-ai/image-classification/README.ipynb @@ -58,7 +58,7 @@ " \n", " \n", " \n", - " \"Google
Open in Colab Enterprise\n", + " \"Google
Open in Colab Enterprise\n", " \n", " \n", " \n", @@ -96,7 +96,7 @@ "\n", "Classify animal species and extract ecological recovery signals from [camera trap](https://en.wikipedia.org/wiki/Camera_trap) pictures using **Gemini 3.8 Flash** (`gemini-3.8-flash`), **Gemini Embedding 2** (`gemini-embedding-2`), and **Apache Beam `RunInference`** on **Dataflow**.\n", "\n", - "- **Time estimate**: ~25โ€“30 minutes total (~10 minutes of Dataflow pipeline execution time, plus ~15โ€“20 minutes for project setup and interactive exploration)\n", + "- **Time estimate**: ~15โ€“20 minutes for project setup and interactive exploration, plus ~1.5โ€“3.5 hours if you run the full 639-species Dataflow batch inference pipeline in the background\n", "- **Cost estimate**: less than \\$1.00 (with zero always-on endpoint charges; see the [Google Cloud Pricing Calculator](https://cloud.google.com/products/calculator))\n", "\n", "> [Watch the video on YouTube ![thumbnail](https://img.youtube.com/vi/hUzODH3uGg0/0.jpg)](https://youtu.be/hUzODH3uGg0)\n", @@ -107,9 +107,9 @@ "\n", "In this notebook, you complete the following steps:\n", "\n", - "1. **Create a table of image metadata** _(~5 minutes to complete, costs a few cents)_:\n", + "1. **Create a table of image metadata** _(~6โ€“8 minutes to complete, costs a few cents)_:\n", " Run [`create_images_metadata_table.py`](create_images_metadata_table.py) on [Dataflow](https://cloud.google.com/dataflow) to ingest and clean the WCS Camera Traps metadata JSON archive directly from `gs://public-datasets-lila/wcs/wcs_camera_traps.json.zip` into a [BigQuery](https://cloud.google.com/bigquery) table.\n", - "2. **Run class-balanced batch inference in Dataflow** _(~5 minutes to complete, costs a few cents)_:\n", + "2. **Run class-balanced batch inference in Dataflow** _(~1.5โ€“3.5 hours for all 639 species; the interactive steps below can be run immediately while the pipeline runs)_:\n", " Run [`run_wildlife_inference.py`](run_wildlife_inference.py) using Apache Beam's `Sample.FixedSizePerKey` and `RunInference` with **Gemini 3.8 Flash** (`gemini-3.8-flash`) to classify images in-place from `gs://public-datasets-lila/wcs-unzipped/` and write structured ecological observations (`species_scientific_name`, `taxon_family`, `common_name`, `animal_count`, `juvenile_present`, `is_empty_or_misfire`, `confidence`) to BigQuery.\n", "3. **Classify and geo-fence camera trap images interactively**:\n", " Use [`classify_wildlife_image.py`](classify_wildlife_image.py) to inspect camera trap photos, see how taxonomic family (`taxon_family`) resolves look-alike species, and pass `habitat_region` context to geo-fence predictions.\n", @@ -150,25 +150,23 @@ "import os\n", "import sys\n", "\n", - "# Install libffi-dev for the Dataflow pipeline.\n", - "!apt-get -qq install libffi-dev\n", - "\n", - "# Sparse-checkout only people-and-planet-ai/image-classification into an isolated directory.\n", - "REPO_DIR = \"ppai-image-classification\"\n", - "SAMPLE_SUBPATH = \"people-and-planet-ai/image-classification\"\n", - "BRANCH = \"main\"\n", - "\n", - "if not os.getcwd().endswith(SAMPLE_SUBPATH):\n", - " if not os.path.exists(REPO_DIR):\n", - " !git clone --filter=blob:none --no-checkout --depth 1 --sparse -b {BRANCH} https://github.com/GoogleCloudPlatform/python-docs-samples.git {REPO_DIR}\n", - " !git -C {REPO_DIR} sparse-checkout set {SAMPLE_SUBPATH}\n", - " %cd {REPO_DIR}\n", - " !git fetch --depth 1 origin {BRANCH}\n", - " !git -c advice.detachedHead=false checkout -f FETCH_HEAD\n", - " %cd {SAMPLE_SUBPATH}\n", - "else:\n", - " !git fetch --depth 1 origin {BRANCH}\n", - " !git -c advice.detachedHead=false checkout -f FETCH_HEAD\n", + "if \"google.colab\" in sys.modules:\n", + " # Install libffi-dev for the Dataflow pipeline in ephemeral Colab runtimes.\n", + " !apt-get -qq install libffi-dev\n", + "\n", + " # Sparse-checkout only people-and-planet-ai/image-classification into an isolated directory.\n", + " REPO_DIR = \"ppai-image-classification\"\n", + " SAMPLE_SUBPATH = \"people-and-planet-ai/image-classification\"\n", + " BRANCH = \"main\"\n", + "\n", + " if not os.getcwd().endswith(SAMPLE_SUBPATH):\n", + " if not os.path.exists(REPO_DIR):\n", + " !git clone --filter=blob:none --no-checkout --depth 1 --sparse -b {BRANCH} https://github.com/GoogleCloudPlatform/python-docs-samples.git {REPO_DIR}\n", + " !git -C {REPO_DIR} sparse-checkout set {SAMPLE_SUBPATH}\n", + " %cd {REPO_DIR}\n", + " !git fetch --depth 1 origin {BRANCH}\n", + " !git -c advice.detachedHead=false checkout -f FETCH_HEAD\n", + " %cd {SAMPLE_SUBPATH}\n", "\n", "if os.getcwd() not in sys.path:\n", " sys.path.insert(0, os.getcwd())\n", @@ -430,7 +428,7 @@ "max_images_per_class = 5 # @param {type:\"integer\"}\n", "\n", "# Run class-balanced sampling and Gemini 3.8 Flash RunInference into BigQuery.\n", - "# Takes ~6-10 minutes on Dataflow (~3-4 min worker VM startup + batch Gemini inference).\n", + "# Takes ~1.5-3.5 hours across all 639 species (~3,000 images); interactive cells below can run immediately.\n", "!python run_wildlife_inference.py \\\n", " --project \"{PROJECT_ID}\" \\\n", " --bigquery-dataset \"{BIGQUERY_DATASET}\" \\\n", diff --git a/people-and-planet-ai/image-classification/classify_wildlife_image.py b/people-and-planet-ai/image-classification/classify_wildlife_image.py index 6a2e93cf070..7027bba487f 100644 --- a/people-and-planet-ai/image-classification/classify_wildlife_image.py +++ b/people-and-planet-ai/image-classification/classify_wildlife_image.py @@ -15,6 +15,7 @@ # limitations under the License. # [START googlegenaisdk_wildlife_classify_image] +import mimetypes import sys from google import genai @@ -54,9 +55,10 @@ def classify_wildlife_image( location = "global" client = genai.Client(vertexai=True, project=project_id, location=location) + mime_type, _ = mimetypes.guess_type(image_gcs_uri) image_part = types.Part.from_uri( file_uri=image_gcs_uri, - mime_type="image/jpeg", + mime_type=mime_type or "image/jpeg", ) habitat_prefix = ( f"Camera trap habitat region: {habitat_region}. " diff --git a/people-and-planet-ai/image-classification/e2e_test.py b/people-and-planet-ai/image-classification/e2e_test.py index a1328cf2269..8193e8f2ec9 100644 --- a/people-and-planet-ai/image-classification/e2e_test.py +++ b/people-and-planet-ai/image-classification/e2e_test.py @@ -32,7 +32,12 @@ import embed_wildlife_image import run_wildlife_inference -PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] +PROJECT = os.environ.get("GOOGLE_CLOUD_PROJECT") +if not PROJECT: + pytest.skip( + "GOOGLE_CLOUD_PROJECT environment variable is required.", + allow_module_level=True, + ) def test_classify_wildlife_image(capsys: pytest.CaptureFixture[str]) -> None: diff --git a/people-and-planet-ai/image-classification/embed_wildlife_image.py b/people-and-planet-ai/image-classification/embed_wildlife_image.py index c664b2e3b5d..4d92a6ba49b 100644 --- a/people-and-planet-ai/image-classification/embed_wildlife_image.py +++ b/people-and-planet-ai/image-classification/embed_wildlife_image.py @@ -15,6 +15,7 @@ # limitations under the License. # [START googlegenaisdk_wildlife_embed_image] +import mimetypes import sys from google import genai @@ -38,9 +39,10 @@ def embed_wildlife_image(project_id: str, image_gcs_uri: str) -> None: location = "global" client = genai.Client(vertexai=True, project=project_id, location=location) + mime_type, _ = mimetypes.guess_type(image_gcs_uri) image_part = types.Part.from_uri( file_uri=image_gcs_uri, - mime_type="image/jpeg", + mime_type=mime_type or "image/jpeg", ) config = types.EmbedContentConfig(output_dimensionality=768) @@ -57,7 +59,8 @@ def embed_wildlife_image(project_id: str, image_gcs_uri: str) -> None: file=sys.stderr, ) else: - values = response.embeddings[0].values + embeddings = response.embeddings or [response.embedding] + values = embeddings[0].values preview = [round(v, 5) for v in values[:3]] print(f"Resource ID: {image_gcs_uri}") print(f"Embedding Dimensions: {len(values)}") diff --git a/people-and-planet-ai/image-classification/run_wildlife_inference.py b/people-and-planet-ai/image-classification/run_wildlife_inference.py index 966a8ff1fc8..038dc12041f 100644 --- a/people-and-planet-ai/image-classification/run_wildlife_inference.py +++ b/people-and-planet-ai/image-classification/run_wildlife_inference.py @@ -18,6 +18,7 @@ from __future__ import annotations from collections.abc import Iterable, Sequence +import mimetypes import sys from typing import Any @@ -79,9 +80,10 @@ def run_inference( ) for item in batch: image_gcs_uri = f"{LILA_WCS_GCS_PREFIX}/{item['file_name']}" + mime_type, _ = mimetypes.guess_type(image_gcs_uri) image_part = types.Part.from_uri( file_uri=image_gcs_uri, - mime_type="image/jpeg", + mime_type=mime_type or "image/jpeg", ) try: response = model.models.generate_content( @@ -169,6 +171,7 @@ def run_wildlife_inference( ) | "FlattenSampledRows" >> beam.FlatMap(lambda sample: sample) + | "ReshuffleSampledRows" >> beam.Reshuffle() | "RunGeminiInference" >> RunInference(model_handler) | "WritePredictionsToBigQuery" >> beam.io.WriteToBigQuery( From f5c7977bcc49b822ff63a0e29ac97ec62916ec6e Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 4 Nov 2026 07:24:58 +0000 Subject: [PATCH 21/21] style(people-and-planet-ai): fix import ordering in e2e_test.py for flake8 I100 Problem: - The repository-level flake8 lint session treats sibling script imports in people-and-planet-ai/image-classification/e2e_test.py as third-party modules because --application-import-names only lists top-level directories, causing I100 ('import classify_wildlife_image' should be before 'import pytest'). Solution: - Order the imports alphabetically in people-and-planet-ai/image-classification/e2e_test.py so flake8-import-order passes. --- people-and-planet-ai/image-classification/e2e_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/people-and-planet-ai/image-classification/e2e_test.py b/people-and-planet-ai/image-classification/e2e_test.py index 8193e8f2ec9..b0b249c4f0c 100644 --- a/people-and-planet-ai/image-classification/e2e_test.py +++ b/people-and-planet-ai/image-classification/e2e_test.py @@ -25,11 +25,10 @@ import apache_beam as beam from apache_beam.ml.inference.base import RunInference from apache_beam.testing.util import assert_that -import pytest - import classify_wildlife_image import create_images_metadata_table import embed_wildlife_image +import pytest import run_wildlife_inference PROJECT = os.environ.get("GOOGLE_CLOUD_PROJECT")