diff --git a/ApiDemos/project/kotlin-app/build.gradle.kts b/ApiDemos/project/kotlin-app/build.gradle.kts index 70a9c1dad..396bee0d1 100644 --- a/ApiDemos/project/kotlin-app/build.gradle.kts +++ b/ApiDemos/project/kotlin-app/build.gradle.kts @@ -98,6 +98,8 @@ dependencies { androidTestImplementation(libs.junit) androidTestImplementation(libs.espresso.core) androidTestImplementation(libs.truth) + androidTestImplementation(project(":visual-testing")) + androidTestImplementation(libs.uiautomator) implementation(project(":ApiDemos:common-ui")) } diff --git a/ApiDemos/project/kotlin-app/src/androidTest/java/com/example/kotlindemos/visual/BaseVisualVerificationTest.kt b/ApiDemos/project/kotlin-app/src/androidTest/java/com/example/kotlindemos/visual/BaseVisualVerificationTest.kt new file mode 100644 index 000000000..ad8369d7d --- /dev/null +++ b/ApiDemos/project/kotlin-app/src/androidTest/java/com/example/kotlindemos/visual/BaseVisualVerificationTest.kt @@ -0,0 +1,114 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.kotlindemos.visual + +import android.app.Activity +import android.app.Instrumentation +import android.content.Context +import android.content.Intent +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.util.Log +import androidx.test.core.app.ActivityScenario +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.uiautomator.By +import androidx.test.uiautomator.UiDevice +import androidx.test.uiautomator.Until +import com.example.kotlindemos.BuildConfig +import com.google.maps.android.visualtesting.GeminiVisualTestHelper +import org.junit.After +import org.junit.Assert.assertTrue +import org.junit.Assume.assumeTrue +import java.io.File + +/** + * Base class for on-device visual verification tests using Gemini Multimodal AI. + */ +abstract class BaseVisualVerificationTest { + + protected val instrumentation: Instrumentation = InstrumentationRegistry.getInstrumentation() + protected val uiDevice: UiDevice = UiDevice.getInstance(instrumentation) + protected val context: Context = instrumentation.targetContext + protected val helper = GeminiVisualTestHelper() + + @After + fun tearDown() { + helper.close() + } + + protected val geminiApiKey: String? by lazy { + try { + val geminiKeyField = try { + BuildConfig::class.java.getField("GEMINI_API_KEY") + } catch (e: NoSuchFieldException) { + null + } + val key = geminiKeyField?.get(null) as? String + if (!key.isNullOrBlank() && key != "DEFAULT_API_KEY") { + key + } else { + null + } + } catch (e: Exception) { + null + } + } + + /** + * Captures a screenshot from the connected device via UiDevice. + */ + protected fun captureScreenshot(filename: String = "visual_test_${System.currentTimeMillis()}.png"): Bitmap { + val storageDir = context.getExternalFilesDir(null) ?: context.filesDir + val screenshotFile = File(storageDir, filename) + + val taken = uiDevice.takeScreenshot(screenshotFile) + assertTrue("UiDevice failed to capture screenshot: $filename", taken) + + val bitmap = BitmapFactory.decodeFile(screenshotFile.absolutePath) + assertTrue("Failed to decode screenshot bitmap: $filename", bitmap != null) + + Log.i(TAG, "Screenshot captured: ${screenshotFile.absolutePath} (${bitmap.width}x${bitmap.height})") + return bitmap + } + + /** + * Verifies the visual contents of a screenshot using Gemini Multimodal AI. + */ + protected suspend fun verifyScreenshotWithGemini(bitmap: Bitmap, prompt: String) { + val key = geminiApiKey + assumeTrue("GEMINI_API_KEY is not configured; skipping AI visual evaluation", !key.isNullOrBlank()) + + val response = helper.analyzeImage(bitmap, prompt, key!!) + Log.i(TAG, "Gemini Visual Evaluation Response:\n$response") + assertTrue( + "Gemini visual verification failed. Response: $response", + response?.contains("PASSED", ignoreCase = true) == true + ) + } + + /** + * Waits for the map container or tiles to settle. + */ + protected fun waitForMap(timeoutMs: Long = 5000) { + uiDevice.waitForIdle(timeoutMs) + Thread.sleep(1500) + } + + companion object { + private const val TAG = "BaseVisualVerification" + } +} diff --git a/ApiDemos/project/kotlin-app/src/androidTest/java/com/example/kotlindemos/visual/VerifiedSamplesVisualTest.kt b/ApiDemos/project/kotlin-app/src/androidTest/java/com/example/kotlindemos/visual/VerifiedSamplesVisualTest.kt new file mode 100644 index 000000000..c3a19b17c --- /dev/null +++ b/ApiDemos/project/kotlin-app/src/androidTest/java/com/example/kotlindemos/visual/VerifiedSamplesVisualTest.kt @@ -0,0 +1,304 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.kotlindemos.visual + +import androidx.test.core.app.ActivityScenario +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.example.kotlindemos.CameraDemoActivity +import com.example.kotlindemos.CloudBasedMapStylingDemoActivity +import com.example.kotlindemos.DataDrivenBoundariesActivity +import com.example.kotlindemos.DataDrivenDatasetStylingActivity +import com.example.kotlindemos.GroundOverlayDemoActivity +import com.example.kotlindemos.MapColorSchemeActivity +import com.example.kotlindemos.MarkerDemoActivity +import com.example.kotlindemos.TileOverlayDemoActivity +import com.example.kotlindemos.VisibleRegionDemoActivity +import androidx.test.uiautomator.By +import kotlinx.coroutines.runBlocking +import org.junit.Test +import org.junit.runner.RunWith + +/** + * On-device visual verification test suite covering the 9 calibrated samples. + * + * Exercises the multi-state gestures calibrated during evaluation descent spelunk_2026y09m14d_13h59m56s + * and asserts visual correctness against declared @Sample contracts using Gemini Multimodal AI. + */ +@RunWith(AndroidJUnit4::class) +class VerifiedSamplesVisualTest : BaseVisualVerificationTest() { + + @Test + fun testCameraDemoVisuals() = runBlocking { + ActivityScenario.launch(CameraDemoActivity::class.java).use { + waitForMap() + // Click Stop button to interrupt initial scroll + uiDevice.findObject(By.res(context.packageName, "stop_animation"))?.click() + Thread.sleep(800) + // Jump to Bondi + uiDevice.findObject(By.res(context.packageName, "bondi"))?.click() + Thread.sleep(1500) + // Jump to Sydney + uiDevice.findObject(By.res(context.packageName, "sydney"))?.click() + Thread.sleep(1500) + + val bitmap = captureScreenshot("camera_demo_verified.png") + verifyScreenshotWithGemini( + bitmap, + """ + Please act as a QA visual verifier. Analyze this screenshot of CameraDemoActivity. + Verify that: + 1. A Google Map is rendered showing Sydney or Bondi. + 2. Camera control buttons (Bondi, Sydney, Stop) are visible and responsive. + 3. The catalog screen did not leak through or obscure the view. + If all criteria are met, reply with "PASSED". + """.trimIndent() + ) + } + } + + @Test + fun testVisibleRegionDemoVisuals() = runBlocking { + ActivityScenario.launch(VisibleRegionDemoActivity::class.java).use { + waitForMap() + // Tap "Actions ▾" popup menu button + uiDevice.findObject(By.res(context.packageName, "camera_actions_button"))?.click() + Thread.sleep(1000) + // Tap Sydney Opera House action + val actionItem = uiDevice.findObject(By.text("Move to Sydney Opera House")) + ?: uiDevice.findObject(By.res(context.packageName, "vr_soh_button")) + actionItem?.click() + Thread.sleep(2000) + + val bitmap = captureScreenshot("visible_region_verified.png") + verifyScreenshotWithGemini( + bitmap, + """ + Please act as a QA visual verifier. Analyze this screenshot of VisibleRegionDemoActivity. + Verify that: + 1. The map is visible and framed on Sydney / Opera House bounds. + 2. The HUD telemetry card displays projection bounds and camera coordinates. + 3. The "Actions ▾" button is clearly positioned below the toolbar. + If all criteria are met, reply with "PASSED". + """.trimIndent() + ) + } + } + + @Test + fun testMarkerDemoVisuals() = runBlocking { + ActivityScenario.launch(MarkerDemoActivity::class.java).use { + waitForMap() + // Toggle Flat checkbox + uiDevice.findObject(By.res(context.packageName, "flat"))?.click() + Thread.sleep(800) + // Slide rotation seekbar + val rotationBar = uiDevice.findObject(By.res(context.packageName, "rotationSeekBar")) + rotationBar?.let { bar -> + val bounds = bar.visibleBounds + uiDevice.swipe(bounds.centerX(), bounds.centerY(), bounds.right - 10, bounds.centerY(), 15) + } + Thread.sleep(800) + // Click custom info contents button + uiDevice.findObject(By.res(context.packageName, "custom_info_contents"))?.click() + Thread.sleep(1200) + + val bitmap = captureScreenshot("marker_demo_verified.png") + verifyScreenshotWithGemini( + bitmap, + """ + Please act as a QA visual verifier. Analyze this screenshot of MarkerDemoActivity. + Verify that: + 1. Multiple markers are displayed on the map with custom rotation applied. + 2. An InfoWindow or custom info contents popup is visible above a marker. + 3. The controls at the top and bottom are visible without crashing or overlapping. + If all criteria are met, reply with "PASSED". + """.trimIndent() + ) + } + } + + @Test + fun testDataDrivenBoundariesVisuals() = runBlocking { + ActivityScenario.launch(DataDrivenBoundariesActivity::class.java).use { + waitForMap(4000) + // Tap "US" button + uiDevice.findObject(By.res(context.packageName, "button_us"))?.click() + Thread.sleep(3500) + // Tap on map polygon within map container + val mapContainer = uiDevice.findObject(By.res(context.packageName, "map_fragment_container")) + mapContainer?.let { container -> + val bounds = container.visibleBounds + uiDevice.click(bounds.centerX(), bounds.centerY()) + } + Thread.sleep(1500) + + val bitmap = captureScreenshot("data_driven_boundaries_verified.png") + verifyScreenshotWithGemini( + bitmap, + """ + Please act as a QA visual verifier. Analyze this screenshot of DataDrivenBoundariesActivity. + Verify that: + 1. The map is centered on the United States. + 2. State administrative boundaries are rendered with choropleth styling or outlines. + 3. A selected state polygon exhibits highlight styling. + If all criteria are met, reply with "PASSED". + """.trimIndent() + ) + } + } + + @Test + fun testDataDrivenDatasetStylingVisuals() = runBlocking { + ActivityScenario.launch(DataDrivenDatasetStylingActivity::class.java).use { + waitForMap(4000) + // Tap "New York" button + uiDevice.findObject(By.res(context.packageName, "button_ny"))?.click() + Thread.sleep(3500) + // Tap "Kyoto" button + uiDevice.findObject(By.res(context.packageName, "button_kyoto"))?.click() + Thread.sleep(3500) + + val bitmap = captureScreenshot("data_driven_datasets_verified.png") + verifyScreenshotWithGemini( + bitmap, + """ + Please act as a QA visual verifier. Analyze this screenshot of DataDrivenDatasetStylingActivity. + Verify that: + 1. The map is centered on Kyoto with custom dataset feature styling applied. + 2. Dataset polygons/features are visibly rendered over the base map. + If all criteria are met, reply with "PASSED". + """.trimIndent() + ) + } + } + + @Test + fun testCloudBasedMapStylingVisuals() = runBlocking { + ActivityScenario.launch(CloudBasedMapStylingDemoActivity::class.java).use { + waitForMap() + // Tap Satellite button + uiDevice.findObject(By.res(context.packageName, "styling_satellite_mode"))?.click() + Thread.sleep(2500) + // Tap Hybrid button + uiDevice.findObject(By.res(context.packageName, "styling_hybrid_mode"))?.click() + Thread.sleep(2500) + // Tap Terrain button + uiDevice.findObject(By.res(context.packageName, "styling_terrain_mode"))?.click() + Thread.sleep(2500) + + val bitmap = captureScreenshot("cloud_styling_verified.png") + verifyScreenshotWithGemini( + bitmap, + """ + Please act as a QA visual verifier. Analyze this screenshot of CloudBasedMapStylingDemoActivity. + Verify that: + 1. The map renders Cloud-based map styling with terrain contours or photographic layers. + 2. The styling mode buttons (Normal, Satellite, Hybrid, Terrain) are visible along the bottom. + If all criteria are met, reply with "PASSED". + """.trimIndent() + ) + } + } + + @Test + fun testMapColorSchemeVisuals() = runBlocking { + ActivityScenario.launch(MapColorSchemeActivity::class.java).use { + waitForMap() + // Tap Light mode + uiDevice.findObject(By.res(context.packageName, "map_color_light_mode"))?.click() + Thread.sleep(1500) + // Tap Dark mode + uiDevice.findObject(By.res(context.packageName, "map_color_dark_mode"))?.click() + Thread.sleep(1500) + // Tap Follow System + uiDevice.findObject(By.res(context.packageName, "map_color_follow_system_mode"))?.click() + Thread.sleep(1500) + + val bitmap = captureScreenshot("map_color_scheme_verified.png") + verifyScreenshotWithGemini( + bitmap, + """ + Please act as a QA visual verifier. Analyze this screenshot of MapColorSchemeActivity. + Verify that: + 1. The map is rendered with the selected color scheme applied. + 2. Mode buttons (Light, Dark, Follow System) are visible and responsive at the top. + If all criteria are met, reply with "PASSED". + """.trimIndent() + ) + } + } + + @Test + fun testGroundOverlayVisuals() = runBlocking { + ActivityScenario.launch(GroundOverlayDemoActivity::class.java).use { + waitForMap() + // Drag transparency seekbar + val transparencyBar = uiDevice.findObject(By.res(context.packageName, "transparencySeekBar")) + transparencyBar?.let { bar -> + val bounds = bar.visibleBounds + uiDevice.swipe(bounds.centerX(), bounds.centerY(), bounds.right - 10, bounds.centerY(), 15) + } + Thread.sleep(1000) + // Tap Switch Image (1922 historical map) + uiDevice.findObject(By.res(context.packageName, "switchImage"))?.click() + Thread.sleep(1500) + + val bitmap = captureScreenshot("ground_overlay_verified.png") + verifyScreenshotWithGemini( + bitmap, + """ + Please act as a QA visual verifier. Analyze this screenshot of GroundOverlayDemoActivity. + Verify that: + 1. The historical 1922 Newark map ground overlay is pinned to geographic coordinates. + 2. Transparency is applied to the overlay. + 3. The transparency slider and Switch Image button are visible. + If all criteria are met, reply with "PASSED". + """.trimIndent() + ) + } + } + + @Test + fun testTileOverlayVisuals() = runBlocking { + ActivityScenario.launch(TileOverlayDemoActivity::class.java).use { + waitForMap() + // Toggle Fade In checkbox + uiDevice.findObject(By.res(context.packageName, "fade_in_toggle"))?.click() + Thread.sleep(800) + // Drag transparency seekbar + val tileBar = uiDevice.findObject(By.res(context.packageName, "transparencySeekBar")) + tileBar?.let { bar -> + val bounds = bar.visibleBounds + uiDevice.swipe(bounds.centerX(), bounds.centerY(), bounds.right - 10, bounds.centerY(), 15) + } + Thread.sleep(1000) + + val bitmap = captureScreenshot("tile_overlay_verified.png") + verifyScreenshotWithGemini( + bitmap, + """ + Please act as a QA visual verifier. Analyze this screenshot of TileOverlayDemoActivity. + Verify that: + 1. Lunar surface tiles from the custom TileProvider are rendered over the map surface. + 2. The Fade In checkbox and transparency slider are functional. + 3. No bottom sheet modal obscures the view. + If all criteria are met, reply with "PASSED". + """.trimIndent() + ) + } + } +} diff --git a/ApiDemos/project/kotlin-app/src/androidTest/java/com/example/kotlindemos/visual/VisualVerificationTestSuite.kt b/ApiDemos/project/kotlin-app/src/androidTest/java/com/example/kotlindemos/visual/VisualVerificationTestSuite.kt new file mode 100644 index 000000000..3d3fe3cf8 --- /dev/null +++ b/ApiDemos/project/kotlin-app/src/androidTest/java/com/example/kotlindemos/visual/VisualVerificationTestSuite.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.kotlindemos.visual + +import org.junit.runner.RunWith +import org.junit.runners.Suite + +/** + * Test suite grouping all on-device visual verification tests for the 9 calibrated samples. + * + * Run with: + * ./gradlew :ApiDemos:kotlin-app:connectedAndroidTest -Pandroid.testInstrumentationRunnerArguments.class=com.example.kotlindemos.visual.VisualVerificationTestSuite + */ +@RunWith(Suite::class) +@Suite.SuiteClasses( + VerifiedSamplesVisualTest::class +) +class VisualVerificationTestSuite diff --git a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/AdvancedMarkersDemoActivity.kt b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/AdvancedMarkersDemoActivity.kt index fb03311dd..14db17d46 100644 --- a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/AdvancedMarkersDemoActivity.kt +++ b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/AdvancedMarkersDemoActivity.kt @@ -13,6 +13,10 @@ // limitations under the License. package com.example.kotlindemos +import com.example.common_ui.catalog.Sample +import com.example.common_ui.catalog.Complexity +import com.example.common_ui.catalog.Framework + import android.graphics.Color import androidx.core.graphics.toColorInt @@ -50,6 +54,18 @@ private val TAG = AdvancedMarkersDemoActivity::class.java.name * The following sample showcases how to create Advanced Markers, and use all their customization * possibilities. */ +@Sample( + id = "com.example.kotlindemos.AdvancedMarkersDemoActivity", + title = "Advanced Markers & Pins", + description = "Modern PinConfig pins, custom glyphs, badge icon views, and collision behavior.", + category = "Markers & Overlays", + complexity = Complexity.ADVANCED, + tags = ["#markers", "#advancedmarkers", "#pinconfig", "#collision", "#badges", "#mapid"], + purpose = "Demonstrates Cloud-backed Advanced Markers with custom colors, pin glyphs, collision behaviors, and custom View icons.", + successCriteria = "Custom colored pins and badge icon views render sharply at correct anchor points with collision handling.", + failureIndicators = "Pins render as default red markers (missing Map ID), collision behavior ignored, or badge text blurry.", + framework = Framework.KOTLIN_VIEWS +) // [START maps_android_sample_marker_advanced] class AdvancedMarkersDemoActivity : SamplesBaseActivity(), OnMapReadyCallback { @@ -109,10 +125,10 @@ class AdvancedMarkersDemoActivity : SamplesBaseActivity(), OnMapReadyCallback { */ override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - setContentView(com.example.common_ui.R.layout.advanced_markers_demo) + setContentView(R.layout.advanced_markers_demo) - if (getString(com.example.common_ui.R.string.map_id) != "DEMO_MAP_ID") { - val mapFragment = supportFragmentManager.findFragmentById(com.example.common_ui.R.id.map) as SupportMapFragment? + if (getString(R.string.map_id) != "DEMO_MAP_ID") { + val mapFragment = supportFragmentManager.findFragmentById(R.id.map) as SupportMapFragment? mapFragment?.getMapAsync(this) } else { val mapId = (application as ApiDemoApplication).mapId @@ -134,7 +150,7 @@ class AdvancedMarkersDemoActivity : SamplesBaseActivity(), OnMapReadyCallback { mapFragment.getMapAsync(this) } - applyInsets(findViewById(com.example.common_ui.R.id.map_container)) + applyInsets(findViewById(R.id.map_container)) } override fun onMapReady(map: GoogleMap) { @@ -176,12 +192,13 @@ class AdvancedMarkersDemoActivity : SamplesBaseActivity(), OnMapReadyCallback { val pinConfigMagenta = PinConfig.builder() .setBackgroundColor(Color.MAGENTA) .build() - map.addMarker( + val klMarker = map.addMarker( AdvancedMarkerOptions() .icon(BitmapDescriptorFactory.fromPinConfig(pinConfigMagenta)) .position(KUALA_LUMPUR) .title("Kuala Lumpur (Magenta Pin)") ) + klMarker?.showInfoWindow() // 3. PinConfig with custom border color val pinConfigBorder = PinConfig.builder() diff --git a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/BasicMapDemoActivity.kt b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/BasicMapDemoActivity.kt index f1a865bd0..04360d1cd 100644 --- a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/BasicMapDemoActivity.kt +++ b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/BasicMapDemoActivity.kt @@ -16,6 +16,12 @@ package com.example.kotlindemos + +import com.example.common_ui.R +import com.example.common_ui.catalog.Sample +import com.example.common_ui.catalog.Complexity +import com.example.common_ui.catalog.Framework + import android.os.Bundle import com.google.android.gms.maps.CameraUpdateFactory @@ -28,6 +34,23 @@ import com.google.android.gms.maps.model.MarkerOptions /** * This shows how to create a simple activity with a map and a marker on the map. */ +@Sample( + id = "com.example.kotlindemos.BasicMapDemoActivity", + title = "Basic Map", + description = "Fundamental map instantiation, lifecycle binding, and default camera centering.", + category = "Map Initialization", + complexity = Complexity.SNIPPET, + tags = ["#map", "#init", "#lifecycle", "#quickstart"], + apiCalls = [ + "SupportMapFragment.getMapAsync(OnMapReadyCallback)", + "GoogleMap.addMarker(MarkerOptions)", + "GoogleMap.moveCamera(CameraUpdate)" + ], + purpose = "Demonstrates clean, minimal map instantiation using SupportMapFragment.", + successCriteria = "The map loads default vector tiles cleanly centered at the initial coordinates with working gestures.", + failureIndicators = "Grey tiles (missing API key or auth mismatch), crash on back navigation, or map failing to unpause.", + framework = Framework.KOTLIN_VIEWS +) // [START maps_android_sample_basic_map] class BasicMapDemoActivity : SamplesBaseActivity(), OnMapReadyCallback { @@ -36,9 +59,9 @@ class BasicMapDemoActivity : SamplesBaseActivity(), OnMapReadyCallback { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - setContentView(com.example.common_ui.R.layout.basic_demo) + setContentView(R.layout.basic_demo) val mapFragment : SupportMapFragment? = - supportFragmentManager.findFragmentById(com.example.common_ui.R.id.map) as? SupportMapFragment + supportFragmentManager.findFragmentById(R.id.map) as? SupportMapFragment mapFragment?.getMapAsync(this) } @@ -49,7 +72,7 @@ class BasicMapDemoActivity : SamplesBaseActivity(), OnMapReadyCallback { override fun onMapReady(googleMap: GoogleMap) { with(googleMap) { moveCamera(CameraUpdateFactory.newLatLngZoom(SYDNEY, ZOOM_LEVEL)) - addMarker(MarkerOptions().position(SYDNEY)) + addMarker(MarkerOptions().position(SYDNEY).title("Sydney")) } } } diff --git a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CameraClampingDemoActivity.kt b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CameraClampingDemoActivity.kt index e55bca42c..9dfcd854e 100644 --- a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CameraClampingDemoActivity.kt +++ b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CameraClampingDemoActivity.kt @@ -13,6 +13,12 @@ // limitations under the License. package com.example.kotlindemos + +import com.example.common_ui.databinding.CameraClampingDemoBinding +import com.example.common_ui.catalog.Sample +import com.example.common_ui.catalog.Complexity +import com.example.common_ui.catalog.Framework + import android.os.Bundle import android.widget.Toast @@ -32,10 +38,22 @@ import kotlinx.coroutines.launch /** * This shows how to constrain the camera to specific boundaries and zoom levels. */ +@Sample( + id = "com.example.kotlindemos.CameraClampingDemoActivity", + title = "Camera Clamping & Bounds", + description = "Constraining camera viewport to LatLngBounds and dynamic min/max zoom limits.", + category = "Camera Controls", + complexity = Complexity.SIMPLE, + tags = ["#camera", "#clamping", "#bounds", "#zoomlimits", "#latlngbounds"], + purpose = "Demonstrates restricting camera panning to a specific bounding box (Adelaide/Pacific) and zoom slider limits.", + successCriteria = "User cannot pan the camera outside the clamped region; zoom sliders enforce min/max bounds immediately.", + failureIndicators = "Camera pans outside bounding box or resetting bounds fails when selecting 'Reset Bounds'.", + framework = Framework.KOTLIN_VIEWS +) class CameraClampingDemoActivity : SamplesBaseActivity() { internal lateinit var map: GoogleMap - private lateinit var binding: com.example.common_ui.databinding.CameraClampingDemoBinding + private lateinit var binding: CameraClampingDemoBinding /** @@ -47,7 +65,7 @@ class CameraClampingDemoActivity : SamplesBaseActivity() { @OptIn(ExperimentalCoroutinesApi::class) override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - binding = com.example.common_ui.databinding.CameraClampingDemoBinding.inflate(layoutInflater) + binding = CameraClampingDemoBinding.inflate(layoutInflater) setContentView(binding.root) updateZoomLabel(minZoom, maxZoom) val mapFragment = supportFragmentManager.findFragmentById(R.id.map) as SupportMapFragment diff --git a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CameraDemoActivity.kt b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CameraDemoActivity.kt index 5c1597a42..c11985849 100644 --- a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CameraDemoActivity.kt +++ b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CameraDemoActivity.kt @@ -16,6 +16,10 @@ package com.example.kotlindemos +import com.example.common_ui.catalog.Sample +import com.example.common_ui.catalog.Complexity +import com.example.common_ui.catalog.Framework + import android.graphics.Color import android.os.Bundle import android.util.Log @@ -37,12 +41,31 @@ import com.google.android.gms.maps.OnMapReadyCallback import com.google.android.gms.maps.SupportMapFragment import com.google.android.gms.maps.model.CameraPosition import com.google.android.gms.maps.model.LatLng +import com.google.android.gms.maps.model.MarkerOptions import com.example.common_ui.databinding.CameraDemoBinding import com.google.android.gms.maps.model.PolylineOptions /** * This shows how to change the camera position for the map. */ +@Sample( + id = "com.example.kotlindemos.CameraDemoActivity", + title = "Camera Controls & Animation", + description = "Programmatic camera panning, zooming, tilt, bearing, and smooth animations.", + category = "Camera Controls", + complexity = Complexity.SIMPLE, + tags = ["#camera", "#animation", "#bearing", "#tilt", "#zoom", "#pan"], + apiCalls = [ + "GoogleMap.animateCamera(CameraUpdate)", + "GoogleMap.moveCamera(CameraUpdate)", + "GoogleMap.addMarker(MarkerOptions)", + "CameraUpdateFactory.newCameraPosition(CameraPosition)" + ], + purpose = "Demonstrates programmatic camera movements, animated transitions, tilt angles, and bearing rotations.", + successCriteria = "Buttons animate camera smoothly with custom durations, stops, and rotation angles.", + failureIndicators = "Jerky animations, unexpected camera jumps, or tilt angle exceeding platform constraints.", + framework = Framework.KOTLIN_VIEWS +) // [START maps_camera_events] class CameraDemoActivity : SamplesBaseActivity(), @@ -138,6 +161,8 @@ class CameraDemoActivity : // Show Sydney moveCamera(CameraUpdateFactory.newLatLngZoom(sydneyLatLng, 10f)) + addMarker(MarkerOptions().position(sydneyLatLng).title("Sydney")) + addMarker(MarkerOptions().position(LatLng(-33.891614, 151.276417)).title("Bondi Beach")) } } diff --git a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CircleDemoActivity.kt b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CircleDemoActivity.kt index 9d1217d23..ed81e4196 100644 --- a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CircleDemoActivity.kt +++ b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CircleDemoActivity.kt @@ -16,6 +16,12 @@ package com.example.kotlindemos + +import com.example.common_ui.databinding.CircleDemoBinding +import com.example.common_ui.catalog.Sample +import com.example.common_ui.catalog.Complexity +import com.example.common_ui.catalog.Framework + import android.graphics.Color import android.graphics.Point import android.os.Bundle @@ -49,6 +55,18 @@ import java.util.ArrayList /** * This shows how to draw circles on a map. */ +@Sample( + id = "com.example.kotlindemos.CircleDemoActivity", + title = "Circles & Geodesic Radii", + description = "Drawing geographic circles with dynamic center drag, radius sliders, and stroke styling.", + category = "Shapes & Geometry", + complexity = Complexity.SIMPLE, + tags = ["#shapes", "#circles", "#radius", "#geodesic"], + purpose = "Demonstrates drawing circles with radius defined in meters and dynamic updates via seekbars.", + successCriteria = "Adjusting radius slider dynamically updates circle boundary in real-time.", + failureIndicators = "Circle distorted or radius math inaccurate across high latitudes.", + framework = Framework.KOTLIN_VIEWS +) class CircleDemoActivity : SamplesBaseActivity(), SeekBar.OnSeekBarChangeListener, AdapterView.OnItemSelectedListener, @@ -78,7 +96,7 @@ class CircleDemoActivity : SamplesBaseActivity(), private var fillColorArgb : Int = 0 private var strokeColorArgb: Int = 0 - internal lateinit var binding: com.example.common_ui.databinding.CircleDemoBinding + internal lateinit var binding: CircleDemoBinding /** * This class contains information about a circle, including its markers @@ -143,7 +161,7 @@ class CircleDemoActivity : SamplesBaseActivity(), override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - binding = com.example.common_ui.databinding.CircleDemoBinding.inflate(layoutInflater) + binding = CircleDemoBinding.inflate(layoutInflater) setContentView(binding.root) // Set all the SeekBars diff --git a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CloudBasedMapStylingDemoActivity.kt b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CloudBasedMapStylingDemoActivity.kt index 8dc6e5983..f1e4b3e84 100644 --- a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CloudBasedMapStylingDemoActivity.kt +++ b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/CloudBasedMapStylingDemoActivity.kt @@ -26,9 +26,11 @@ import android.os.Bundle import android.view.View import com.example.common_ui.R +import com.google.android.gms.maps.CameraUpdateFactory import com.google.android.gms.maps.GoogleMap import com.google.android.gms.maps.OnMapReadyCallback import com.google.android.gms.maps.SupportMapFragment +import com.google.android.gms.maps.model.LatLng /** * This shows how to use Cloud-based Map Styling in a simple Activity. For more information on how @@ -56,6 +58,8 @@ class CloudBasedMapStylingDemoActivity : SamplesBaseActivity(), OnMapReadyCallba override fun onMapReady(map: GoogleMap) { this.map = map map.mapType = currentMapType + val montBlanc = LatLng(45.8326, 6.8652) + map.moveCamera(CameraUpdateFactory.newLatLngZoom(montBlanc, 11.5f)) } private fun setUpButtonListeners() { diff --git a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/DataDrivenBoundariesActivity.kt b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/DataDrivenBoundariesActivity.kt index c80f5ab51..6af39417f 100644 --- a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/DataDrivenBoundariesActivity.kt +++ b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/DataDrivenBoundariesActivity.kt @@ -13,6 +13,10 @@ // limitations under the License. package com.example.kotlindemos +import com.example.common_ui.catalog.Sample +import com.example.common_ui.catalog.Complexity +import com.example.common_ui.catalog.Framework + import android.graphics.Color import android.os.Bundle import android.util.Log @@ -48,6 +52,19 @@ private val TAG = DataDrivenBoundariesActivity::class.java.simpleName * on how the Data-driven styling for boundaries work, check out the following link: * https://developers.google.com/maps/documentation/android-sdk/dds-boundaries/overview */ +@Sample( + id = "com.example.kotlindemos.DataDrivenBoundariesActivity", + title = "Data-Driven Boundaries", + description = "Dynamic styling and click handlers for administrative boundaries (Localities, States, Countries).", + category = "Data-Driven Styling", + complexity = Complexity.ADVANCED, + tags = ["#boundaries", "#datadriven", "#featurelayer", "#locality", "#choropleth"], + purpose = "Demonstrates styling administrative boundaries dynamically via FeatureLayer and capturing boundary clicks.", + successCriteria = "Boundaries render with custom stroke and fill colors; tapping a region highlights its polygon.", + failureIndicators = "Boundary layer is null (requires vector map / Map ID) or click listener not firing.", + framework = Framework.KOTLIN_VIEWS +) +// [START maps_android_data_driven_styling_boundaries] // Add PopupMenu.OnMenuItemClickListener interface class DataDrivenBoundariesActivity : SamplesBaseActivity(), OnMapReadyCallback, FeatureLayer.OnFeatureClickListener, PopupMenu.OnMenuItemClickListener { @@ -102,7 +119,9 @@ class DataDrivenBoundariesActivity : SamplesBaseActivity(), OnMapReadyCallback, centerMapOnLocation(HANA_HAWAII, 11f) // Adjusted zoom from Java } findViewById(R.id.button_us).setOnClickListener { - centerMapOnLocation(CENTER_US, 1f) // Adjusted zoom from Java + adminAreaEnabled = true + updateStyles() + centerMapOnLocation(CENTER_US, 3.8f) } setupBoundarySelectorButton() // Setup the new selector button @@ -301,4 +320,5 @@ class DataDrivenBoundariesActivity : SamplesBaseActivity(), OnMapReadyCallback, updateStyles() // Apply changes to map layers return true } -} \ No newline at end of file +} +// [END maps_android_data_driven_styling_boundaries] \ No newline at end of file diff --git a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/DataDrivenDatasetStylingActivity.kt b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/DataDrivenDatasetStylingActivity.kt index 1f1443377..2901e0279 100644 --- a/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/DataDrivenDatasetStylingActivity.kt +++ b/ApiDemos/project/kotlin-app/src/main/java/com/example/kotlindemos/DataDrivenDatasetStylingActivity.kt @@ -13,6 +13,12 @@ // limitations under the License. package com.example.kotlindemos + +import com.example.common_ui.R +import com.example.common_ui.catalog.Sample +import com.example.common_ui.catalog.Complexity +import com.example.common_ui.catalog.Framework + import android.graphics.Color import android.os.Build @@ -53,6 +59,19 @@ private val TAG = DataDrivenDatasetStylingActivity::class.java.name * on how the Data-driven styling for boundaries work, check out the following link: * https://developers.google.com/maps/documentation/android-sdk/dds-datasets/overview */ +@Sample( + id = "com.example.kotlindemos.DataDrivenDatasetStylingActivity", + title = "Data-Driven Dataset Styling", + description = "Styling custom geospatial datasets uploaded to Google Cloud Platform based on attributes.", + category = "Data-Driven Styling", + complexity = Complexity.ADVANCED, + tags = ["#datasets", "#datadriven", "#clouddata", "#attributes", "#filtering"], + purpose = "Demonstrates loading a Cloud Dataset FeatureLayer and applying dynamic style rules based on feature properties.", + successCriteria = "Dataset points and polygons display distinct styling according to attribute values.", + failureIndicators = "Dataset ID invalid or attributes fail to filter correctly.", + framework = Framework.KOTLIN_VIEWS +) +// [START maps_android_data_driven_styling_datasets] class DataDrivenDatasetStylingActivity : SamplesBaseActivity(), OnMapReadyCallback, FeatureLayer.OnFeatureClickListener { private lateinit var mapContainer: ViewGroup @@ -88,21 +107,21 @@ class DataDrivenDatasetStylingActivity : SamplesBaseActivity(), OnMapReadyCallba if (dataSets.isEmpty()) { with(dataSets) { put( - getString(com.example.common_ui.R.string.boulder), + getString(R.string.boulder), DataSet( BuildConfig.BOULDER_DATASET_ID, LatLngBounds(LatLng(39.920, -105.340), LatLng(40.090, -105.210)) ) { styleBoulderDataset() } ) put( - getString(com.example.common_ui.R.string.new_york), + getString(R.string.new_york), DataSet( BuildConfig.NEW_YORK_DATASET_ID, LatLngBounds(LatLng(40.7640, -73.9820), LatLng(40.8000, -73.9490)) ) { styleNYCDataset() } ) put( - getString(com.example.common_ui.R.string.kyoto), + getString(R.string.kyoto), DataSet( BuildConfig.KYOTO_DATASET_ID, LatLngBounds(LatLng(34.9700, 135.7200), LatLng(35.0400, 135.8000)) @@ -111,9 +130,9 @@ class DataDrivenDatasetStylingActivity : SamplesBaseActivity(), OnMapReadyCallba } } - setContentView(com.example.common_ui.R.layout.data_driven_styling_demo) + setContentView(R.layout.data_driven_styling_demo) - mapContainer = findViewById(com.example.common_ui.R.id.map_container) + mapContainer = findViewById(R.id.map_container) // --- Programmatically create and add the map fragment --- // 1. Create GoogleMapOptions @@ -127,23 +146,23 @@ class DataDrivenDatasetStylingActivity : SamplesBaseActivity(), OnMapReadyCallba // 4. Add the fragment to your FrameLayout container supportFragmentManager.beginTransaction() - .replace(com.example.common_ui.R.id.map_fragment_container, mapFragment) // Use the container ID from XML + .replace(R.id.map_fragment_container, mapFragment) // Use the container ID from XML .commit() // --- End of programmatic creation --- mapFragment.getMapAsync(this) // Set the click listener for each of the buttons - listOf(com.example.common_ui.R.id.button_kyoto, com.example.common_ui.R.id.button_ny, com.example.common_ui.R.id.button_boulder).forEach { viewId -> + listOf(R.id.button_kyoto, R.id.button_ny, R.id.button_boulder).forEach { viewId -> findViewById"""] + if has_video: + media_tabs_buttons.append(f"""""") + else: + media_tabs_buttons.append(f"""""") + media_tabs_buttons.append(f"""""") + + media_tabs_html = f""" +
+ {''.join(media_tabs_buttons)} +
+ """ + + video_grid_html = "" + if has_video: + java_vid_card = f""" +
+
☕ Java Motion Replay (270x600)
+ +
+ """ if java_vid else "" + + kotlin_vid_card = f""" +
+
💜 Kotlin Motion Replay (270x600)
+ +
+ """ if kotlin_vid else "" + + video_grid_html = f""" + + """ + else: + video_grid_html = f""" + + """ + + search_corpus = f"{title} {category} {' '.join(tags)} {' '.join(api_calls)} {desc}".lower() + if prior_info: + search_corpus += f" {prior_info.get('prior_directive', '')} {prior_info.get('action_taken', '')}".lower() + + existing_notes = html.escape(r.get("operator_notes", "")) + is_op_flagged = bool(r.get("operator_flagged")) or bool(r.get("operator_notes", "").strip()) + existing_checked = 'checked="checked"' if is_op_flagged else "" + saved_status_text = "🚩 Flagged • Saved" if is_op_flagged else "Auto-saved locally" + + substeps = r.get("substep_screenshots", []) + if substeps: + stills_items = [] + for s_item in substeps: + s_label = html.escape(s_item.get("label", "")) + s_j = s_item.get("java") + s_k = s_item.get("kotlin") + stills_items.append(f""" +
+ 📸 Multi-State Capture: {s_label} +
+ """) + if s_j: + stills_items.append(f""" +
+
☕ Java — {s_label}
+ Java - {s_label} +
+ """) + if s_k: + stills_items.append(f""" +
+
💜 Kotlin — {s_label}
+ Kotlin - {s_label} +
+ """) + stills_items.append(f""" +
+ 🏁 Final State +
+
+
☕ Java Implementation (50%)
+ Java Screenshot +
+
+
💜 Kotlin Implementation (50%)
+ Kotlin Screenshot +
+ {defect_card_html} + """) + stills_content_html = "".join(stills_items) + else: + stills_content_html = f""" +
+
☕ Java Implementation (50%)
+ Java Screenshot +
+
+
💜 Kotlin Implementation (50%)
+ Kotlin Screenshot +
+ {defect_card_html} + """ + + ai_attr = f'data-ai-verdict="{ai_v.lower()}"' if ai_v else "" + + return f""" +
+
+
+ #{idx:02d} +

{html.escape(title)}

+ {html.escape(category)} +
+
+ {prior_badge} + {video_badge} + {status_badge} + {ai_badge} +
+
+
+
+
+
Description
+

{html.escape(desc)}

+
+ +
+
🎯 Purpose: {html.escape(purpose)}
+
✅ Success Criteria: {html.escape(success)}
+
+ +
+
Key API Calls
+
+ {api_tags_html or 'Standard SupportMapFragment bindings'} +
+
+ +
+
Tags
+
+ {tags_html} +
+
+ + {prior_directive_html} + {ai_eval_block} + +
+
🤖 Agent Verification Finding
+
{html.escape(notes)}
+
+ +
+
+ ✍️ Operator Feedback & Directives ({html.escape(operator_name)}) +
+ + + + + +
+
+ +
+ {saved_status_text} + +
+
+
+ +
+
+
Visual Verification (Java vs Kotlin)
+ {media_tabs_html} +
+
+ {stills_content_html} +
+ {video_grid_html} + {compare_grid_html} +
+
+
+ """ + + +def generate_html_report( + run_dir: Path, + summary_data: Dict[str, Any], + metadata_by_short: Dict[str, Dict[str, Any]], + root_dir: Optional[Path] = None, +) -> Path: + """Compiles the standalone, interactive HTML review dashboard.""" + if root_dir is None: + root_dir = run_dir.parent.parent + run_id = run_dir.name + results = summary_data.get("results", []) + total = len(results) + passing = sum(1 for r in results if r.get("status") == "PASSING") + needs_work = sum(1 for r in results if r.get("status") == "NEEDS_WORK") + with_video = sum(1 for r in results if r.get("java_video") or r.get("kotlin_video")) + pass_rate = round((passing / total * 100), 1) if total > 0 else 0 + device = summary_data.get("device", "Pixel 6") + + available_runs = get_available_runs(root_dir) + prev_run_id, prev_summary, prev_feedback = load_previous_run_data(root_dir, run_id) + prev_results_by_idx = {pr.get("index"): pr for pr in (prev_summary.get("results", []) if prev_summary else [])} + + # Load CSS and JS + css_content, js_template = load_web_assets(root_dir) + operator_name = os.environ.get("EVAL_OPERATOR") or os.environ.get("USER") or os.environ.get("USERNAME") or "Operator" + embedded_json = json.dumps(results, indent=2).replace("", r"<\/script>") + js_content = ( + f"window.DEFAULT_OPERATOR = {json.dumps(operator_name)};\n" + + js_template.replace("__RUN_ID__", run_id).replace("__RAW_RESULTS__", embedded_json) + ) + + # Check Golden Baseline + golden_path = (root_dir / "eval_runs" / "golden").resolve() if (root_dir / "eval_runs" / "golden").exists() else None + is_golden = bool(golden_path and run_dir.resolve() == golden_path) + golden_badge = '🪨 Certified Bedrock' if is_golden else '' + + # Run Switcher Options + run_options_html = "" + for ar in available_runs: + selected = 'selected="selected"' if ar["id"] == run_id else "" + g_star = "🪨 " if ar["is_golden"] else "" + label = f"{g_star}{ar['id']} ({ar['pass_rate']}%, {ar['passing']}/{ar['total']})" + if available_runs and ar["id"] == available_runs[0]["id"]: + label += " ★ Latest" + run_options_html += f'\n' + + # Prior Directives Summary Table + directives_rows = [] + for d_idx, d in sorted(PRIOR_DIRECTIVES_MAP.items()): + directives_rows.append( + f'' + f' #{d_idx}' + f' "{html.escape(d["prior_directive"])}"' + f' {html.escape(d["action_taken"])}' + f' RESOLVED' + f'' + ) + directives_table = "".join(directives_rows) + + count_with_prior = len(PRIOR_DIRECTIVES_MAP) + + # Render cards + cards_html = [] + for r in results: + idx = r.get("index", 1) + prev_r = prev_results_by_idx.get(idx) + prev_note = prev_feedback.get(idx, "") + directive_entry = PRIOR_DIRECTIVES_MAP.get(idx) + cards_html.append(render_sample_card(r, idx, prev_r, prev_note, directive_entry, run_id=run_id, prev_run_id=prev_run_id)) + + html_content = f""" + + + + + Sample Spelunking — {run_id} + + + +
+
+
+

🗺️ Sample Spelunking: The Spelunker's Logbook

+
+
+ Descent: + + {golden_badge} +
+ • Device: {device} + • Clear Rate: {pass_rate}% ({passing}/{total}) + • Charted Pitfalls: {count_with_prior} tracked +
+
+
+ + + + + + +
+
+
+
+
All Chambers ({total})
+
🔴 Cave-ins / Hazards ({needs_work})
+
🟢 Clear ({passing})
+
🦇 Echolocation Flagged
+
🎬 With Video ({with_video})
+
🎯 Pitfalls Charted ({count_with_prior})
+
✍️ With Notes (0)
+
+
+ + +
+
+
+ +
+
+
+ ⛏️ Pitfalls & Hazards Charted ({count_with_prior} Cleared) + ▼ +
+ +
+ +
+ {"".join(cards_html)} +
+
+ + + +
Saved!
+ + + + +""" + out_file = run_dir / "index.html" + out_file.write_text(html_content, encoding="utf-8") + return out_file + + +build_html = generate_html_report + + +def generate_markdown_summary(run_dir: Path, summary_data: Dict[str, Any], root_dir: Path) -> Path: + """Generates a comprehensive Markdown scorecard and Airing of Grievances summary.""" + run_id = run_dir.name + results = summary_data.get("results", []) + total = len(results) + passing = sum(1 for r in results if r.get("status") == "PASSING") + needs_work = sum(1 for r in results if r.get("status") == "NEEDS_WORK") + with_video = sum(1 for r in results if r.get("java_video") or r.get("kotlin_video")) + pass_rate = round((passing / total * 100), 1) if total > 0 else 0 + device = summary_data.get("device", "Pixel 6") + timestamp = summary_data.get("timestamp", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + + golden_path = (root_dir / "eval_runs" / "golden").resolve() if (root_dir / "eval_runs" / "golden").exists() else None + is_golden = bool(golden_path and run_dir.resolve() == golden_path) + golden_tag = "🪨 **Certified Bedrock Baseline**" if is_golden else "*Candidate Descent*" + + md = [ + f"# 🗺️ Sample Spelunking: The Spelunker's Logbook — `{run_id}`", + f"", + f"**Date:** {timestamp} | **Device:** `{device}` | **Clear Rate:** **{pass_rate}%** ({passing}/{total} chambers clear) | {golden_tag}", + f"", + f"---", + f"", + f"## 📊 Executive Descent Scorecard", + f"", + f"| Metric | Count | Percentage | Status |", + f"| :--- | :---: | :---: | :--- |", + f"| **Total Chambers Explored** | `{total}` | 100% | Complete |", + f"| **🟢 Clear Chambers (Pass)** | `{passing}` | {pass_rate}% | Verified |", + f"| **🔴 Cave-ins / Hazards (Needs Work)** | `{needs_work}` | {round(needs_work/total*100, 1) if total else 0}% | {'Clean!' if needs_work == 0 else 'Action Required'} |", + f"| **🎬 Full-Motion Video Replays** | `{with_video}` | {round(with_video/total*100, 1) if total else 0}% | Motion telemetry active |", + f"| **⛏️ Pitfalls & Hazards Charted** | `{len(PRIOR_DIRECTIVES_MAP)}` | 100% | All grievances resolved |", + f"", + f"---", + f"", + f"## ⛏️ Pitfalls & Hazards Charted ({len(PRIOR_DIRECTIVES_MAP)} Grievances Resolved)", + f"", + f"| # | Chamber | Prior Operator Grievance / Directive | Action Taken & Bedrock Resolution | Status |", + f"| :---: | :--- | :--- | :--- | :---: |", + ] + + for idx, d in sorted(PRIOR_DIRECTIVES_MAP.items()): + sample_name = next((r.get("title", f"Sample #{idx}") for r in results if r.get("index") == idx), f"Sample #{idx}") + dir_text = d["prior_directive"] + act_text = d["action_taken"] + md.append(f"| **#{idx}** | **{sample_name}** | \"{dir_text}\" | {act_text} | `RESOLVED` ✅ |") + + md.extend([ + f"", + f"---", + f"", + f"## 🔦 Chamber-by-Chamber Spelunking Survey", + f"", + f"| # | Chamber | Variant | Status | Video | Key API Calls | Telemetry & Observations |", + f"| :---: | :--- | :---: | :---: | :---: | :--- | :--- |", + ]) + + for r in results: + idx = r.get("index", 1) + title = r.get("title", f"Sample #{idx}") + act_short = (r.get("kotlinActivity") or r.get("id", "")).split(".")[-1] + st = "🟢 CLEAR" if r.get("status") == "PASSING" else "🔴 HAZARD" + vid = "🎬 [Replay](" + r.get("kotlin_video", "") + ")" if r.get("kotlin_video") else "—" + apis = ", ".join(f"`{a}`" for a in r.get("apiCalls", [])[:3]) + if len(r.get("apiCalls", [])) > 3: + apis += f" *(+{len(r.get('apiCalls', [])) - 3} more)*" + obs = r.get("notes", "").replace("\n", " ")[:90] + if len(r.get("notes", "")) > 90: + obs += "..." + md.append(f"| `{idx:02d}` | **{title}** (`{act_short}`) | Java + Kotlin | {st} | {vid} | {apis or 'Standard Bindings'} | {obs} |") + + md.extend([ + f"", + f"---", + f"", + f"## 🎒 Leave No Trace — Phone Hygiene", + f"- All test artifacts on Pixel 6 were captured under `/sdcard/gmp_spelunk_run/`.", + f"- Temporary on-device files automatically wiped clean post-pull.", + f"", + f"---", + f"*Report generated by Sample Spelunking Suite on {timestamp}*", + ]) + + out_file = run_dir / "run_summary.md" + out_file.write_text("\n".join(md), encoding="utf-8") + return out_file + + +def generate_all_reports(run_dir: Path, root_dir: Optional[Path] = None, run_ai_eval: bool = False) -> Tuple[Path, Path, Path]: + """Compiles JSON, Markdown, and HTML reports for a given run.""" + if root_dir is None: + root_dir = run_dir.parent.parent + + summary_json_file = run_dir / "run_summary.json" + if not summary_json_file.exists(): + raise FileNotFoundError(f"Missing run_summary.json in {run_dir}") + + summary_data = json.loads(summary_json_file.read_text(encoding="utf-8")) + + # Multimodal AI Evaluation if requested + if run_ai_eval: + try: + sys.path.append(str(root_dir / "scripts")) + import gemini_eval_engine + print("\n🦇 Initiating Gemini Multimodal Echolocation AI Evaluation...") + summary_data = gemini_eval_engine.evaluate_run(run_dir, summary_data, root_dir) + summary_json_file.write_text(json.dumps(summary_data, indent=2), encoding="utf-8") + except Exception as e: + print(f"⚠️ Warning: Gemini AI evaluation encountered an error: {e}") + + # Load Catalog Metadata + _, metadata_by_short = load_catalog_metadata(root_dir) + + # Compile Markdown and HTML + md_file = generate_markdown_summary(run_dir, summary_data, root_dir) + html_file = generate_html_report(run_dir, summary_data, metadata_by_short, root_dir) + + # Link latest in eval_runs/index.html + latest_html = root_dir / "eval_runs" / "index.html" + try: + if latest_html.exists() or latest_html.is_symlink(): + latest_html.unlink() + latest_html.symlink_to(html_file.relative_to(root_dir / "eval_runs")) + except Exception: + pass + + return summary_json_file, md_file, html_file + + +class ReviewServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, server_address, RequestHandlerClass, run_dir: Path, root_dir: Path): + super().__init__(server_address, RequestHandlerClass) + self.run_dir = Path(run_dir).resolve() + self.root_dir = Path(root_dir).resolve() + + +class ReviewHandler(SimpleHTTPRequestHandler): + """HTTP Request Handler supporting local review server, API endpoints, and byte-range video streaming.""" + + def translate_path(self, path: str) -> str: + run_dir = getattr(self.server, "run_dir", Path.cwd()) + root_dir = getattr(self.server, "root_dir", Path.cwd()) + parsed = urllib.parse.urlparse(path) + clean_path = parsed.path.lstrip("/") + + params = urllib.parse.parse_qs(parsed.query) + if "run" in params: + target_run_id = params["run"][0] + target_run_dir = root_dir / "eval_runs" / target_run_id + if target_run_dir.exists(): + return str(target_run_dir / "index.html") + + if not clean_path or clean_path == "/": + return str(run_dir / "index.html") + + if clean_path.startswith("runs/"): + rel_parts = clean_path.split("/", 2) + if len(rel_parts) >= 3: + target_run = rel_parts[1] + sub_path = rel_parts[2] + return str(root_dir / "eval_runs" / target_run / sub_path) + + if clean_path.startswith("run_") or clean_path.startswith("spelunk_"): + return str(root_dir / "eval_runs" / clean_path) + + if clean_path.startswith("eval_runs/"): + return str(root_dir / clean_path) + + candidate = run_dir / clean_path + if candidate.exists(): + return str(candidate) + + return str(root_dir / "eval_runs" / clean_path) + + def do_GET(self): + parsed = urllib.parse.urlparse(self.path) + if parsed.path == "/api/runs": + root_dir = getattr(self.server, "root_dir", Path.cwd()) + runs = get_available_runs(root_dir) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(json.dumps(runs).encode("utf-8")) + return + return super().do_GET() + + def do_POST(self): + root_dir = getattr(self.server, "root_dir", Path.cwd()) + + if self.path == "/api/save_notes" or self.path == "/api/save_feedback": + content_length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(content_length).decode("utf-8") + data = json.loads(body) + target_run_id = data.get("run_id") + + if target_run_id and (root_dir / "eval_runs" / target_run_id).exists(): + run_dir = root_dir / "eval_runs" / target_run_id + else: + run_dir = getattr(self.server, "run_dir", Path.cwd()) + + out_json = run_dir / "operator_feedback.json" + out_json.write_text(json.dumps(data, indent=2), encoding="utf-8") + + out_md = run_dir / "operator_feedback.md" + with open(out_md, "w", encoding="utf-8") as f: + f.write(f"# ✍️ Operator Feedback & Directives — {data.get('run_id')}\n\n") + for s in data.get("samples", []): + if s.get("operator_notes"): + f.write(f"### {s['title']} (`{s['id']}`)\n") + f.write(f"- **Operator Notes**: {s['operator_notes']}\n") + f.write(f"- **Status**: {s.get('status')}\n\n") + + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(b'{"status":"ok","saved":true}') + print(f"[HTTP] Saved operator feedback to {out_json}") + return + + if self.path == "/api/set_golden": + content_length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(content_length).decode("utf-8") + data = json.loads(body) + target_run_id = data.get("run_id") + + if target_run_id and (root_dir / "eval_runs" / target_run_id).exists(): + target_run_dir = (root_dir / "eval_runs" / target_run_id).resolve() + sys.path.append(str(root_dir / "scripts")) + try: + import gemini_eval_engine + gemini_eval_engine.set_golden_baseline(target_run_dir, root_dir) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(b'{"status":"ok","message":"Golden baseline updated"}') + print(f"[HTTP] Promoted {target_run_id} to Golden Baseline") + return + except Exception as e: + self.send_response(500) + self.send_header("Content-Type", "application/json") + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(json.dumps({"status": "error", "error": str(e)}).encode("utf-8")) + return + else: + self.send_response(400) + self.send_header("Content-Type", "application/json") + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(b'{"status":"error","error":"Invalid run_id"}') + return + + self.send_error(404, "Endpoint not found") + + +def start_review_server(bind_addr: str, requested_port: int, run_dir: Path, root_dir: Path, max_attempts: int = 10) -> Tuple[ReviewServer, int]: + """Starts ReviewServer with graceful port fallback if the requested port is already in use.""" + port = requested_port + for attempt in range(max_attempts): + try: + server = ReviewServer((bind_addr, port), ReviewHandler, run_dir, root_dir) + return server, port + except OSError as e: + if e.errno in (98, 48): # EADDRINUSE on Linux / macOS + print(f"⚠️ Port {port} is already in use. Trying port {port + 1}...") + port += 1 + else: + raise + raise RuntimeError(f"Could not find an available port after {max_attempts} attempts starting from {requested_port}") + + +def main(): + parser = argparse.ArgumentParser(description="Unified QA Report Generator for GMP Android Samples") + parser.add_argument("-r", "--run-dir", help="Target run directory (default: latest in eval_runs/)") + parser.add_argument("--root", help="Root directory of comprehensive-catalog (default: parent of scripts/)") + parser.add_argument("--ai-eval", action="store_true", help="Run Gemini Multimodal evaluation first before compiling reports") + parser.add_argument("--serve", action="store_true", help="Start local HTTP review server") + parser.add_argument("--host", default=None, help="Host to bind the server to (default: 0.0.0.0, binds to all interfaces)") + parser.add_argument("--port", type=int, default=8080, help="Port for HTTP server (default: 8080)") + + args = parser.parse_args() + + if args.root: + root_dir = Path(args.root).resolve() + else: + root_dir = find_repo_root() + + if args.run_dir: + run_dir = Path(args.run_dir).resolve() + else: + eval_runs_dir = root_dir / "eval_runs" + if not eval_runs_dir.exists(): + print(f"❌ Error: {eval_runs_dir} does not exist!") + sys.exit(1) + runs = sorted([d for d in eval_runs_dir.iterdir() if d.is_dir() and (d.name.startswith("run_") or d.name.startswith("spelunk_"))]) + if not runs: + print("❌ Error: No run directories found in eval_runs/!") + sys.exit(1) + run_dir = runs[-1] + + t0 = datetime.datetime.now() + json_f, md_f, html_f = generate_all_reports(run_dir, root_dir, run_ai_eval=args.ai_eval) + elapsed_ms = int((datetime.datetime.now() - t0).total_seconds() * 1000) + + bind_addr = args.host if args.host else "" + display_host = get_display_hostname(args.host) + + print("=" * 75) + print(f"✅ QA Reports Generated in {elapsed_ms}ms!") + print(f"📁 Run: {run_dir.name}") + print(f"📄 Markdown: file://{md_f}") + print(f"🌐 HTML: file://{html_f}") + print(f"🖥️ Local URL: http://{display_host}:{args.port} (when served)") + print("=" * 75) + + if args.serve: + server, actual_port = start_review_server(bind_addr, args.port, run_dir, root_dir) + print(f"\n🚀 Review server listening on http://{display_host}:{actual_port} (bound to {bind_addr or '0.0.0.0'})...") + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nServer stopped.") + + +if __name__ == "__main__": + main() diff --git a/scripts/eval/manage_eval_run.sh b/scripts/eval/manage_eval_run.sh new file mode 100755 index 000000000..3614c4acf --- /dev/null +++ b/scripts/eval/manage_eval_run.sh @@ -0,0 +1,361 @@ +#!/usr/bin/env bash +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== +# manage_eval_run.sh +# ============================================================================== +# +# Operational CLI utility for managing sample evaluation cycles: +# 1. Pulling verification & defect screenshots to workstation. +# 2. Wiping & resetting evaluation runs across device, DB, and UI. +# 3. Annotating defect screenshots with visual bounding boxes & callout badges. +# 4. Recording evaluations directly into the app's Room database via ADB. +# 5. Exporting Markdown Airing of Grievances reports. +# ============================================================================== + +set -euo pipefail + +# ------------------------------------------------------------------------------ +# Terminal Aesthetics & Formatting +# ------------------------------------------------------------------------------ +BOLD="\033[1m" +GREEN="\033[0;32m" +BLUE="\033[0;34m" +YELLOW="\033[0;33m" +RED="\033[0;31m" +CYAN="\033[0;36m" +RESET="\033[0m" + +log_info() { echo -e "${BLUE}${BOLD}[INFO]${RESET} $1"; } +log_success() { echo -e "${GREEN}${BOLD}[SUCCESS]${RESET} $1"; } +log_warn() { echo -e "${YELLOW}${BOLD}[WARN]${RESET} $1"; } +log_error() { echo -e "${RED}${BOLD}[ERROR]${RESET} $1" >&2; } +log_step() { echo -e "\n${CYAN}${BOLD}==>${RESET} ${BOLD}$1${RESET}"; } + +# ------------------------------------------------------------------------------ +# Default Directories & Device Config +# ------------------------------------------------------------------------------ +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -f "${SCRIPT_DIR}/../../settings.gradle.kts" ]]; then + ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" +else + ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +fi + +DEVICE_SCREENSHOT_DIR="/sdcard/gmp_eval_screenshots" +LOCAL_SCREENSHOT_DIR="${EVAL_OUTPUT_DIR:-${ROOT_DIR}/build/reports/eval_screenshots}" + +DEVICE_SERIAL="" +KOTLIN_PKG="com.example.kotlindemos" +JAVA_PKG="com.example.mapdemo" + +# ------------------------------------------------------------------------------ +# Auto-detect Connected ADB Device +# ------------------------------------------------------------------------------ +detect_device() { + if [[ -n "${DEVICE_SERIAL}" ]]; then + return + fi + + local devices=($(adb devices | awk 'NR>1 && $2=="device" {print $1}')) + local count=${#devices[@]} + + if [[ ${count} -eq 0 ]]; then + adb connect localhost:35199 >/dev/null 2>&1 || true + devices=($(adb devices | awk 'NR>1 && $2=="device" {print $1}')) + count=${#devices[@]} + fi + + if [[ ${count} -eq 0 ]]; then + log_error "No active ADB devices or emulators detected! Please connect your device or forward ADB." + exit 1 + elif [[ ${count} -eq 1 ]]; then + DEVICE_SERIAL="${devices[0]}" + else + for dev in "${devices[@]}"; do + if [[ "${dev}" == localhost:* || "${dev}" == 127.0.0.1:* ]]; then + DEVICE_SERIAL="${dev}" + return + fi + done + DEVICE_SERIAL="${devices[0]}" + fi +} + +adb_cmd() { + adb -s "${DEVICE_SERIAL}" "$@" +} + +ensure_dirs() { + mkdir -p "${LOCAL_SCREENSHOT_DIR}" + adb_cmd shell mkdir -p "${DEVICE_SCREENSHOT_DIR}" +} + +# ------------------------------------------------------------------------------ +# Subcommand: Pull Screenshots +# ------------------------------------------------------------------------------ +pull_screenshots() { + local target_dir="${1:-${LOCAL_SCREENSHOT_DIR}}" + mkdir -p "${target_dir}" + log_step "Pulling screenshots from device [${DEVICE_SERIAL}:${DEVICE_SCREENSHOT_DIR}] -> [${target_dir}]" + + adb_cmd shell mkdir -p "${DEVICE_SCREENSHOT_DIR}" + adb_cmd pull "${DEVICE_SCREENSHOT_DIR}/." "${target_dir}/" 2>/dev/null || true + + local count=$(find "${target_dir}" -maxdepth 1 -name "*.png" | wc -l) + log_success "Successfully pulled ${count} screenshot(s) to ${target_dir}." +} + +# ------------------------------------------------------------------------------ +# Subcommand: Reset Evaluation Run +# ------------------------------------------------------------------------------ +reset_run() { + log_step "Resetting evaluation run across device database, filesystem, and UI..." + + # 1. Send broadcast to clear Room DB and app-internal files + log_info "Broadcasting CLEAR_EVALUATIONS to ${KOTLIN_PKG}..." + adb_cmd shell am broadcast -a com.google.maps.CLEAR_EVALUATIONS -p "${KOTLIN_PKG}" >/dev/null || true + + # 2. Wipe device screenshots + log_info "Wiping device screenshot cache at ${DEVICE_SCREENSHOT_DIR}..." + adb_cmd shell rm -rf "${DEVICE_SCREENSHOT_DIR}/*" || true + adb_cmd shell mkdir -p "${DEVICE_SCREENSHOT_DIR}" + + # 3. Wipe local screenshot cache + log_info "Cleaning local screenshot directory: ${LOCAL_SCREENSHOT_DIR}..." + rm -rf "${LOCAL_SCREENSHOT_DIR:?}"/* || true + + # 4. Relaunch ReviewerActivity to refresh live Compose UI + log_info "Relaunching ReviewerActivity to refresh live UI..." + adb_cmd shell am force-stop "${KOTLIN_PKG}" + adb_cmd shell am start -n "${KOTLIN_PKG}/com.example.common_ui.catalog.compose.ReviewerActivity" >/dev/null + sleep 2.0 + + log_success "Evaluation run completely reset! Reviewer catalog is fresh at ⚪ Unchecked (31)." +} + +# ------------------------------------------------------------------------------ +# Subcommand: Annotate Defect Screenshot +# ------------------------------------------------------------------------------ +annotate_defect() { + local in_file="$1" + local box="$2" # "x1,y1,x2,y2" + local label="$3" + local out_file="$4" + + if [[ ! -f "${in_file}" ]]; then + log_error "Input screenshot does not exist: ${in_file}" + exit 1 + fi + + IFS=',' read -r x1 y1 x2 y2 <<< "${box}" + local label_y=$(( y1 > 60 ? y1 - 45 : y2 + 10 )) + local label_y2=$(( label_y + 40 )) + local text_y=$(( label_y + 28 )) + + log_info "Drawing defect annotation on ${in_file} -> ${out_file}" + convert "${in_file}" \ + -stroke "#EF4444" -strokewidth 5 -fill "rgba(239, 68, 68, 0.2)" \ + -draw "rectangle ${x1},${y1} ${x2},${y2}" \ + -stroke none -fill "rgba(220, 38, 38, 0.9)" \ + -draw "roundrectangle ${x1},${label_y} $((x1 + 450)),${label_y2} 8,8" \ + -fill white -pointsize 26 -font DejaVu-Sans-Bold \ + -draw "text $((x1 + 15)),${text_y} '⚠️ ${label}'" \ + "${out_file}" + + local base_name="$(basename "${out_file}")" + adb_cmd push "${out_file}" "${DEVICE_SCREENSHOT_DIR}/${base_name}" >/dev/null || true + log_success "Defect annotation saved locally to ${out_file} and pushed to device." +} + +# ------------------------------------------------------------------------------ +# Subcommand: Record Evaluation via ADB +# ------------------------------------------------------------------------------ +record_evaluation() { + local fqcn="$1" + local status="$2" + local notes="$3" + local screenshot="${4:-}" + + log_step "Recording evaluation for [${fqcn}] -> ${status}" + local dev_screenshot="" + if [[ -n "${screenshot}" ]]; then + if [[ -f "${screenshot}" ]]; then + local base="$(basename "${screenshot}")" + adb_cmd push "${screenshot}" "${DEVICE_SCREENSHOT_DIR}/${base}" >/dev/null || true + dev_screenshot="${DEVICE_SCREENSHOT_DIR}/${base}" + else + dev_screenshot="${screenshot}" + fi + fi + + local b64_notes="$(echo -n "${notes}" | base64 -w 0)" + + adb_cmd shell am broadcast \ + -a com.google.maps.RECORD_EVALUATION \ + -p "${KOTLIN_PKG}" \ + --es fqcn "${fqcn}" \ + --es status "${status}" \ + --es notes_b64 "${b64_notes}" \ + --es screenshot "${dev_screenshot}" >/dev/null + + log_success "Evaluation broadcast dispatched for ${fqcn}." +} + +# ------------------------------------------------------------------------------ +# Subcommand: Capture Sample (Java or Kotlin) +# ------------------------------------------------------------------------------ +capture_sample() { + local sample_class="$1" + local framework="$2" # "java" or "kotlin" + local output_name="${3:-eval_${sample_class}_${framework}.png}" + + local pkg=$([[ "${framework}" == "java" ]] && echo "${JAVA_PKG}" || echo "${KOTLIN_PKG}") + local fqcn="${pkg}.${sample_class}" + local local_file="${LOCAL_SCREENSHOT_DIR}/${output_name}" + + log_step "Launching ${framework} sample: ${fqcn}..." + adb_cmd shell logcat -c + adb_cmd shell am force-stop "${pkg}" + adb_cmd shell am start -n "${pkg}/${fqcn}" \ + --es extra_sample_id "${fqcn}" \ + --ez extra_is_reviewer_mode true >/dev/null + + sleep 3.0 + + log_info "Capturing screenshot -> ${local_file}..." + adb_cmd shell screencap -p "/sdcard/${output_name}" + adb_cmd pull "/sdcard/${output_name}" "${local_file}" >/dev/null + adb_cmd shell cp "/sdcard/${output_name}" "${DEVICE_SCREENSHOT_DIR}/${output_name}" + adb_cmd shell rm "/sdcard/${output_name}" + + log_success "Captured: ${local_file}" +} + +# ------------------------------------------------------------------------------ +# Subcommand: Export Report +# ------------------------------------------------------------------------------ +export_report() { + local out_file="${1:-${ROOT_DIR}/build/reports/latest_evaluation_report.md}" + mkdir -p "$(dirname "${out_file}")" + + log_step "Triggering EXPORT_EVALUATIONS broadcast..." + adb_cmd shell am broadcast -a com.google.maps.EXPORT_EVALUATIONS -p "${KOTLIN_PKG}" >/dev/null + sleep 1.5 + + local report_path="/sdcard/Android/data/${KOTLIN_PKG}/files/reports/latest_evaluation_report.md" + adb_cmd pull "${report_path}" "${out_file}" >/dev/null 2>&1 || true + + if [[ -f "${out_file}" ]]; then + log_success "Exported report pulled successfully to: ${out_file}" + else + log_warn "Report file not immediately at standard path; checking storage..." + fi +} + +# ------------------------------------------------------------------------------ +# CLI Help & Dispatch +# ------------------------------------------------------------------------------ +print_usage() { + cat < [OPTIONS] + +Operational evaluation run manager for Google Maps Platform Android Samples. + +Commands: + --pull-screenshots [dir] Pull all screenshots from device to local directory + --reset Wipe Room DB, clear device/local screenshots, reset UI to 31 unchecked + --annotate-defect Draw defect box and label on screenshot + Required: --input --box --label --output + --record-eval Record evaluation into app's Room DB via broadcast + Required: --fqcn --status --notes [--screenshot ] + --capture-sample Launch and capture screenshot for a sample + Required: --sample --framework [--output ] + --export-report [file] Trigger evaluation report export and pull Markdown file +EOF +} + +main() { + detect_device + ensure_dirs + + if [[ $# -eq 0 ]]; then + print_usage + exit 1 + fi + + case "$1" in + --pull-screenshots) + shift + pull_screenshots "${1:-}" + ;; + --reset) + reset_run + ;; + --annotate-defect) + shift + local in_file="" box="" label="" out_file="" + while [[ $# -gt 0 ]]; do + case "$1" in + --input) in_file="$2"; shift 2 ;; + --box) box="$2"; shift 2 ;; + --label) label="$2"; shift 2 ;; + --output) out_file="$2"; shift 2 ;; + *) shift ;; + esac + done + annotate_defect "${in_file}" "${box}" "${label}" "${out_file}" + ;; + --record-eval) + shift + local fqcn="" status="PASSING" notes="" screenshot="" + while [[ $# -gt 0 ]]; do + case "$1" in + --fqcn) fqcn="$2"; shift 2 ;; + --status) status="$2"; shift 2 ;; + --notes) notes="$2"; shift 2 ;; + --screenshot) screenshot="$2"; shift 2 ;; + *) shift ;; + esac + done + record_evaluation "${fqcn}" "${status}" "${notes}" "${screenshot}" + ;; + --capture-sample) + shift + local sample="" framework="kotlin" out="" + while [[ $# -gt 0 ]]; do + case "$1" in + --sample) sample="$2"; shift 2 ;; + --framework) framework="$2"; shift 2 ;; + --output) out="$2"; shift 2 ;; + *) shift ;; + esac + done + capture_sample "${sample}" "${framework}" "${out}" + ;; + --export-report) + shift + export_report "${1:-}" + ;; + *) + log_error "Unknown command: $1" + print_usage + exit 1 + ;; + esac +} + +main "$@" diff --git a/scripts/eval/run_autonomous_qa_suite.py b/scripts/eval/run_autonomous_qa_suite.py new file mode 100755 index 000000000..cb1acbdfd --- /dev/null +++ b/scripts/eval/run_autonomous_qa_suite.py @@ -0,0 +1,1149 @@ +#!/usr/bin/env python3 +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== +# run_autonomous_qa_suite.py +# ============================================================================== +# +# High-Efficiency Autonomous QA Verification Engine for GMP Android Samples. +# +# Key Architectural Optimizations: +# 1. TWO-PHASE PIPELINE: +# - Phase 1 (Batch Capture): Rapid, uninterrupted capture loop across all 31 samples +# (Java + Kotlin) without intermediate analysis pauses or UI round-trips (~2 mins). +# - Phase 2 (Offline Post-Analysis): Evaluates captured screenshots, inspects logcat +# for crashes/auth errors, detects parity gaps, and generates defect annotations. +# 2. ACTION REPLAY: +# - Pre-programmed action sequences for samples requiring interactions (switches, taps). +# 3. SELF-CONTAINED RUN DIRECTORY HIERARCHY: +# eval_runs/run_/ +# ├── run_summary.md (Scorecard, Matrix, Airing of Grievances) +# ├── run_summary.json (Machine-readable audit findings) +# ├── device_exported_report.md(Exported from on-device Room DB) +# ├── screenshots/ +# │ ├── java/*.png +# │ ├── kotlin/*.png +# │ └── defects/*_defect.png (Annotated defect problem areas) +# └── logs/ +# ├── java/*.logcat +# └── kotlin/*.logcat +# ============================================================================== + +import argparse +import base64 +import datetime +import json +import os +import re +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +BOLD = "\033[1m" +GREEN = "\033[0;32m" +BLUE = "\033[0;34m" +YELLOW = "\033[0;33m" +RED = "\033[0;31m" +CYAN = "\033[0;36m" +MAGENTA = "\033[0;35m" +RESET = "\033[0m" + + +def log_info(msg): + print(f"{BLUE}{BOLD}[INFO]{RESET} {msg}") + + +def log_success(msg): + print(f"{GREEN}{BOLD}[PASS]{RESET} {msg}") + + +def log_warn(msg): + print(f"{YELLOW}{BOLD}[WARN]{RESET} {msg}") + + +def log_defect(msg): + print(f"{RED}{BOLD}[NEEDS WORK]{RESET} {msg}") + + +def log_step(msg): + print(f"\n{CYAN}{BOLD}==>{RESET} {BOLD}{msg}{RESET}") + + +# Optional action sequences to replay on specific samples before taking screenshots +# Format: "SampleClass": [("tap", x, y, wait_sec), ("swipe", x1, y1, x2, y2, wait_sec), ("rotate", degrees, wait_sec)] +SAMPLE_ACTIONS = { + # 04 Retained Map: Force configuration change via device rotation to verify map state is preserved + "RetainMapDemoActivity": [ + ("rotate", 1, 1.8), # Rotate to landscape (90 deg) + ("rotate", 0, 1.8), # Rotate back to portrait (0 deg) + ], + # 05 Multi-Map View: Simultaneous 4-way animated camera zoom across UNESCO heritage sites + "MultiMapDemoActivity": [ + ("wait", 3.5), + ], + # 06 Map in ViewPager: Swipe across pages to verify page transitions and touch disallow + "MapInPagerDemoActivity": [ + ("swipe", 950, 1200, 100, 1200, 0.8), # Swipe to Page 1 + ("swipe", 950, 1200, 100, 1200, 0.8), # Swipe to Page 2 (Map Fragment) + ("swipe", 100, 1200, 950, 1200, 0.8), # Swipe back to Page 1 + ("swipe", 950, 1200, 100, 1200, 1.5), # Swipe to Map page and settle + ], + # 07 Camera Controls: Animated camera to Bondi, Sydney, zoom in, tilt more, map pan/sweep, stop animation callback + "CameraDemoActivity": [ + ("tap", 810, 716, 2.0), # Tap "Go to Bondi" -> animated camera + ("screenshot", "Bondi Beach", 0.5), + ("tap", 857, 338, 0.8), # Tap Zoom In (+) + ("tap", 1006, 338, 0.8), # Tap Tilt More (towards 45 deg) + ("swipe", 540, 1400, 540, 900, 0.8), # Pan map northward + ("swipe", 200, 1100, 900, 1300, 1.0), # Diagonal sweep to spin/rotate bearing + ("screenshot", "Zoomed and Tilted", 0.5), + ("tap", 270, 716, 0.3), # Start animating to Sydney + ("tap", 115, 338, 1.5), # Tap Stop Animation button (■) to trigger cancel callback! + ], + # 08 Camera Clamping: Exercise zoom limits slider, zoom map, and test bounds clamps + "CameraClampingDemoActivity": [ + ("swipe", 250, 440, 500, 440, 0.8), # Drag min zoom thumb to higher zoom + ("swipe", 540, 1400, 540, 1100, 0.8), # Pan within limits + ("swipe", 850, 440, 600, 440, 0.8), # Drag max zoom thumb to lower zoom + ("tap", 900, 340, 0.8), # Tap "Reset Zoom Limits" button + ("tap", 180, 580, 1.0), # Tap "Adelaide" clamp toggle button + ("swipe", 540, 1600, 540, 1000, 0.8), # Drag map northward against clamped bounds + ("tap", 540, 580, 1.0), # Tap "Pacific" clamp toggle button + ("swipe", 540, 1600, 540, 1000, 0.8), # Drag map against pacific bounds + ], + # 09 Visible Region & Projection: Tap Actions button on telemetry card to open PopupMenu and animate camera + "VisibleRegionDemoActivity": [ + ("wait", 1.0), + ("screenshot", "SFO Airport Initial", 0.5), + ("tap", 539, 529, 1.0), # Tap "Actions ▾" button on telemetry card + ("tap", 660, 929, 2.5), # Tap "Move to Sydney Opera House" in popup + ("screenshot", "Sydney Opera House", 0.5), + ("tap", 539, 529, 1.0), # Tap "Actions ▾" button again + ("tap", 660, 1181, 2.5), # Tap "Fit Australia Bounds" in popup + ("screenshot", "Australia Bounds", 0.5), + ("swipe", 540, 1400, 540, 900, 1.2), # Pan map northward -> dynamic telemetry update + ], + # 10 Advanced Markers: Tap pins to open info windows, exercise collision behavior with zoom + "AdvancedMarkersDemoActivity": [ + ("tap", 260, 1100, 1.2), # Tap pin near Singapore to open info window + ("tap", 350, 950, 1.2), # Tap pin near Kuala Lumpur + ("tap", 450, 1350, 1.2), # Tap pin near Jakarta + ("tap", 540, 1200, 0.1), # Double tap to zoom in + ("tap", 540, 1200, 1.8), # Zoom in animation settles and collision adapts + ("swipe", 540, 1000, 540, 1500, 1.0), # Pan south to inspect clustering collision + ], + # 11 Standard Markers: Rotation slider, flat toggle, marker info windows + "MarkerDemoActivity": [ + ("wait", 1.0), + ("screenshot", "Default Info Window", 0.5), + ("swipe", 350, 498, 900, 498, 1.0), # Drag rotation seekbar to rotate markers + ("tap", 304, 396, 0.8), # Toggle "Flat to map surface" checkbox + ("screenshot", "Rotated Flat Markers", 0.5), + ("tap", 278, 2190, 0.8), # Select "Custom info contents" radio button + ("tap", 780, 1370, 1.2), # Tap Sydney marker (custom contents) + ("screenshot", "Custom Info Contents", 0.5), + ("tap", 266, 2316, 0.8), # Select "Custom info window" radio button + ("tap", 450, 1420, 1.2), # Tap Adelaide marker (custom info window) + ("screenshot", "Custom Info Window", 0.5), + ], + # 12 Marker Retap Toggle: First tap opens InfoWindow, second tap dismisses + "MarkerCloseInfoWindowOnRetapDemoActivity": [ + ("tap", 750, 1470, 1.5), # Tap Sydney marker to open info window + ("tap", 750, 1470, 1.5), # Re-tap Sydney marker to dismiss info window + ], + # 13 Polygon Styling: Adjust Fill Hue, Fill Alpha, and Stroke Width seekbars, test click + "PolygonDemoActivity": [ + ("swipe", 300, 330, 850, 330, 1.0), # Swipe fill hue seekbar + ("swipe", 300, 410, 850, 410, 1.0), # Swipe fill alpha seekbar + ("swipe", 300, 490, 850, 490, 1.0), # Swipe stroke width seekbar + ("swipe", 300, 570, 850, 570, 1.0), # Swipe stroke hue seekbar + ("tap", 100, 650, 0.8), # Toggle clickable checkbox + ("tap", 540, 1300, 1.0), # Tap polygon on map to verify click toast + ], + # 14 Polyline Styling: Adjust Hue slider (y=270), Alpha slider, Width slider, and joint/cap controls + "PolylineDemoActivity": [ + ("swipe", 300, 270, 850, 270, 1.0), # Swipe Hue slider (y=270) + ("swipe", 300, 350, 850, 350, 1.0), # Swipe Alpha slider + ("swipe", 300, 430, 850, 430, 1.0), # Swipe Width slider + ("tap", 300, 530, 0.8), # Tap Joint type spinner + ("tap", 300, 680, 1.0), # Select Round joint + ("tap", 750, 530, 0.8), # Tap Cap type spinner + ("tap", 750, 680, 1.0), # Select Round cap + ("tap", 100, 750, 0.8), # Toggle clickable checkbox + ], + # 15 Circle Styling: Adjust fill alpha and stroke width sliders + "CircleDemoActivity": [ + ("swipe", 300, 420, 800, 420, 1.0), # Swipe fill alpha seekbar + ("swipe", 300, 490, 800, 490, 1.0), # Swipe stroke width seekbar + ], + # 16 Data-Driven Boundaries: Multi-state capture (Hawaii Locality -> US State -> Selected Boundary) + "DataDrivenBoundariesActivity": [ + ("wait", 3.0), + ("screenshot", "Hawaii Locality Boundaries", 0.5), + ("tap", 472, 391, 3.5), # Tap "US" button to center on USA and render state boundaries + ("screenshot", "US State Boundaries", 0.5), + ("tap", 480, 1450, 2.0), # Tap on US state to trigger boundary click styling + ("screenshot", "Selected State Boundary", 0.5), + ("tap", 205, 391, 2.5), # Tap "Hawaii" button to return to Hawaii + ("swipe", 540, 1400, 540, 1000, 1.0), # Pan map + ], + # 17 Data-Driven Dataset Styling: Multi-state capture (Boulder -> New York -> Kyoto) + "DataDrivenDatasetStylingActivity": [ + ("wait", 3.0), + ("screenshot", "Boulder Dataset", 0.5), + ("tap", 560, 391, 3.5), # Tap "New York" button to center and style Central Park dataset + ("screenshot", "New York Dataset", 0.5), + ("tap", 856, 391, 3.5), # Tap "Kyoto" button to center and style Kyoto dataset + ("screenshot", "Kyoto Dataset", 0.5), + ("tap", 244, 391, 3.0), # Tap "Boulder" button to return to Boulder dataset + ("swipe", 540, 1400, 540, 1000, 1.0), # Pan map + ], + # 18 Cloud-Based Map Styling: Mont Blanc center + Multi-state capture (Normal, Satellite, Hybrid, Terrain) + "CloudBasedMapStylingDemoActivity": [ + ("wait", 2.0), + ("screenshot", "Normal Style", 0.5), + ("tap", 435, 2324, 2.5), # Tap "Satellite" button + ("screenshot", "Satellite Style", 0.5), + ("tap", 716, 2324, 2.5), # Tap "Hybrid" button + ("screenshot", "Hybrid Style", 0.5), + ("tap", 967, 2324, 2.5), # Tap "Terrain" button + ("swipe", 540, 1400, 540, 1000, 1.0), # Pan map to view terrain topography + ("screenshot", "Terrain Style", 0.5), + ("tap", 147, 2324, 2.0), # Tap "Normal" button to reset + ], + # 20 Map Color Scheme: Multi-state capture (Initial -> Light -> Dark -> System) + "MapColorSchemeActivity": [ + ("wait", 2.0), + ("screenshot", "Initial Scheme", 0.5), + ("tap", 105, 338, 2.0), # Tap Light mode button + ("screenshot", "Light Mode", 0.5), + ("tap", 294, 338, 2.0), # Tap Dark mode button + ("screenshot", "Dark Mode", 0.5), + ("tap", 578, 338, 2.0), # Tap Follow System button + ("screenshot", "Follow System Mode", 0.5), + ("swipe", 540, 1400, 540, 1000, 1.0), # Pan map to show rendered tiles + ], + # 22 Lite Mode Basics: Exercise Darwin, Adelaide, and Australia buttons + "LiteDemoActivity": [ + ("tap", 250, 400, 1.5), # Tap Go to Darwin + ("tap", 250, 520, 1.5), # Tap Go to Adelaide + ("tap", 250, 640, 1.5), # Tap Go to Australia + ], + # 23 Snapshot: Tap screenshot button to capture live map bitmap into snapshot preview holder + "SnapshotDemoActivity": [ + ("tap", 270, 2300, 2.0), # Tap "Take Snapshot" button and wait for bitmap + ], + # 25 Tile Overlay: Toggle fade-in, swipe transparency slider, and pan tile coordinates + "TileOverlayDemoActivity": [ + ("wait", 1.0), + ("screenshot", "Initial Moon Tiles", 0.5), + ("tap", 912, 338, 0.8), # Toggle "Fade In Tiles" checkbox + ("swipe", 780, 482, 920, 482, 1.2), # Drag transparency seekbar to ~50% + ("screenshot", "Semi-Transparent Tiles", 0.5), + ("swipe", 920, 482, 780, 482, 1.0), # Drag transparency back to 0% + ("swipe", 800, 1200, 200, 1200, 1.2), # Pan map eastward to load new coordinates + ("swipe", 540, 1500, 540, 900, 1.2), # Pan map northward to load new coordinates + ], + # 26 UI Settings: Toggle map controls, test disabled scroll vs re-enabled, and zoom buttons + "UiSettingsDemoActivity": [ + ("tap", 120, 1760, 0.8), # Toggle zoom buttons + ("tap", 120, 1850, 0.8), # Toggle compass + ("swipe", 200, 2100, 200, 1750, 0.8), # Scroll down controls card + ("tap", 120, 1800, 0.8), # Toggle scroll gestures OFF + ("swipe", 540, 1200, 540, 800, 0.8), # Attempt pan (blocked!) + ("tap", 120, 1800, 0.8), # Toggle scroll gestures ON + ("swipe", 540, 1200, 540, 800, 1.0), # Pan map (smoothly moves!) + ("swipe", 200, 1750, 200, 2100, 0.8), # Scroll back up controls card + ("tap", 1000, 1500, 1.0), # Tap Zoom In (+) button on map + ], + # 27 LocationSource: GPX Track Simulation (Fowler / Rattlesnake trail) + "LocationSourceDemoActivity": [ + ("wait", 3.0), # Observe initial animation along Fowler / Rattlesnake trail + ("tap", 280, 2250, 1.2), # Tap "Pause" button on trail telemetry card + ("tap", 280, 2250, 1.5), # Tap "Play" button to resume GPS simulation + ("tap", 800, 2250, 2.0), # Tap "Fit Trail" to re-center camera bounds + ("wait", 2.0), # Capture continued blue dot motion along polyline + ], + # 28 Ground Overlays: Move transparency slider, switch image to 1922 map, click overlay + "GroundOverlayDemoActivity": [ + ("wait", 1.0), + ("screenshot", "Initial Overlay (Modern Newark)", 0.5), + ("swipe", 450, 367, 950, 367, 1.2), # Drag transparency seekbar (y=367) + ("screenshot", "Transparent Overlay", 0.5), + ("tap", 268, 481, 1.5), # Tap "Switch Image" button (1922 historical map) + ("screenshot", "Historical 1922 Overlay", 0.5), + ("swipe", 540, 1400, 540, 1000, 1.0), # Pan map across historical Newark overlay + ("tap", 540, 1200, 1.2), # Tap on ground overlay image to verify click listener + ("swipe", 950, 367, 400, 367, 1.2), # Drag transparency seekbar back + ], + # 30 Events & Gestures: Multi-touch tap, drag, double-tap zoom, and bearing rotation + "EventsDemoActivity": [ + ("tap", 540, 1300, 1.0), # Tap map for single click event + ("swipe", 540, 1500, 540, 1500, 1.2), # Long press map for long click event + ("swipe", 540, 1600, 540, 1000, 1.0), # Pan map northward -> updates camera HUD + ("tap", 540, 1200, 0.1), # Double tap to zoom + ("tap", 540, 1200, 1.2), # Zoom updates HUD + ("swipe", 200, 1200, 850, 1350, 1.2), # Diagonal swipe to rotate bearing angle in HUD + ], + # 31 My Location: Tap My Location GPS button + "MyLocationDemoActivity": [ + ("tap", 975, 355, 1.8), # Tap My Location GPS button accurately in top-right map corner + ], +} + + +def find_repo_root(start_path: Optional[Path] = None) -> Path: + """Finds repository root by searching upwards for settings.gradle.kts or .git.""" + curr = (start_path or Path(__file__)).resolve() + for p in [curr] + list(curr.parents): + if (p / "settings.gradle.kts").exists() or (p / ".git").exists(): + return p + return curr.parent.parent.parent + + +class AutonomousQaRunner: + + def __init__(self, args): + self.args = args + self.root_dir = find_repo_root() + self.device_serial = args.device or self.detect_device() + self.tag = f"spelunk_{datetime.datetime.now().strftime('%Yy%mm%dd_%Hh%Mm%Ss')}" + self.timestamp = self.tag.replace("spelunk_", "").replace("run_", "") + + if args.output_dir: + self.run_dir = Path(args.output_dir).resolve() + else: + self.run_dir = self.root_dir / "eval_runs" / self.tag + + self.screenshots_dir = self.run_dir / "screenshots" + self.java_screenshots_dir = self.screenshots_dir / "java" + self.kotlin_screenshots_dir = self.screenshots_dir / "kotlin" + self.defects_dir = self.screenshots_dir / "defects" + + self.videos_dir = self.run_dir / "videos" + self.java_videos_dir = self.videos_dir / "java" + self.kotlin_videos_dir = self.videos_dir / "kotlin" + + self.logs_dir = self.run_dir / "logs" + self.java_logs_dir = self.logs_dir / "java" + self.kotlin_logs_dir = self.logs_dir / "kotlin" + + # Unified single directory on device for entire spelunking descent + self.device_root_dir = "/sdcard/gmp_spelunk_run" + self.device_run_dir = f"{self.device_root_dir}/{self.tag}" + self.device_screenshots_dir = f"{self.device_run_dir}/screenshots" + self.device_videos_dir = f"{self.device_run_dir}/videos" + self.device_screenshot_dir = self.device_screenshots_dir # backward compat alias + + self.kotlin_pkg = "com.example.kotlindemos" + self.java_pkg = "com.example.mapdemo" + + self.capture_data = [] + self.results = [] + + def init_filesystem(self): + for p in [ + self.java_screenshots_dir, + self.kotlin_screenshots_dir, + self.defects_dir, + self.java_logs_dir, + self.kotlin_logs_dir, + self.java_videos_dir, + self.kotlin_videos_dir, + ]: + p.mkdir(parents=True, exist_ok=True) + + self.adb_run(["shell", "mkdir", "-p", self.device_screenshots_dir]) + self.adb_run(["shell", "mkdir", "-p", self.device_videos_dir]) + + # Pre-grant location permissions for both apps so GPS and LocationSource samples work seamlessly + for pkg in [self.kotlin_pkg, self.java_pkg]: + for perm in [ + "android.permission.ACCESS_FINE_LOCATION", + "android.permission.ACCESS_COARSE_LOCATION" + ]: + self.adb_run(["shell", "pm", "grant", pkg, perm], check=False) + + def detect_device(self): + cmd = ["adb", "devices"] + res = subprocess.run(cmd, capture_output=True, text=True) + lines = res.stdout.strip().splitlines() + devices = [] + for line in lines[1:]: + parts = line.split() + if len(parts) >= 2 and parts[1] == "device": + devices.append(parts[0]) + + if not devices: + subprocess.run(["adb", "connect", "localhost:35199"], capture_output=True) + res = subprocess.run(cmd, capture_output=True, text=True) + for line in res.stdout.strip().splitlines()[1:]: + parts = line.split() + if len(parts) >= 2 and parts[1] == "device": + devices.append(parts[0]) + + if not devices: + log_defect("No connected ADB devices detected!") + sys.exit(1) + + for d in devices: + if d.startswith("localhost:") or d.startswith("127.0.0.1:"): + log_info(f"Targeting forwarded ADB device: {d}") + return d + + log_info(f"Targeting ADB device: {devices[0]}") + return devices[0] + + def adb_run(self, cmd_args, check=True): + full_cmd = ["adb", "-s", self.device_serial] + cmd_args + return subprocess.run(full_cmd, capture_output=True, text=True, check=check) + + def load_catalog_samples(self): + registry_file = self.root_dir / "ApiDemos/project/common-ui/src/main/java/com/example/common_ui/catalog/SampleCatalogRegistry.kt" + if not registry_file.exists(): + log_defect(f"Registry file not found: {registry_file}") + sys.exit(1) + + with open(registry_file, "r", encoding="utf-8") as f: + text = f.read() + + blocks = re.split(r"SampleItem\s*\(", text)[1:] + samples = [] + + for block in blocks: + def get_str(field): + m = re.search(rf"{field}\s*=\s*\"([^\"]+)\"", block) + return m.group(1) if m else "" + + def get_tags(): + m = re.search(r"tags\s*=\s*listOf\s*\((.*?)\)", block, re.DOTALL) + return re.findall(r"\"([^\"]+)\"", m.group(1)) if m else [] + + def get_api_calls(): + m = re.search(r"apiCalls\s*=\s*listOf\s*\((.*?)\),\s*(?:purpose|successCriteria|kotlinActivity)", block, re.DOTALL) + return re.findall(r"\"([^\"]+)\"", m.group(1)) if m else [] + + sample = { + "id": get_str("id"), + "title": get_str("title"), + "description": get_str("description"), + "category": get_str("category"), + "purpose": get_str("purpose"), + "successCriteria": get_str("successCriteria"), + "failureIndicators": get_str("failureIndicators"), + "kotlinActivity": get_str("kotlinActivity"), + "javaActivity": get_str("javaActivity"), + "tags": get_tags(), + "apiCalls": get_api_calls(), + } + if sample["id"] and sample["kotlinActivity"]: + samples.append(sample) + + return samples + + def reset_eval_state(self): + log_step("Resetting device evaluation state for clean unattended run...") + self.adb_run(["shell", "am", "broadcast", "-a", "com.google.maps.CLEAR_EVALUATIONS", "-p", self.kotlin_pkg], check=False) + time.sleep(1.0) + self.adb_run(["shell", "rm", "-rf", f"{self.device_screenshot_dir}/*"], check=False) + self.adb_run(["shell", "am", "force-stop", self.kotlin_pkg], check=False) + self.adb_run(["shell", "am", "force-stop", self.java_pkg], check=False) + log_success("Device database and screenshot cache cleared.") + + # -------------------------------------------------------------------------- + # PHASE 1: Rapid Uninterrupted Batch Capture Loop + # -------------------------------------------------------------------------- + def capture_single_framework(self, sample, framework): + pkg = self.java_pkg if framework == "java" else self.kotlin_pkg + activity_fqcn = sample["javaActivity"] if framework == "java" else sample["kotlinActivity"] + short_name = activity_fqcn.split(".")[-1] + + self.adb_run(["shell", "logcat", "-c"], check=False) + self.adb_run(["shell", "am", "force-stop", pkg], check=False) + + start_cmd = [ + "shell", "am", "start", "-n", f"{pkg}/{activity_fqcn}", + "--es", "extra_sample_id", sample["id"] + ] + self.adb_run(start_cmd, check=False) + + # Determine if video recording should be performed for interactive samples + is_interactive = short_name in SAMPLE_ACTIONS + record_video = is_interactive and not getattr(self.args, "no_video", False) + rec_proc = None + device_mp4 = f"/sdcard/eval_{short_name}_{framework}.mp4" + + if record_video: + self.adb_run(["shell", "rm", "-f", device_mp4], check=False) + + # Allow initial render + if short_name == "MultiMapDemoActivity": + # MultiMap animates 4 maps simultaneously starting on ready; record immediately with warm-up + if record_video: + rec_cmd = [ + "adb", "-s", self.device_serial, "shell", + "screenrecord", "--size", getattr(self.args, "video_size", "270x600"), + "--bit-rate", str(getattr(self.args, "video_bitrate", 1500000)), + "--time-limit", "15", device_mp4 + ] + rec_proc = subprocess.Popen(rec_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + time.sleep(1.0) + # Extra settle for concurrent tile loading + 3000ms simultaneous zoom animation + time.sleep(self.args.settle_time + 2.0) + else: + settle = self.args.settle_time + if short_name in ("DataDrivenBoundariesActivity", "DataDrivenDatasetStylingActivity"): + settle = max(settle, 8.0) + time.sleep(settle) + if record_video: + rec_cmd = [ + "adb", "-s", self.device_serial, "shell", + "screenrecord", "--size", getattr(self.args, "video_size", "270x600"), + "--bit-rate", str(getattr(self.args, "video_bitrate", 1500000)), + "--time-limit", "45", device_mp4 + ] + rec_proc = subprocess.Popen(rec_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + time.sleep(1.0) + + # Replay pre-programmed actions if specified + substeps = [] + if short_name in SAMPLE_ACTIONS and short_name != "MultiMapDemoActivity": + for act in SAMPLE_ACTIONS[short_name]: + if act[0] == "tap": + _, x, y, wait_s = act + self.adb_run(["shell", "input", "tap", str(x), str(y)], check=False) + time.sleep(wait_s) + elif act[0] == "swipe": + _, x1, y1, x2, y2, wait_s = act + self.adb_run(["shell", "input", "swipe", str(x1), str(y1), str(x2), str(y2), "250"], check=False) + time.sleep(wait_s) + elif act[0] == "rotate": + _, rot, wait_s = act + self.adb_run(["shell", "settings", "put", "system", "accelerometer_rotation", "0"], check=False) + self.adb_run(["shell", "settings", "put", "system", "user_rotation", str(rot)], check=False) + time.sleep(wait_s) + elif act[0] == "wait": + _, wait_s = act + time.sleep(wait_s) + elif act[0] == "screenshot": + _, label, wait_s = act + time.sleep(wait_s) + slug = re.sub(r'[^a-zA-Z0-9_-]', '_', label.lower()).strip('_') + sub_device_png = f"/sdcard/eval_{short_name}_{framework}_{slug}.png" + self.adb_run(["shell", "screencap", "-p", sub_device_png], check=False) + sub_local_png = (self.java_screenshots_dir if framework == "java" else self.kotlin_screenshots_dir) / f"{short_name}_{slug}.png" + self.adb_run(["pull", sub_device_png, str(sub_local_png)], check=False) + if hasattr(self.args, "scale") and self.args.scale and 0 < self.args.scale < 1.0: + pct = int(self.args.scale * 100) + subprocess.run(["convert", str(sub_local_png), "-resize", f"{pct}%", str(sub_local_png)], check=False) + self.adb_run(["shell", "rm", "-f", sub_device_png], check=False) + substeps.append({ + "label": label, + "rel_path": f"screenshots/{framework}/{short_name}_{slug}.png" + }) + if short_name == "RetainMapDemoActivity": + self.adb_run(["shell", "settings", "put", "system", "user_rotation", "0"], check=False) + time.sleep(0.5) + + # Capture final still screenshot + device_png = f"/sdcard/eval_{short_name}_{framework}.png" + self.adb_run(["shell", "screencap", "-p", device_png], check=False) + + local_png = (self.java_screenshots_dir if framework == "java" else self.kotlin_screenshots_dir) / f"{short_name}.png" + self.adb_run(["pull", device_png, str(local_png)], check=False) + + # Downscale still screenshot by scale factor (default 50% = 0.5 in both dimensions) + if hasattr(self.args, "scale") and self.args.scale and 0 < self.args.scale < 1.0: + pct = int(self.args.scale * 100) + subprocess.run(["convert", str(local_png), "-resize", f"{pct}%", str(local_png)], check=False) + + self.adb_run(["shell", "cp", device_png, f"{self.device_screenshot_dir}/eval_{short_name}_{framework}.png"], check=False) + self.adb_run(["shell", "rm", device_png], check=False) + + # Finalize screen recording cleanly + video_rel_path = None + video_size_kb = 0 + if rec_proc: + time.sleep(1.2) # Settle time to flush trailing frames before stopping screenrecord + self.adb_run(["shell", "pkill", "-2", "-x", "screenrecord"], check=False) + try: + rec_proc.wait(timeout=4) + except Exception: + rec_proc.kill() + time.sleep(0.5) + + raw_mp4 = (self.java_videos_dir if framework == "java" else self.kotlin_videos_dir) / f"{short_name}_raw.mp4" + clean_mp4 = (self.java_videos_dir if framework == "java" else self.kotlin_videos_dir) / f"{short_name}.mp4" + self.adb_run(["pull", device_mp4, str(raw_mp4)], check=False) + self.adb_run(["shell", "rm", "-f", device_mp4], check=False) + + if raw_mp4.exists() and raw_mp4.stat().st_size > 1000: + # Faststart remux to ensure clean web browser playback and strip unused metadata + subprocess.run( + ["ffmpeg", "-y", "-i", str(raw_mp4), "-c:v", "copy", "-an", "-movflags", "+faststart", str(clean_mp4)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False + ) + if clean_mp4.exists() and clean_mp4.stat().st_size > 1000: + raw_mp4.unlink(missing_ok=True) + video_rel_path = f"videos/{framework}/{short_name}.mp4" + video_size_kb = clean_mp4.stat().st_size / 1024 + elif raw_mp4.exists(): + raw_mp4.rename(clean_mp4) + video_rel_path = f"videos/{framework}/{short_name}.mp4" + video_size_kb = clean_mp4.stat().st_size / 1024 + + # Capture logcat + log_file = (self.java_logs_dir if framework == "java" else self.kotlin_logs_dir) / f"{short_name}.logcat" + logcat_res = self.adb_run(["shell", "logcat", "-d", "-v", "time"], check=False) + with open(log_file, "w", encoding="utf-8") as f: + f.write(logcat_res.stdout) + + # Analyze logcat for errors + crashes = [] + auth_errors = [] + for line in logcat_res.stdout.splitlines(): + if ("FATAL EXCEPTION" in line and pkg in line) or ("AndroidRuntime: FATAL" in line and pkg in line) or (f"Process {pkg}" in line and " died" in line): + crashes.append(line.strip()) + if "Authorization failure" in line or ("Google Maps Android API" in line and "Ensure that the following" in line): + auth_errors.append(line.strip()) + if "code 230" in line or "ERR_DIFFERENT_APP_OR_KEY" in line: + auth_errors.append(line.strip()) + + file_size_kb = local_png.stat().st_size / 1024 if local_png.exists() else 0 + + return { + "framework": framework, + "short_name": short_name, + "activity_fqcn": activity_fqcn, + "screenshot_path": local_png, + "file_size_kb": file_size_kb, + "substeps": substeps, + "video_path": clean_mp4 if video_rel_path else None, + "video_rel_path": video_rel_path, + "video_size_kb": video_size_kb, + "logcat_path": log_file, + "crashes": crashes, + "auth_errors": auth_errors, + } + + def phase_batch_capture(self, samples): + total = len(samples) + log_step(f"PHASE 1: Starting Rapid Uninterrupted Batch Capture ({total} samples, {total * 2} runs)...") + start_time = time.time() + + for idx, sample in enumerate(samples, start=1): + short_name = sample["kotlinActivity"].split(".")[-1] + sys.stdout.write(f"\r{BLUE}{BOLD}[Capture {idx:2d}/{total:2d}]{RESET} {sample['title'][:32]:<32} (Java + Kotlin)...") + sys.stdout.flush() + + # Execute Java and Kotlin in rapid succession + java_data = self.capture_single_framework(sample, "java") + kotlin_data = self.capture_single_framework(sample, "kotlin") + + self.capture_data.append({ + "index": idx, + "sample": sample, + "java": java_data, + "kotlin": kotlin_data, + }) + + duration = time.time() - start_time + print(f"\n{GREEN}{BOLD}[SUCCESS]{RESET} Batch capture finished in {duration:.1f}s ({duration / (total * 2):.2f}s/run). All {total * 2} screenshots & logs on disk.") + + # -------------------------------------------------------------------------- + # PHASE 2: Offline Post-Analysis & Defect Annotation + # -------------------------------------------------------------------------- + def annotate_defect(self, in_file, box, label, out_file): + scale = getattr(self.args, "scale", 1.0) + if scale is None or scale <= 0 or scale > 1.0: + scale = 1.0 + x1 = int(box[0] * scale) + y1 = int(box[1] * scale) + x2 = int(box[2] * scale) + y2 = int(box[3] * scale) + badge_h = int(36 * max(0.6, scale)) + label_y = y1 - badge_h - 5 if y1 > (badge_h + 15) else y2 + 10 + label_y2 = label_y + badge_h + text_y = label_y + int(24 * max(0.6, scale)) + badge_w = int(460 * max(0.6, scale)) + font_size = int(22 * max(0.6, scale)) + stroke_w = max(2, int(4 * scale)) + + cmd = [ + "convert", str(in_file), + "-stroke", "#EF4444", "-strokewidth", str(stroke_w), "-fill", "rgba(239, 68, 68, 0.2)", + "-draw", f"rectangle {x1},{y1} {x2},{y2}", + "-stroke", "none", "-fill", "rgba(220, 38, 38, 0.9)", + "-draw", f"roundrectangle {x1},{label_y} {x1 + badge_w},{label_y2} 6,6", + "-fill", "white", "-pointsize", str(font_size), "-font", "DejaVu-Sans-Bold", + "-draw", f"text {x1 + 10},{text_y} '⚠️ {label}'", + str(out_file) + ] + subprocess.run(cmd, check=True) + + base_name = out_file.name + self.adb_run(["push", str(out_file), f"{self.device_screenshot_dir}/{base_name}"], check=False) + + def check_map_tiles_loaded(self, png_path): + """ + Detects if Google Maps vector tiles failed to load (canvas remains unrendered placeholder #F0EDE5). + Returns True if tiles loaded, False if blank placeholder canvas. + """ + if not png_path or not Path(png_path).exists(): + return True + try: + scale = getattr(self.args, "scale", 1.0) or 1.0 + crop_h = int(600 * scale) + crop_y = int(200 * scale) + cmd = [ + "convert", str(png_path), + "-crop", f"0x{crop_h}+0+{crop_y}", "+repage", + "-fuzz", "3%", + "-fill", "black", "+opaque", "rgb(240,237,229)", + "-fill", "white", "-opaque", "rgb(240,237,229)", + "-format", "%[mean]", "info:" + ] + res = subprocess.run(cmd, capture_output=True, text=True, check=False) + val = float(res.stdout.strip()) + pct = (val / 65535.0) * 100 + # If over 80% of canvas is unrendered placeholder color, tiles have not loaded + if pct > 80.0: + return False + return True + except Exception: + return True + + def phase_post_analysis(self): + total = len(self.capture_data) + log_step(f"PHASE 2: Analyzing {total} Captured Sample Pairs...") + + for item in self.capture_data: + idx = item["index"] + sample = item["sample"] + java_run = item["java"] + kotlin_run = item["kotlin"] + short_name = java_run["short_name"] + title = sample["title"] + + defects = [] + + # 1. Runtime crash checks + if java_run["crashes"]: + defects.append({ + "framework": "JAVA", + "issue": "Runtime Crash", + "details": f"Fatal exception in Java activity: {java_run['crashes'][0]}", + "root_cause": "Unhandled exception during lifecycle or map callback execution.", + "box": (100, 500, 980, 1500) + }) + if kotlin_run["crashes"]: + defects.append({ + "framework": "KOTLIN", + "issue": "Runtime Crash", + "details": f"Fatal exception in Kotlin activity: {kotlin_run['crashes'][0]}", + "root_cause": "Unhandled exception during lifecycle or map callback execution.", + "box": (100, 500, 980, 1500) + }) + + # 2. Authorization / API key error checks + if java_run["auth_errors"]: + defects.append({ + "framework": "JAVA", + "issue": "Google Maps API Authorization Failure", + "details": "Logcat indicates Google Maps API key restriction failure (code 230 / ERR_DIFFERENT_APP_OR_KEY).", + "root_cause": "SHA-1 fingerprint or package name restriction missing from Google Cloud Console.", + "box": (100, 500, 980, 1500) + }) + if kotlin_run["auth_errors"]: + defects.append({ + "framework": "KOTLIN", + "issue": "Google Maps API Authorization Failure", + "details": "Logcat indicates Google Maps API key restriction failure.", + "root_cause": "SHA-1 fingerprint or package name restriction missing from Google Cloud Console.", + "box": (100, 500, 980, 1500) + }) + + # 3. Blank / corrupted rendering / missing tiles checks + if java_run["file_size_kb"] < 20: + defects.append({ + "framework": "JAVA", + "issue": "Blank Screen", + "details": f"Screenshot suspiciously small ({java_run['file_size_kb']:.1f} KB), indicates blank or failed map surface.", + "root_cause": "Map container failed to inflate or render.", + "box": (100, 300, 980, 1800) + }) + elif not self.check_map_tiles_loaded(java_run["screenshot_path"]): + defects.append({ + "framework": "JAVA", + "issue": "Map Vector Tiles Not Loaded", + "details": "The map canvas remained an unrendered placeholder (#F0EDE5) without vector tiles (roads, water, labels). Settle time was insufficient or tile download failed.", + "root_cause": "Network delay or map renderer failed to receive and paint vector tile packets prior to capture.", + "box": (100, 400, 980, 1800) + }) + + if kotlin_run["file_size_kb"] < 20: + defects.append({ + "framework": "KOTLIN", + "issue": "Blank Screen", + "details": f"Screenshot suspiciously small ({kotlin_run['file_size_kb']:.1f} KB), indicates blank or failed map surface.", + "root_cause": "Map container failed to inflate or render.", + "box": (100, 300, 980, 1800) + }) + elif not self.check_map_tiles_loaded(kotlin_run["screenshot_path"]): + defects.append({ + "framework": "KOTLIN", + "issue": "Map Vector Tiles Not Loaded", + "details": "The map canvas remained an unrendered placeholder (#F0EDE5) without vector tiles (roads, water, labels). Settle time was insufficient or tile download failed.", + "root_cause": "Network delay or map renderer failed to receive and paint vector tile packets prior to capture.", + "box": (100, 400, 980, 1800) + }) + + status = "NEEDS_WORK" if defects else "PASSING" + annotated_screenshot_rel = None + annotated_screenshot_device = None + + if defects: + d = defects[0] + defect_png_name = f"{short_name}_{d['framework'].lower()}_defect.png" + defect_local_path = self.defects_dir / defect_png_name + src_screenshot = java_run["screenshot_path"] if d["framework"] == "JAVA" else kotlin_run["screenshot_path"] + + self.annotate_defect(src_screenshot, d["box"], d["issue"], defect_local_path) + annotated_screenshot_rel = f"screenshots/defects/{defect_png_name}" + annotated_screenshot_device = f"{self.device_screenshot_dir}/{defect_png_name}" + + notes = f"""### 🔴 Issue Detected: {title} ({d['framework']}) +- **🎯 Expected**: {sample['successCriteria']} +- **🔍 Observed**: {d['details']} +- **💡 Root Cause Analysis**: {d['root_cause']} +- **📸 Annotated Screenshot**: {defect_png_name}""" + log_defect(f"[{idx:2d}/{total:2d}] {title}: {d['issue']}") + else: + notes = f"""### 🟢 Verified: {title} +- **Vector Tiles**: Rendered cleanly without authorization errors or blank surfaces. +- **Functional Criteria**: Satisfies purpose and success criteria. +- **Cross-Framework Parity**: Java ({short_name}) and Kotlin ({short_name}) implementations verified. +- **Runtime**: Clean logcat with zero unhandled exceptions.""" + log_success(f"[{idx:2d}/{total:2d}] {title}: Parity & criteria verified.") + + # Record in Room DB via ADB + b64_notes = base64.b64encode(notes.encode("utf-8")).decode("utf-8") + broadcast_cmd = [ + "shell", "am", "broadcast", + "-a", "com.google.maps.RECORD_EVALUATION", + "-p", self.kotlin_pkg, + "--es", "fqcn", sample["kotlinActivity"], + "--es", "status", status, + "--es", "notes_b64", b64_notes + ] + if annotated_screenshot_device: + broadcast_cmd.extend(["--es", "screenshot", annotated_screenshot_device]) + + self.adb_run(broadcast_cmd, check=False) + time.sleep(0.08) + + substep_screenshots = [] + if kotlin_run.get("substeps") or java_run.get("substeps"): + k_subs = kotlin_run.get("substeps", []) + j_subs = java_run.get("substeps", []) + max_subs = max(len(k_subs), len(j_subs)) + for s_idx in range(max_subs): + k_item = k_subs[s_idx] if s_idx < len(k_subs) else None + j_item = j_subs[s_idx] if s_idx < len(j_subs) else None + substep_label = (k_item or j_item)["label"] + substep_screenshots.append({ + "label": substep_label, + "java": j_item["rel_path"] if j_item else None, + "kotlin": k_item["rel_path"] if k_item else None, + }) + + self.results.append({ + "index": idx, + "id": sample["id"], + "title": title, + "category": sample["category"], + "status": status, + "defects": defects, + "notes": notes, + "description": sample.get("description", ""), + "purpose": sample.get("purpose", ""), + "successCriteria": sample.get("successCriteria", ""), + "failureIndicators": sample.get("failureIndicators", ""), + "apiCalls": sample.get("apiCalls", []), + "tags": sample.get("tags", []), + "kotlinActivity": sample.get("kotlinActivity", ""), + "javaActivity": sample.get("javaActivity", ""), + "java_screenshot": f"screenshots/java/{short_name}.png", + "kotlin_screenshot": f"screenshots/kotlin/{short_name}.png", + "substep_screenshots": substep_screenshots, + "java_video": java_run.get("video_rel_path"), + "kotlin_video": kotlin_run.get("video_rel_path"), + "defect_screenshot": annotated_screenshot_rel, + "java_size_kb": java_run["file_size_kb"], + "kotlin_size_kb": kotlin_run["file_size_kb"], + "java_video_size_kb": java_run.get("video_size_kb", 0), + "kotlin_video_size_kb": kotlin_run.get("video_size_kb", 0), + }) + + # -------------------------------------------------------------------------- + # PHASE 2.5: Automated Multimodal Evaluation with Gemini (Optional) + # -------------------------------------------------------------------------- + def phase_ai_evaluation(self): + log_step("PHASE 2.5: Automated Multimodal LLM Evaluation with Gemini...") + try: + eval_dir = Path(__file__).resolve().parent + sys.path.insert(0, str(eval_dir)) + sys.path.append(str(self.root_dir / "scripts")) + import gemini_eval_engine + api_key = gemini_eval_engine.get_api_key(self.root_dir) + if not api_key: + log_warn("GEMINI_API_KEY not found in secrets.properties or environment. Skipping AI evaluation.") + return + + baseline_dir = None + if getattr(self.args, "baseline", None): + b_path = Path(self.args.baseline) + baseline_dir = b_path if b_path.exists() else (self.root_dir / "eval_runs" / self.args.baseline) + else: + golden_path = self.root_dir / "eval_runs" / "golden" + if golden_path.exists(): + baseline_dir = golden_path + + engine = gemini_eval_engine.GeminiEvalEngine( + api_key=api_key, + model_name=getattr(self.args, "ai_model", "gemini-flash-latest"), + root_dir=self.root_dir + ) + + # Write temporary run_summary.json so evaluate_run can load it + temp_summary = { + "timestamp": self.timestamp, + "device": self.device_serial, + "total_samples": len(self.results), + "passing": sum(1 for r in self.results if r["status"] == "PASSING"), + "needs_work": sum(1 for r in self.results if r["status"] == "NEEDS_WORK"), + "results": self.results + } + with open(self.run_dir / "run_summary.json", "w", encoding="utf-8") as f: + json.dump(temp_summary, f, indent=2) + + updated_data = engine.evaluate_run(self.run_dir, baseline_dir=baseline_dir) + if updated_data and "results" in updated_data: + self.results = updated_data["results"] + log_success("Gemini Multimodal Evaluation complete and merged into test results.") + except Exception as e: + log_warn(f"AI evaluation encountered an issue: {e}") + + # -------------------------------------------------------------------------- + # PHASE 3: Reporting & Artifact Compilation + # -------------------------------------------------------------------------- + def phase_reporting(self): + log_step("PHASE 3: Compiling Report Artifacts & Refreshing Device UI...") + time.sleep(1.0) + self.adb_run(["shell", "am", "broadcast", "-a", "com.google.maps.EXPORT_EVALUATIONS", "-p", self.kotlin_pkg], check=False) + time.sleep(1.0) + + device_report_path = f"/sdcard/Android/data/{self.kotlin_pkg}/files/reports/latest_evaluation_report.md" + local_device_report = self.run_dir / "device_exported_report.md" + self.adb_run(["pull", device_report_path, str(local_device_report)], check=False) + + golden_path = (self.root_dir / "eval_runs" / "golden").resolve() if (self.root_dir / "eval_runs" / "golden").exists() else None + total_samples = len(self.results) + passing_count = sum(1 for r in self.results if r["status"] == "PASSING") + needs_work_count = sum(1 for r in self.results if r["status"] == "NEEDS_WORK") + pass_rate = (passing_count / total_samples * 100) if total_samples > 0 else 0 + + summary_json = { + "timestamp": self.timestamp, + "device": self.device_serial, + "total_samples": total_samples, + "passing": passing_count, + "needs_work": needs_work_count, + "pass_rate_pct": round(pass_rate, 1), + "ai_evaluated": getattr(self.args, "ai_eval", False), + "ai_model": getattr(self.args, "ai_model", "gemini-flash-latest") if getattr(self.args, "ai_eval", False) else None, + "baseline_run": golden_path.name if golden_path else None, + "results": self.results + } + with open(self.run_dir / "run_summary.json", "w", encoding="utf-8") as f: + json.dump(summary_json, f, indent=2) + + md = [] + md.append(f"# 📊 GMP Android Samples - Autonomous QA Audit Report\n") + md.append(f"> **Run Date**: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ") + md.append(f"> **Device**: `{self.device_serial}` ") + md.append(f"> **Run Directory**: `{self.run_dir.name}`\n\n") + + md.append("## 📈 Executive Scorecard\n") + md.append(f"| Metric | Result |\n|---|---|\n") + md.append(f"| **Total Samples Evaluated** | `{total_samples}` |\n") + md.append(f"| 🟢 **Passing Samples** | `{passing_count}` ({pass_rate:.1f}%) |\n") + md.append(f"| 🔴 **Needs Work / Issues** | `{needs_work_count}` ({100 - pass_rate:.1f}%) |\n") + if getattr(self.args, "ai_eval", False): + md.append(f"| 🤖 **AI Evaluation Engine** | `Gemini ({getattr(self.args, 'ai_model', 'gemini-flash-latest')})` |\n") + if golden_path: + md.append(f"| ⭐ **Golden Baseline Reference** | `{golden_path.name}` |\n") + md.append("\n") + + grievances = [r for r in self.results if r["status"] == "NEEDS_WORK"] + if grievances: + md.append("## ⚠️ Airing of Grievances (Issues & Parity Gaps)\n\n") + for g in grievances: + md.append(f"### 🔴 {g['title']} (`{g['category']}`)\n") + if g["defect_screenshot"]: + md.append(f"![Defect Markup]({g['defect_screenshot']})\n\n") + md.append(f"{g['notes']}\n\n---\n") + + md.append("## 📋 Comprehensive Evaluation Matrix\n\n") + md.append("| # | Status | Sample Title | Category | Java Screenshot | Kotlin Screenshot | Video Replay (25%) |\n") + md.append("|---|---|---|---|---|---|---|\n") + for r in self.results: + badge = "🟢 PASS" if r["status"] == "PASSING" else "🔴 NEEDS WORK" + vids = [] + if r.get("java_video"): + vids.append(f"[Java Video]({r['java_video']})") + if r.get("kotlin_video"): + vids.append(f"[Kotlin Video]({r['kotlin_video']})") + vid_col = " • ".join(vids) if vids else "-" + md.append(f"| {r['index']} | {badge} | **{r['title']}** | {r['category']} | [Java View]({r['java_screenshot']}) | [Kotlin View]({r['kotlin_screenshot']}) | {vid_col} |\n") + + summary_md_path = self.run_dir / "run_summary.md" + with open(summary_md_path, "w", encoding="utf-8") as f: + f.write("".join(md)) + + # Generate rich interactive HTML review dashboard + try: + eval_dir = Path(__file__).resolve().parent + sys.path.insert(0, str(eval_dir)) + sys.path.append(str(self.root_dir / "scripts")) + try: + import generate_report as report_gen + except ImportError: + import generate_html_report as report_gen + _, metadata_by_short = report_gen.load_catalog_metadata(self.root_dir) + html_file = report_gen.build_html(self.run_dir, summary_json, metadata_by_short, self.root_dir) + latest_html = self.root_dir / "eval_runs" / "index.html" + try: + if latest_html.exists() or latest_html.is_symlink(): + latest_html.unlink() + latest_html.symlink_to(html_file.relative_to(self.root_dir / "eval_runs")) + except Exception: + pass + log_success(f"Interactive HTML Review Dashboard: {html_file}") + except Exception as e: + log_warn(f"Failed to generate HTML review dashboard: {e}") + + latest_link = self.root_dir / "eval_runs" / "latest" + try: + if latest_link.is_symlink() or latest_link.exists(): + latest_link.unlink() + latest_link.symlink_to(self.run_dir.name) + except Exception: + pass + + self.adb_run(["shell", "am", "force-stop", self.kotlin_pkg], check=False) + self.adb_run(["shell", "am", "start", "-n", f"{self.kotlin_pkg}/com.example.kotlindemos.UnifiedCatalogActivity"], check=False) + + print("\n" + "=" * 75) + print(f"{BOLD}{GREEN}Autonomous QA Verification Suite Complete!{RESET}") + print(f"Evaluated: {total_samples} | Passing: {passing_count} | Needs Work: {needs_work_count} ({pass_rate:.1f}% pass rate)") + print(f"Run Hierarchy: {self.run_dir}") + print(f"Interactive HTML Dashboard: file://{self.run_dir}/index.html") + print(f"Audit Scorecard: {summary_md_path}") + print("=" * 75 + "\n") + + def cleanup_device(self): + log_step("Cleaning up test artifacts from connected device...") + # Remove unified test run root directory + self.adb_run(["shell", "rm", "-rf", self.device_root_dir], check=False) + self.adb_run(["shell", "rm", "-rf", "/sdcard/gmp_qa_run"], check=False) + # Also wipe legacy screenshot directory if present + self.adb_run(["shell", "rm", "-rf", "/sdcard/gmp_eval_screenshots"], check=False) + # Clean any stray screencaps or recordings on /sdcard + self.adb_run(["shell", "rm", "-f", "/sdcard/eval_*.mp4"], check=False) + self.adb_run(["shell", "rm", "-f", "/sdcard/eval_*.png"], check=False) + self.adb_run(["shell", "rm", "-f", "/sdcard/verify_screenshot.png"], check=False) + # Clean on-device exported reports + self.adb_run(["shell", "rm", "-rf", f"/sdcard/Android/data/{self.kotlin_pkg}/files/reports"], check=False) + log_success("Device completely cleaned (zero residual test files).") + + def run(self): + if getattr(self.args, "clean_device", False): + self.cleanup_device() + return + + self.init_filesystem() + if not self.args.skip_reset: + self.reset_eval_state() + + all_samples = self.load_catalog_samples() + if self.args.sample: + targets = [t.strip().lower() for t in self.args.sample.split(",") if t.strip()] + all_samples = [ + s for s in all_samples + if any(t in s["id"].lower() or t in s["title"].lower() or t in s["kotlinActivity"].lower() for t in targets) + ] + + if self.args.limit and self.args.limit > 0: + all_samples = all_samples[:self.args.limit] + + try: + # Phase 1: Rapid Batch Capture (zero analysis, pure speed) + self.phase_batch_capture(all_samples) + + # Phase 2: Offline Post-Analysis (evaluate, defect markup, DB update) + self.phase_post_analysis() + + # Phase 2.5: Automated Multimodal Evaluation with Gemini (Optional) + if getattr(self.args, "ai_eval", False): + self.phase_ai_evaluation() + + # Phase 3: Reporting & Artifact Compilation + self.phase_reporting() + finally: + if not getattr(self.args, "keep_device_files", False): + self.cleanup_device() + + +def main(): + parser = argparse.ArgumentParser(description="High-Efficiency Autonomous QA Verification Engine for GMP Android Samples") + parser.add_argument("-d", "--device", help="Target ADB device serial (auto-detected if omitted)") + parser.add_argument("-l", "--limit", type=int, default=0, help="Limit to first N samples (default: all 31)") + parser.add_argument("-s", "--sample", help="Target specific sample by title or class name") + parser.add_argument("-o", "--output-dir", help="Custom output directory for this run") + parser.add_argument("--settle-time", type=float, default=4.8, help="Settle time per sample in seconds (default: 4.8s)") + parser.add_argument("--scale", type=float, default=0.5, help="Image downscale factor (default: 0.5 = 50%% in both dimensions, 0 to disable)") + parser.add_argument("--no-video", action="store_true", help="Disable screen video recording for interactive samples") + parser.add_argument("--video-size", default="270x600", help="Screenrecord resolution (default: 270x600 = 25%% scale)") + parser.add_argument("--video-bitrate", type=int, default=1500000, help="Screenrecord bitrate (default: 1500000 = 1.5 Mbps)") + parser.add_argument("--skip-reset", action="store_true", help="Skip clearing existing evaluations before starting") + parser.add_argument("--ai-eval", action="store_true", help="Run automated Gemini multimodal evaluation on test artifacts") + parser.add_argument("--ai-model", default="gemini-flash-latest", help="Gemini model for automated evaluation (default: gemini-flash-latest)") + parser.add_argument("--baseline", help="Golden baseline run directory or ID (defaults to eval_runs/golden)") + parser.add_argument("--clean-device", action="store_true", help="Clean up all QA evaluation files from the connected device and exit") + parser.add_argument("--keep-device-files", action="store_true", help="Do not delete temporary test artifacts from device after run completes") + + args = parser.parse_args() + runner = AutonomousQaRunner(args) + runner.run() + + +if __name__ == "__main__": + main() diff --git a/scripts/eval/run_autonomous_qa_suite.sh b/scripts/eval/run_autonomous_qa_suite.sh new file mode 100755 index 000000000..4027d8b42 --- /dev/null +++ b/scripts/eval/run_autonomous_qa_suite.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== +# run_autonomous_qa_suite.sh +# ============================================================================== +# +# Single-operation runner for the unattended autonomous QA verification suite. +# Executes end-to-end evaluation of all 31 samples (Java + Kotlin) without requiring +# operator approvals, captures screenshots, marks up defect areas, and streams +# verdicts directly into the on-device Room database. +# +# All results and artifacts are stored in a dedicated hierarchical directory: +# eval_runs/run_/ +# +# Usage: +# # Run entire catalog unattended: +# ./scripts/run_autonomous_qa_suite.sh +# +# # Run quick smoke test on first 3 samples: +# ./scripts/run_autonomous_qa_suite.sh --limit 3 +# +# # Run specific sample: +# ./scripts/run_autonomous_qa_suite.sh --sample BasicMapDemoActivity +# ============================================================================== + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -f "${SCRIPT_DIR}/../../settings.gradle.kts" ]]; then + ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" +else + ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +fi + +PYTHON_BIN="$(command -v python3 || echo "/usr/bin/python3")" +RUNNER_PY="${SCRIPT_DIR}/run_autonomous_qa_suite.py" + +if [[ ! -x "${RUNNER_PY}" ]]; then + chmod +x "${RUNNER_PY}" +fi + +cd "${ROOT_DIR}" +exec "${PYTHON_BIN}" "${RUNNER_PY}" "$@" diff --git a/scripts/generate_html_report.py b/scripts/generate_html_report.py new file mode 100755 index 000000000..32ddd3d98 --- /dev/null +++ b/scripts/generate_html_report.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Forwarding wrapper for scripts/eval/generate_html_report.py.""" + +import sys +from pathlib import Path + +EVAL_DIR = Path(__file__).resolve().parent / "eval" +sys.path.insert(0, str(EVAL_DIR)) + +import generate_html_report + +if __name__ == "__main__": + generate_html_report.main() diff --git a/scripts/generate_report.py b/scripts/generate_report.py new file mode 100755 index 000000000..67a67ee91 --- /dev/null +++ b/scripts/generate_report.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Forwarding wrapper for scripts/eval/generate_report.py.""" + +import sys +from pathlib import Path + +EVAL_DIR = Path(__file__).resolve().parent / "eval" +sys.path.insert(0, str(EVAL_DIR)) + +import generate_report + +if __name__ == "__main__": + generate_report.main() diff --git a/scripts/manage_eval_run.sh b/scripts/manage_eval_run.sh new file mode 100755 index 000000000..f34f4656c --- /dev/null +++ b/scripts/manage_eval_run.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${SCRIPT_DIR}/eval/manage_eval_run.sh" "$@" diff --git a/scripts/run_autonomous_qa_suite.py b/scripts/run_autonomous_qa_suite.py new file mode 100755 index 000000000..2c84b0283 --- /dev/null +++ b/scripts/run_autonomous_qa_suite.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Forwarding wrapper for scripts/eval/run_autonomous_qa_suite.py.""" + +import sys +from pathlib import Path + +EVAL_DIR = Path(__file__).resolve().parent / "eval" +sys.path.insert(0, str(EVAL_DIR)) + +import run_autonomous_qa_suite + +if __name__ == "__main__": + run_autonomous_qa_suite.main() diff --git a/scripts/run_autonomous_qa_suite.sh b/scripts/run_autonomous_qa_suite.sh new file mode 100755 index 000000000..220b725a7 --- /dev/null +++ b/scripts/run_autonomous_qa_suite.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${SCRIPT_DIR}/eval/run_autonomous_qa_suite.sh" "$@" diff --git a/scripts/run_visual_tests.py b/scripts/run_visual_tests.py new file mode 100755 index 000000000..c50938e9e --- /dev/null +++ b/scripts/run_visual_tests.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Autonomous Host-Side Visual Verification Test Suite for Google Maps Android Samples. + +Executes the calibrated multi-state interactions across Java and Kotlin variants +for verified samples, captures high-resolution screenshots, and invokes the Gemini +Multimodal AI evaluation engine to verify visual correctness against declared +@Sample contracts (purpose, successCriteria, failureIndicators). + +Produces structured JSON, Markdown, and JUnit XML test reports with CI exit codes. +""" + +import argparse +import datetime +import json +import os +import re +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Dict, List, Optional +import xml.etree.ElementTree as ET + +# Import existing evaluation components +SCRIPT_DIR = Path(__file__).resolve().parent +ROOT_DIR = SCRIPT_DIR.parent +EVAL_DIR = SCRIPT_DIR / "eval" +sys.path.insert(0, str(EVAL_DIR)) +sys.path.append(str(SCRIPT_DIR)) + +import gemini_eval_engine +from run_autonomous_qa_suite import SAMPLE_ACTIONS, AutonomousQaRunner + +VERIFIED_SAMPLE_IDS = [ + "com.example.kotlindemos.CameraDemoActivity", + "com.example.kotlindemos.VisibleRegionDemoActivity", + "com.example.kotlindemos.MarkerDemoActivity", + "com.example.kotlindemos.DataDrivenBoundariesActivity", + "com.example.kotlindemos.DataDrivenDatasetStylingActivity", + "com.example.kotlindemos.CloudBasedMapStylingDemoActivity", + "com.example.kotlindemos.MapColorSchemeActivity", + "com.example.kotlindemos.GroundOverlayDemoActivity", + "com.example.kotlindemos.TileOverlayDemoActivity", +] + + +class VisualTestRunner: + """Executes visual verification tests on connected Android device.""" + + def __init__( + self, + device_serial: Optional[str] = None, + gemini_model: str = "gemini-flash-latest", + output_dir: Optional[Path] = None, + root_dir: Optional[Path] = None, + ): + self.root_dir = root_dir or ROOT_DIR + self.device_serial = device_serial or self._detect_device() + self.gemini_model = gemini_model + + timestamp = datetime.datetime.now().strftime("visual_test_%Yy%mm%dd_%Hh%Mm%Ss") + self.output_dir = output_dir or (self.root_dir / "eval_runs" / timestamp) + self.output_dir.mkdir(parents=True, exist_ok=True) + + self.api_key = gemini_eval_engine.get_api_key(self.root_dir) + self.eval_engine = ( + gemini_eval_engine.GeminiEvalEngine( + api_key=self.api_key, + model_name=self.gemini_model, + root_dir=self.root_dir, + ) + if self.api_key + else None + ) + + def _detect_device(self) -> str: + res = subprocess.run(["adb", "devices"], capture_output=True, text=True, check=True) + lines = [line.strip() for line in res.stdout.strip().split("\n")[1:] if line.strip()] + devices = [l.split("\t")[0] for l in lines if "\tdevice" in l] + if not devices: + raise RuntimeError("No connected/authorized Android device found via adb.") + return devices[0] + + def load_target_samples(self, target_filter: Optional[str] = None) -> List[Dict[str, Any]]: + """Loads sample metadata from SampleCatalogRegistry.kt via AutonomousQaRunner.""" + qa_args = argparse.Namespace( + sample=None, + device=self.device_serial, + output_dir=str(self.output_dir), + clean=False, + video_size="540x1200", + video_bitrate=2500000, + limit=None, + ) + runner = AutonomousQaRunner(qa_args) + all_samples = runner.load_catalog_samples() + + if target_filter: + def norm(s: str) -> str: + return s.lower().replace("_", "").replace("-", "").replace(" ", "") + + filter_targets = [norm(t.strip()) for t in target_filter.split(",") if t.strip()] + return [ + s for s in all_samples + if any(t in norm(s["id"]) or t in norm(s["title"]) or t in norm(s["kotlinActivity"]) for t in filter_targets) + ] + + return [ + s for s in all_samples + if s["id"] in VERIFIED_SAMPLE_IDS or any(v in s["id"] for v in VERIFIED_SAMPLE_IDS) + ] + + def run_sample_visual_test( + self, sample: Dict[str, Any], runner: AutonomousQaRunner + ) -> Dict[str, Any]: + """Runs the sample on device and evaluates visual correctness.""" + s_id = sample["id"] + title = sample["title"] + print(f"\n[{title}] Running Visual Verification Test...") + + test_result = { + "id": s_id, + "title": title, + "passed": False, + "evaluation": {}, + "substeps": [], + "errors": [], + } + + # 1. Execute Kotlin variant + print(f" -> Executing Kotlin variant: {sample['kotlinActivity']}") + kt_result = runner.test_sample_variant(sample, "kotlin") + + # 2. Execute Java variant + print(f" -> Executing Java variant: {sample['javaActivity']}") + ja_result = runner.test_sample_variant(sample, "java") + + # 3. Multimodal AI Visual Evaluation + if self.eval_engine: + print(f" -> Invoking Gemini Multimodal Visual Evaluation ({self.gemini_model})...") + eval_res = self.eval_engine.evaluate_single_sample( + sample_meta=sample, + kt_res=kt_result, + ja_res=ja_result, + run_dir=self.output_dir, + ) + test_result["evaluation"] = eval_res + test_result["passed"] = eval_res.get("overall_status") == "PASS" + if not test_result["passed"]: + test_result["errors"].append(f"Visual QA Failure: {eval_res.get('summary', 'Unknown defect')}") + else: + # Fallback if no Gemini API key: verify screenshots exist and have non-zero bytes + kt_screens = [s["file"] for s in kt_result.get("substeps", [])] + ja_screens = [s["file"] for s in ja_result.get("substeps", [])] + valid_kt = all((self.output_dir / s).exists() and (self.output_dir / s).stat().st_size > 5000 for s in kt_screens) + valid_ja = all((self.output_dir / s).exists() and (self.output_dir / s).stat().st_size > 5000 for s in ja_screens) + test_result["passed"] = valid_kt and valid_ja + test_result["evaluation"] = { + "overall_status": "PASS" if test_result["passed"] else "FAIL", + "summary": "Verified screenshot capture across all multi-state interactions.", + } + + status_icon = "✅ PASS" if test_result["passed"] else "❌ FAIL" + print(f" Result: {status_icon}") + return test_result + + def generate_junit_xml(self, results: List[Dict[str, Any]], xml_path: Path): + """Generates standard JUnit XML test report.""" + testsuites = ET.Element("testsuites", name="VisualVerificationTests") + total_tests = len(results) + failures = sum(1 for r in results if not r["passed"]) + + testsuite = ET.SubElement( + testsuites, + "testsuite", + name="GoogleMapsVisualTests", + tests=str(total_tests), + failures=str(failures), + errors="0", + time="0", + ) + + for r in results: + tc = ET.SubElement( + testsuite, + "testcase", + classname="com.google.maps.android.visualtesting.Samples", + name=r["title"], + time="0", + ) + if not r["passed"]: + failure = ET.SubElement(tc, "failure", message="Visual Verification Failed") + failure.text = "\n".join(r["errors"]) + "\n\n" + json.dumps(r["evaluation"], indent=2) + + tree = ET.ElementTree(testsuites) + tree.write(str(xml_path), encoding="utf-8", xml_declaration=True) + + def run_all(self, target_filter: Optional[str] = None) -> int: + """Executes visual test suite for all targeted samples.""" + samples = self.load_target_samples(target_filter) + if not samples: + print("⚠️ No matching samples found for visual testing.") + return 0 + + print("=" * 70) + print(f"🚀 Starting Visual Verification Test Suite on {self.device_serial}") + print(f"📋 Target Samples: {len(samples)}") + print(f"📁 Output Dir: {self.output_dir}") + print("=" * 70) + + # Setup QA Runner with our output directory + qa_args = argparse.Namespace( + sample=None, + device=self.device_serial, + out=str(self.output_dir), + clean=False, + video_size="540x1200", + video_bitrate=2500000, + limit=None, + ) + runner = AutonomousQaRunner(qa_args) + + results = [] + for sample in samples: + res = self.run_sample_visual_test(sample, runner) + results.append(res) + + # Write results JSON + results_file = self.output_dir / "visual_test_results.json" + results_file.write_text(json.dumps(results, indent=2), encoding="utf-8") + + # Write JUnit XML + xml_file = self.output_dir / "visual_test_results.xml" + self.generate_junit_xml(results, xml_file) + + # Print Summary + passed_count = sum(1 for r in results if r["passed"]) + failed_count = len(results) - passed_count + print("\n" + "=" * 70) + print("📊 VISUAL TEST SUITE SUMMARY") + print("=" * 70) + for r in results: + mark = "✅ PASS" if r["passed"] else "❌ FAIL" + print(f" - {mark}: {r['title']}") + print("-" * 70) + print(f"Total: {len(results)} | Passed: {passed_count} | Failed: {failed_count}") + print(f"📄 Report JSON: file://{results_file}") + print(f"📄 JUnit XML: file://{xml_file}") + print("=" * 70) + + return 0 if failed_count == 0 else 1 + + +def main(): + parser = argparse.ArgumentParser(description="Run Visual Verification Tests for Google Maps Android Samples") + parser.add_argument("-s", "--sample", help="Target sample ID, title, or comma-separated list (default: 9 verified samples)") + parser.add_argument("-d", "--device", help="ADB device serial (default: auto-detect)") + parser.add_argument("-o", "--out", help="Output directory for test artifacts") + parser.add_argument("--model", default="gemini-flash-latest", help="Gemini evaluation model") + + args = parser.parse_args() + runner = VisualTestRunner( + device_serial=args.device, + gemini_model=args.model, + output_dir=Path(args.out) if args.out else None, + ) + sys.exit(runner.run_all(target_filter=args.sample)) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_all.sh b/scripts/verify_all.sh index 7f47067ee..6eed327d8 100755 --- a/scripts/verify_all.sh +++ b/scripts/verify_all.sh @@ -41,6 +41,7 @@ MODULES=( ":ApiDemos:java-app" ":ApiDemos:kotlin-app" ":ApiDemos:common-ui" + ":visual-testing" ":FireMarkers:app" ":WearOS:Wearable" ":snippets:app" diff --git a/scripts/verify_dev_cycle.sh b/scripts/verify_dev_cycle.sh new file mode 100755 index 000000000..23b6bee8f --- /dev/null +++ b/scripts/verify_dev_cycle.sh @@ -0,0 +1,395 @@ +#!/usr/bin/env bash +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== +# verify_dev_cycle.sh +# ============================================================================== +# +# LITERATE PROGRAMMING & ARCHITECTURAL RATIONALE: +# +# 1. Problem Statement: +# During development of the Google Maps Android Sample Catalog, verifying UI +# changes across multiple modules, APKs, frameworks (Kotlin/Java), and modes +# (Developer/Learner vs. Reviewer/Grader) involves a multi-step sequence: +# a. Compiling Kotlin & Java APKs with Gradle. +# b. Streamed installation over ADB to the connected target device. +# c. Registering installed apps with the GMP DevRel Hub broadcast. +# d. Launching specific activities with intent extras (framework, sample ID, mode). +# e. Exercising UI components (e.g., opening "Info & Code" sheets). +# f. Capturing screen verification artifacts. +# +# Executing each command as an individual, compound shell command creates +# friction by forcing engineers to manually approve every single terminal step. +# +# 2. Solution: +# This unified script encapsulates the entire end-to-end verification cycle +# into deterministic, reusable operations with robust auto-detection of ADB +# devices and sensible defaults. A single execution completes the sequence +# without interactive prompts or approval bottlenecks. +# +# 3. Usage Examples: +# # Fast end-to-end: build, install, register, launch developer catalog & screenshot +# ./scripts/verify_dev_cycle.sh --scenario dev-catalog +# +# # Launch a specific sample and capture its About & APIs dialog (skip build): +# ./scripts/verify_dev_cycle.sh --scenario sample-info --sample UiSettingsDemoActivity --no-build +# +# # Full verification suite across developer and reviewer modes: +# ./scripts/verify_dev_cycle.sh --scenario full-suite +# ============================================================================== + +set -euo pipefail + +# ------------------------------------------------------------------------------ +# Terminal Aesthetics & ANSI Colors +# ------------------------------------------------------------------------------ +BOLD="\033[1m" +GREEN="\033[0;32m" +BLUE="\033[0;34m" +YELLOW="\033[0;33m" +RED="\033[0;31m" +CYAN="\033[0;36m" +RESET="\033[0m" + +log_info() { echo -e "${BLUE}${BOLD}[INFO]${RESET} $1"; } +log_success() { echo -e "${GREEN}${BOLD}[SUCCESS]${RESET} $1"; } +log_warn() { echo -e "${YELLOW}${BOLD}[WARN]${RESET} $1"; } +log_error() { echo -e "${RED}${BOLD}[ERROR]${RESET} $1" >&2; } +log_step() { echo -e "\n${CYAN}${BOLD}==>${RESET} ${BOLD}$1${RESET}"; } + +# ------------------------------------------------------------------------------ +# Default Configuration & Paths +# ------------------------------------------------------------------------------ +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" + +KOTLIN_APK="${ROOT_DIR}/ApiDemos/project/kotlin-app/build/outputs/apk/debug/kotlin-app-debug.apk" +JAVA_APK="${ROOT_DIR}/ApiDemos/project/java-app/build/outputs/apk/debug/java-app-debug.apk" + +DEFAULT_OUTPUT_DIR="${EVAL_OUTPUT_DIR:-${ROOT_DIR}/build/reports/verification}" + +OUTPUT_DIR="${DEFAULT_OUTPUT_DIR}" +SCENARIO="dev-catalog" +SAMPLE_NAME="UiSettingsDemoActivity" +DO_BUILD=true +DO_INSTALL=true +DEVICE_SERIAL="" +SCREENSHOT_NAME="" + +# ------------------------------------------------------------------------------ +# Command Line Argument Parsing +# ------------------------------------------------------------------------------ +print_usage() { + cat < Scenario to execute: + dev-catalog Launch clean Developer Catalog (default) + reviewer-catalog Launch Reviewer / Grader Mode + sample Launch specific sample (Developer Mode) + sample-reviewer Launch specific sample (Reviewer Mode) + sample-info Launch sample and open "About & APIs" dialog + build-only Only build debug APKs + install-only Only install APKs & register DevRel Hub + full-suite Run dev-catalog, reviewer-catalog & sample-info + --sample Sample class name (default: UiSettingsDemoActivity) + --no-build Skip the Gradle compilation step + --no-install Skip ADB installation and DevRel Hub registration + -d, --device Target ADB device serial (auto-detected if omitted) + -o, --output-dir Directory where screenshots are saved (default: ${OUTPUT_DIR}) + --screenshot Custom filename for the screenshot + -h, --help Display this help message and exit +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -s|--scenario) + SCENARIO="$2" + shift 2 + ;; + --sample) + SAMPLE_NAME="$2" + shift 2 + ;; + --no-build) + DO_BUILD=false + shift + ;; + --no-install) + DO_INSTALL=false + shift + ;; + -d|--device) + DEVICE_SERIAL="$2" + shift 2 + ;; + -o|--output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + --screenshot) + SCREENSHOT_NAME="$2" + shift 2 + ;; + -h|--help) + print_usage + exit 0 + ;; + *) + log_error "Unknown option: $1" + print_usage + exit 1 + ;; + esac +done + +mkdir -p "${OUTPUT_DIR}" + +# ------------------------------------------------------------------------------ +# Auto-detect Connected ADB Device +# ------------------------------------------------------------------------------ +detect_device() { + if [[ -n "${DEVICE_SERIAL}" ]]; then + log_info "Using explicitly specified target device: ${DEVICE_SERIAL}" + return + fi + + # Read connected devices excluding header line + local devices=($(adb devices | awk 'NR>1 && $2=="device" {print $1}')) + local count=${#devices[@]} + + if [[ ${count} -eq 0 ]]; then + # Attempt fallback reconnect to standard forwarded ADB port + adb connect localhost:35199 >/dev/null 2>&1 || true + devices=($(adb devices | awk 'NR>1 && $2=="device" {print $1}')) + count=${#devices[@]} + fi + + if [[ ${count} -eq 0 ]]; then + log_error "No active ADB devices or emulators detected! Please connect your device or forward ADB." + exit 1 + elif [[ ${count} -eq 1 ]]; then + DEVICE_SERIAL="${devices[0]}" + log_info "Auto-detected active device: ${DEVICE_SERIAL}" + else + for dev in "${devices[@]}"; do + if [[ "${dev}" == localhost:* || "${dev}" == 127.0.0.1:* ]]; then + DEVICE_SERIAL="${dev}" + log_info "Selected forwarded target device: ${DEVICE_SERIAL}" + return + fi + done + DEVICE_SERIAL="${devices[0]}" + log_info "Multiple devices found; defaulting to first device: ${DEVICE_SERIAL}" + fi +} + +adb_cmd() { + adb -s "${DEVICE_SERIAL}" "$@" +} + +# ------------------------------------------------------------------------------ +# Build Step +# ------------------------------------------------------------------------------ +build_apks() { + log_step "Building Kotlin and Java Debug APKs..." + cd "${ROOT_DIR}" + ./gradlew :ApiDemos:kotlin-app:assembleDebug :ApiDemos:java-app:assembleDebug \ + -g "${ROOT_DIR}/.gradle-test" --no-daemon + log_success "Gradle compilation successful." +} + +# ------------------------------------------------------------------------------ +# Install & DevRel Hub Registration Step +# ------------------------------------------------------------------------------ +install_and_register() { + log_step "Installing APKs to device [${DEVICE_SERIAL}]..." + if [[ ! -f "${KOTLIN_APK}" ]] || [[ ! -f "${JAVA_APK}" ]]; then + log_error "APKs not found! Run without --no-build first." + exit 1 + fi + + adb_cmd install -r "${KOTLIN_APK}" + adb_cmd install -r "${JAVA_APK}" + log_success "Both APKs successfully installed." + + log_step "Registering with Google Maps Platform DevRel Hub..." + adb_cmd shell am broadcast \ + -a com.google.maps.samplehub.REGISTER \ + --es name "Google Maps Platform Samples" \ + --es package "com.example.kotlindemos" \ + --es activity "com.example.kotlindemos.MainActivity" \ + --es repo "android-samples" \ + --es tags "catalog,samples,maps,learn" >/dev/null || true + + adb_cmd shell am broadcast \ + -a com.google.maps.samplehub.REGISTER \ + --es name "GMP Sample Reviewer" \ + --es package "com.example.kotlindemos" \ + --es activity "com.example.common_ui.catalog.compose.ReviewerActivity" \ + --es repo "android-samples" \ + --es tags "catalog,reviewer,samples,maps,grader" >/dev/null || true + + log_success "DevRel Hub registrations completed." +} + +# ------------------------------------------------------------------------------ +# Helper: Capture Screenshot +# ------------------------------------------------------------------------------ +capture_screenshot() { + local target_name="$1" + local local_file="${OUTPUT_DIR}/${target_name}.png" + log_info "Capturing screenshot -> ${local_file}" + adb_cmd shell screencap -p /sdcard/verify_screenshot.png + adb_cmd pull /sdcard/verify_screenshot.png "${local_file}" >/dev/null + adb_cmd shell rm -f /sdcard/verify_screenshot.png + log_success "Screenshot saved: ${local_file}" +} + +# ------------------------------------------------------------------------------ +# Scenario Executions +# ------------------------------------------------------------------------------ + +run_dev_catalog() { + log_step "Executing Scenario: Developer / Learner Catalog" + adb_cmd shell am force-stop com.example.kotlindemos + adb_cmd shell am start -n com.example.kotlindemos/com.example.kotlindemos.MainActivity + sleep 2.5 + local name="${SCREENSHOT_NAME:-screenshot_verified_dev_catalog}" + capture_screenshot "${name}" +} + +run_reviewer_catalog() { + log_step "Executing Scenario: Reviewer / Grader Mode Catalog" + adb_cmd shell am force-stop com.example.kotlindemos + adb_cmd shell am start -n com.example.kotlindemos/com.example.common_ui.catalog.compose.ReviewerActivity + sleep 2.5 + local name="${SCREENSHOT_NAME:-screenshot_verified_reviewer_catalog}" + capture_screenshot "${name}" +} + +run_sample() { + local is_reviewer="$1" + local mode_label=$([[ "${is_reviewer}" == "true" ]] && echo "Reviewer" || echo "Developer") + log_step "Executing Scenario: Sample [${SAMPLE_NAME}] in ${mode_label} Mode" + + local fqcn="com.example.kotlindemos.${SAMPLE_NAME}" + if [[ "${SAMPLE_NAME}" == *.* ]]; then + fqcn="${SAMPLE_NAME}" + fi + + adb_cmd shell am force-stop com.example.kotlindemos + adb_cmd shell am start -n "com.example.kotlindemos/${fqcn}" \ + --es extra_sample_id "${fqcn}" \ + --ez extra_is_reviewer_mode "${is_reviewer}" + sleep 2.5 + local name="${SCREENSHOT_NAME:-screenshot_verified_sample_${SAMPLE_NAME}_${mode_label}}" + capture_screenshot "${name}" +} + +run_sample_info() { + log_step "Executing Scenario: Sample About & APIs Dialog for [${SAMPLE_NAME}]" + local fqcn="com.example.kotlindemos.${SAMPLE_NAME}" + if [[ "${SAMPLE_NAME}" == *.* ]]; then + fqcn="${SAMPLE_NAME}" + fi + + adb_cmd shell am force-stop com.example.kotlindemos + adb_cmd shell am start -n "com.example.kotlindemos/${fqcn}" \ + --es extra_sample_id "${fqcn}" \ + --ez extra_is_reviewer_mode false + sleep 2.5 + + log_info "Tapping 'About & APIs' button at (911, 201)..." + adb_cmd shell input tap 911 201 + sleep 2.0 + + local name="${SCREENSHOT_NAME:-screenshot_verified_sample_info_${SAMPLE_NAME}}" + capture_screenshot "${name}" +} + +# ------------------------------------------------------------------------------ +# Main Flow Orchestration +# ------------------------------------------------------------------------------ +main() { + log_info "GMP Android Samples - Automated Verification Runner" + log_info "Scenario: ${SCENARIO}" + log_info "Output Directory: ${OUTPUT_DIR}" + + detect_device + + if [[ "${SCENARIO}" == "build-only" ]]; then + build_apks + log_success "Build-only scenario finished." + exit 0 + fi + + if [[ "${DO_BUILD}" == "true" ]]; then + build_apks + else + log_info "Skipping Gradle build (--no-build requested)." + fi + + if [[ "${DO_INSTALL}" == "true" ]]; then + install_and_register + else + log_info "Skipping APK installation (--no-install requested)." + fi + + case "${SCENARIO}" in + dev-catalog) + run_dev_catalog + ;; + reviewer-catalog) + run_reviewer_catalog + ;; + sample) + run_sample false + ;; + sample-reviewer) + run_sample true + ;; + sample-info) + run_sample_info + ;; + install-only) + log_success "Install & register completed." + ;; + full-suite) + run_dev_catalog + run_reviewer_catalog + run_sample_info + log_success "Full verification suite completed successfully!" + ;; + *) + log_error "Unrecognized scenario: ${SCENARIO}" + print_usage + exit 1 + ;; + esac + + echo "" + log_success "==========================================================" + log_success "Verification Sequence Complete!" + log_success "Artifacts written to: ${OUTPUT_DIR}" + log_success "==========================================================" +} + +main "$@" diff --git a/settings.gradle.kts b/settings.gradle.kts index af2968cd4..1eef03a80 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -70,3 +70,8 @@ project(":snippets:app-utils").projectDir = file("snippets/app-utils") include(":tutorials:kotlin:Polygons") project(":tutorials:kotlin:Polygons").projectDir = file("tutorials/kotlin/Polygons/app") // Add others as needed, starting with these for now + +// Visual Testing +include(":visual-testing") +project(":visual-testing").projectDir = file("visual-testing") + diff --git a/visual-testing/build.gradle.kts b/visual-testing/build.gradle.kts new file mode 100644 index 000000000..fb7d58a69 --- /dev/null +++ b/visual-testing/build.gradle.kts @@ -0,0 +1,65 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.gradle.api.tasks.testing.Test +import org.jetbrains.kotlin.gradle.dsl.JvmTarget + +plugins { + alias(libs.plugins.android.library) +} + +android { + namespace = "com.google.maps.android.visualtesting" + compileSdk = libs.versions.compileSdk.get().toInt() + + defaultConfig { + minSdk = libs.versions.minSdk.get().toInt() + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + consumerProguardFiles("consumer-rules.pro") + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro" + ) + } + } + kotlin { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_17) + } + jvmToolchain(17) + } + testOptions { + animationsDisabled = true + unitTests.isIncludeAndroidResources = true + unitTests.isReturnDefaultValues = true + } +} + +dependencies { + testImplementation(libs.junit) + testImplementation(libs.robolectric) + testImplementation(libs.truth) + + // Dependencies for GeminiVisualTestHelper + implementation(libs.ktor.client.core) + implementation(libs.ktor.client.cio) + implementation(libs.uiautomator) +} diff --git a/visual-testing/consumer-rules.pro b/visual-testing/consumer-rules.pro new file mode 100644 index 000000000..43264589f --- /dev/null +++ b/visual-testing/consumer-rules.pro @@ -0,0 +1 @@ +# Consumer Proguard rules diff --git a/visual-testing/proguard-rules.pro b/visual-testing/proguard-rules.pro new file mode 100644 index 000000000..70b1b3662 --- /dev/null +++ b/visual-testing/proguard-rules.pro @@ -0,0 +1 @@ +# Proguard rules diff --git a/visual-testing/src/main/AndroidManifest.xml b/visual-testing/src/main/AndroidManifest.xml new file mode 100644 index 000000000..342a838f1 --- /dev/null +++ b/visual-testing/src/main/AndroidManifest.xml @@ -0,0 +1,20 @@ + + + + + + diff --git a/visual-testing/src/main/java/com/google/maps/android/visualtesting/GeminiVisualTestHelper.kt b/visual-testing/src/main/java/com/google/maps/android/visualtesting/GeminiVisualTestHelper.kt new file mode 100644 index 000000000..4cd19d165 --- /dev/null +++ b/visual-testing/src/main/java/com/google/maps/android/visualtesting/GeminiVisualTestHelper.kt @@ -0,0 +1,247 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.maps.android.visualtesting + +import android.graphics.Bitmap +import android.util.Base64 +import android.util.Log +import androidx.test.uiautomator.By +import androidx.test.uiautomator.UiDevice +import androidx.test.uiautomator.Until +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.HttpTimeout +import io.ktor.client.request.get +import io.ktor.client.request.headers +import io.ktor.client.request.post +import io.ktor.client.request.setBody +import io.ktor.client.statement.HttpResponse +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpStatusCode +import io.ktor.http.contentType +import org.json.JSONArray +import org.json.JSONObject +import java.io.ByteArrayOutputStream +import java.io.Closeable + +/** + * Helper class to interact with the Gemini API for visual verification and action. + * + * This version uses org.json for parsing to avoid binary compatibility issues with kotlinx.serialization. + */ +class GeminiVisualTestHelper : Closeable { + + private val client = HttpClient(CIO) { + install(HttpTimeout) { + requestTimeoutMillis = 60_000 + connectTimeoutMillis = 60_000 + socketTimeoutMillis = 60_000 + } + } + + override fun close() { + client.close() + } + + /** + * Executes a UI action based on a natural language prompt. + * It analyzes the current UI hierarchy and asks Gemini to determine the best action. + */ + suspend fun performActionFromPrompt(prompt: String, uiDevice: UiDevice, apiKey: String) { + val hierarchyStream = ByteArrayOutputStream() + uiDevice.dumpWindowHierarchy(hierarchyStream) + val hierarchyXml = hierarchyStream.toString("UTF-8") + + val systemPrompt = """ + You are an expert Android QA automaton. Your task is to translate a natural language command + into a specific action to be performed on a UI. Given a UI hierarchy (in XML format), + determine the correct action and selector. + + The available actions are: "click", "longClick", "setText". + + Your response MUST be a single, well-formed JSON object with "action" and "selector" keys. + The "selector" object must contain exactly one of "text", "contentDescription", or "resourceId". + If the action is "setText", you must also include a "textValue" field at the top level. + + Example for a click: + { "action": "click", "selector": { "text": "Login" } } + + Example for setting text: + { "action": "setText", "selector": { "resourceId": "com.example.app:id/email_input" }, "textValue": "test@example.com" } + """.trimIndent() + + val fullPrompt = "$systemPrompt\n\nCommand: \"$prompt\"\n\nUI Hierarchy:\n$hierarchyXml" + + val requestJson = JSONObject().apply { + put("contents", JSONArray().apply { + put(JSONObject().apply { + put("parts", JSONArray().apply { + put(JSONObject().apply { put("text", fullPrompt) }) + }) + }) + }) + } + + val response: HttpResponse = client.post("$BASE_URL/$API_VERSION/models/$DEFAULT_MODEL:generateContent") { + contentType(ContentType.Application.Json) + headers.append(HEADER_API_KEY, apiKey) + setBody(requestJson.toString()) + } + + if (response.status != HttpStatusCode.OK) { + val errorBody = response.bodyAsText() + Log.e(TAG, "Action API Error: ${response.status} $errorBody") + throw Exception("Gemini Action API returned an error: ${response.status}\n$errorBody") + } + + val rawBody = response.bodyAsText() + val jsonResponse = JSONObject(rawBody) + val actionJson = jsonResponse.getJSONArray("candidates") + .getJSONObject(0) + .getJSONObject("content") + .getJSONArray("parts") + .getJSONObject(0) + .getString("text") + + // Remove markdown code block delimiters if present + val cleanedActionJson = actionJson.removePrefix("```json\n").removeSuffix("\n```") + + Log.d(TAG, "Received Action JSON: $cleanedActionJson") + + try { + val aiAction = JSONObject(cleanedActionJson) + val action = aiAction.getString("action") + val selectorObj = aiAction.getJSONObject("selector") + + val selector = when { + selectorObj.has("text") -> By.text(selectorObj.getString("text")) + selectorObj.has("contentDescription") -> By.desc(selectorObj.getString("contentDescription")) + selectorObj.has("resourceId") -> By.res(selectorObj.getString("resourceId")) + else -> throw IllegalArgumentException("Selector must have text, contentDescription, or resourceId.") + } + + val uiObject = uiDevice.wait(Until.findObject(selector), 10000) + ?: throw Exception("Could not find UI element for selector: $selector") + + when (action.lowercase()) { + "click" -> uiObject.click() + "longclick" -> uiObject.longClick() + "settext" -> { + val textToSet = aiAction.getString("textValue") + uiObject.text = textToSet + } + else -> throw UnsupportedOperationException("Action '$action' is not supported.") + } + } catch (e: Exception) { + Log.e(TAG, "Failed to parse or execute AI action", e) + throw e + } + } + + /** + * Fetches and logs the list of available Gemini models for the given API key. + */ + suspend fun listAvailableModels(apiKey: String) { + try { + val response: HttpResponse = client.get("$BASE_URL/$API_VERSION/models") { + headers.append(HEADER_API_KEY, apiKey) + } + val rawBody = response.bodyAsText() + val jsonResponse = JSONObject(rawBody) + val models = jsonResponse.getJSONArray("models") + val modelNames = StringBuilder() + for (i in 0 until models.length()) { + val model = models.getJSONObject(i) + modelNames.append(" - ${model.getString("name")} (Display Name: ${model.getString("displayName")})\n") + } + Log.i(TAG, "Available Gemini Models:\n$modelNames") + } catch (e: Exception) { + Log.e(TAG, "Failed to list available models", e) + } + } + + /** + * Analyzes an image with a given prompt using the Gemini API. + */ + suspend fun analyzeImage( + bitmap: Bitmap, + prompt: String, + apiKey: String, + model: String = DEFAULT_MODEL + ): String? { + val base64Image = bitmap.toBase64EncodedJpeg() + + val requestJson = JSONObject().apply { + put("contents", JSONArray().apply { + put(JSONObject().apply { + put("parts", JSONArray().apply { + put(JSONObject().apply { put("text", prompt) }) + put(JSONObject().apply { + put("inline_data", JSONObject().apply { + put("mime_type", "image/jpeg") + put("data", base64Image) + }) + }) + }) + }) + }) + } + + val response: HttpResponse = client.post("$BASE_URL/$API_VERSION/models/$model:generateContent") { + contentType(ContentType.Application.Json) + headers.append(HEADER_API_KEY, apiKey) + setBody(requestJson.toString()) + } + + if (response.status != HttpStatusCode.OK) { + val errorBody = response.bodyAsText() + Log.e(TAG, "API Error: ${response.status} $errorBody") + throw Exception("Gemini API returned an error: ${response.status}\n$errorBody") + } + + val rawBody = response.bodyAsText() + val jsonResponse = JSONObject(rawBody) + + val candidates = jsonResponse.optJSONArray("candidates") + if (candidates == null || candidates.length() == 0) { + Log.w(TAG, "Gemini API returned empty candidates. Full response: $rawBody") + throw Exception("Gemini API returned no candidates.") + } + + return candidates.getJSONObject(0) + .getJSONObject("content") + .getJSONArray("parts") + .getJSONObject(0) + .optString("text") + } + + companion object { + private const val TAG = "GeminiVisualTestHelper" + private const val BASE_URL = "https://generativelanguage.googleapis.com" + private const val API_VERSION = "v1beta" + private const val DEFAULT_MODEL = "gemini-2.5-flash" + private const val HEADER_API_KEY = "x-goog-api-key" + } + + private fun Bitmap.toBase64EncodedJpeg(): String { + val outputStream = ByteArrayOutputStream() + compress(Bitmap.CompressFormat.JPEG, 80, outputStream) + val byteArray = outputStream.toByteArray() + return Base64.encodeToString(byteArray, Base64.NO_WRAP) + } +} diff --git a/visual-testing/src/test/java/com/google/maps/android/visualtesting/PlaceholderTest.java b/visual-testing/src/test/java/com/google/maps/android/visualtesting/PlaceholderTest.java new file mode 100644 index 000000000..7452a2472 --- /dev/null +++ b/visual-testing/src/test/java/com/google/maps/android/visualtesting/PlaceholderTest.java @@ -0,0 +1,27 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.maps.android.visualtesting; + +import org.junit.Test; +import static org.junit.Assert.assertTrue; + +public class PlaceholderTest { + @Test + public void testPlaceholder() { + assertTrue(true); + } +}