[Logs] Adding debug logs for Swarming & Android

src/clusterfuzz/_internal/bot/testcase_manager.py

@@ -341,6 +341,10 @@ def run_testcase(thread_index, file_path, gestures, env_copy):
     command = get_command_line_for_application(
         file_path, user_profile_index=thread_index, needs_http=needs_http)
 
+    logs.debug(
+        f'[TestcaseManager] Running testcase (thread {thread_index}): '
+        f'command={command}, needs_http={needs_http}, gestures={gestures}')
+
     # Run testcase.
     return process_handler.run_process(
         command,

src/clusterfuzz/_internal/platforms/android/adb.py

@@ -679,14 +679,18 @@ def run_command(cmd, log_output=False, timeout=None, recover=True):
   if isinstance(cmd, list):
     cmd = ' '.join([str(i) for i in cmd])
   if log_output:
-    logs.info('Running: adb %s' % cmd)
+    logs.info(f'[ADB] Running: {cmd}')
+  else:
+    logs.debug(f'[ADB] Running: {cmd}')
   if not timeout:
     timeout = ADB_TIMEOUT
 
   output = execute_command(get_adb_command_line(cmd), timeout)
   if not recover:
     if log_output:
-      logs.info('Output: (%s)' % output)
+      logs.info(f'[ADB] Output for {cmd}: {output}')
+    else:
+      logs.debug(f'[ADB] Output for {cmd}: {output}')
     return output
 
   device_not_found_string_with_serial = DEVICE_NOT_FOUND_STRING.format(

src/clusterfuzz/_internal/platforms/android/fetch_artifact.py

@@ -56,7 +56,7 @@ def _call_android_api_enabled():
   Disabled always if invoked in a uworker
   """
   if environment.is_uworker():
-    logs.info('AndroidBuildAPI access disabled for uworker.')
+    logs.debug('AndroidBuildAPI access disabled for uworker.')
     return False
 
   flag = feature_flags.FeatureFlags.CALL_ANDROID_API.flag
@@ -66,19 +66,7 @@ def _call_android_api_enabled():
 def _download_artifact(client, bid, target, attempt_id, name, output_directory,
                        output_filename):
   """Download one artifact."""
-  logs.info('reached download_artifact')
-  logs.info('artifact to download: %s' % name)
-  logs.info('output_directory: %s' % output_directory)
-  logs.info('output_filename: %s' % output_filename)
-
-  logs.info(
-      'AndroidBuildAPI download_artifact started.',
-      api_version=API_VERSION,
-      operation='download_artifact',
-      build_id=bid,
-      target=target,
-      attempt_id=attempt_id,
-      artifact_name=name)
+  logs.debug('Artifact to download: %s' % name)
 
   artifact = client.get_artifact_metadata(bid, target, attempt_id, name)
 
@@ -126,28 +114,15 @@ def _download_artifact(client, bid, target, attempt_id, name, output_directory,
         status='skipped_exists')
     return output_path
 
-  logs.info('Downloading artifact %s.' % name)
   output_dir = os.path.dirname(output_path)
-  logs.info('Output dir: %s' % output_dir)
   if not os.path.exists(output_dir):
     logs.info(f'Creating directory {output_dir}')
     os.makedirs(output_dir, exist_ok=True)
 
-  # TODO(b/537368595) Remove unnecesary logging.
-  # Just like get, except get_media.
-  logs.info(
-      'AndroidBuildAPI download_artifact media download started.',
-      api_version=API_VERSION,
-      operation='download_artifact_media',
-      build_id=bid,
-      target=target,
-      attempt_id=attempt_id,
-      artifact_name=name)
-
   success = client.download_artifact_file(bid, target, attempt_id, name,
                                           output_path)
   if not success:
-    logs.error(
+    logs.warning(
         'AndroidBuildAPI download_artifact failed.',
         api_version=API_VERSION,
         operation='download_artifact',
@@ -185,15 +160,6 @@ def _get_artifacts_for_build(client,
         target=target)
     return []
 
-  logs.info(
-      'AndroidBuildAPI get_artifacts_for_build started.',
-      api_version=API_VERSION,
-      operation='get_artifacts_for_build',
-      build_id=bid,
-      target=target,
-      attempt_id=attempt_id,
-      regexp=regexp)
-
   artifacts = client.list_artifacts(bid, target, attempt_id, regexp=regexp)
 
   logs.info(
@@ -219,13 +185,13 @@ def _get_client():
   build_apiary_service_account_private_key = db_config.get_value(
       'build_apiary_service_account_private_key')
   if not build_apiary_service_account_private_key:
-    logs.info(
+    logs.warning(
         'Android build apiary credentials are not set, skip artifact fetch.')
     return None
 
   key_dict = json.loads(build_apiary_service_account_private_key)
 
-  logs.info(
+  logs.debug(
       'AndroidBuildAPI client initialization started.', api_version=API_VERSION)
 
   try:
@@ -241,7 +207,7 @@ def _get_client():
 
 def _get_stable_build_info():
   """Return stable artifact for cuttlefish branch and target."""
-  logs.info('Reached get_stable_build_info')
+  logs.debug('Reached get_stable_build_info')
   stable_build_info = STABLE_CUTTLEFISH_BUILD
 
   try:
@@ -260,7 +226,7 @@ def _get_stable_build_info():
 def get_latest_artifact_info(branch, target, signed=False, stable_build=False):
   """Return latest artifact for a branch and target."""
   if not _call_android_api_enabled():
-    logs.warning(
+    logs.debug(
         'Android build API is disabled by feature flag call_android_api.')
     return None
 
@@ -279,14 +245,6 @@ def get_latest_artifact_info(branch, target, signed=False, stable_build=False):
     if 'bid' in build_info and build_info['bid'] != '0':
       return build_info
 
-  logs.info(
-      'AndroidBuildAPI get_latest_artifact_info started.',
-      api_version=API_VERSION,
-      operation='get_latest_artifact_info',
-      branch=branch,
-      target=target,
-      signed=signed)
-
   builds = client.list_builds(branch, target, signed)
 
   if not builds:
@@ -319,8 +277,6 @@ def get_latest_artifact_info(branch, target, signed=False, stable_build=False):
 def get(bid, target, regex, output_directory, output_filename=None):
   """Return artifact for a given build id, target and file regex."""
   if not _call_android_api_enabled():
-    logs.warning(
-        'Android build API is disabled by feature flag call_android_api.')
     return None
 
   client = _get_client()

src/clusterfuzz/_internal/platforms/android/symbols_downloader.py

@@ -198,8 +198,8 @@ def download_trusty_symbols_if_needed(symbols_directory, app_name, bid):
   if not bid:
     build_info = fetch_artifact.get_latest_artifact_info(branch, ab_target)
     if not build_info:
-      logs.error(f'Unable to fetch build info for branch {branch} '
-                 f'and target {ab_target}.')
+      logs.info(f'Unable to fetch build info for branch {branch} '
+                f'and target {ab_target}.')
       return
     bid = build_info['bid']
 

src/clusterfuzz/_internal/platforms/android/util.py

@@ -56,6 +56,7 @@ def _to_enum(enum_cls, raw_value: int | str):
   try:
     return enum_cls(int(raw_value))
   except (ValueError, TypeError):
+    logs.debug(f'[Android] Could not convert {raw_value} to {enum_cls}')
     return None
 
 
@@ -72,6 +73,8 @@ def _parse_exit_info_from_dumpsys(dumpsys_output: str,
     None otherwise.
   """
   if not dumpsys_output or target_pid is None:
+    logs.debug('[Android] Could not parse exit info from dumpsys - no output ' \
+               'or no pid')
     return None
 
   current_pid = None

src/clusterfuzz/_internal/swarming/__init__.py

@@ -68,18 +68,18 @@ def is_swarming_job(job_name: str, job: data_types.Job | None = None) -> bool:
   if job is None:
     job = data_types.Job.query(data_types.Job.name == job_name).get()
     if not job:
-      logs.info('[Swarming DEBUG] Job not found', job_name=job_name)
+      logs.debug('[Swarming] Job not found', job_name=job_name)
       return False
 
   if not has_swarming_env_vars(job.get_environment()):
-    logs.info('[Swarming DEBUG] No swarming env var', job_name=job_name)
+    logs.debug('[Swarming] No swarming env var', job_name=job_name)
     return False
 
   swarming_config = get_swarming_config()
   if swarming_config is None:
     logs.warning(
-        """[Swarming DEBUG] current task is not suitable for swarming. 
-    'Reason: failed to retrieve config.""",
+        """[Swarming] current task is not suitable for swarming.
+    Reason: failed to retrieve config.""",
         job_name=job_name)
     return False
 
@@ -110,7 +110,7 @@ def _get_task_dimensions(job: data_types.Job, platform_specific_dimensions: list
   Job dimensions have more precedence than static dimensions"""
   swarming_config = get_swarming_config()
   if not swarming_config:
-    logs.error(
+    logs.debug(
         '[Swarming] No dimensions set. Reason: failed to retrieve config')
     return []
 
@@ -160,7 +160,7 @@ def _append_metadata_env_var(
         swarming_pb2.StringPair(  # pylint: disable=no-member
             key=env_var_name, value=str(value)))
   else:
-    logs.warning(f'{env_var_name} is not set or cannot be fetched.')
+    logs.info(f'[Swarming] Metadata {env_var_name} cannot be fetched.')
 
 
 def _get_env_vars(logs_project_id: str,
@@ -221,14 +221,19 @@ def create_new_task_request(command: str, job_name: str, download_url: str
 
   job = data_types.Job.query(data_types.Job.name == job_name).get()
   if job is None:
+    logs.debug('[Swarming] Job not found', job_name=job_name)
     return None
 
   swarming_config = get_swarming_config()
   if not swarming_config:
+    logs.debug('[Swarming] Config not found', job_name=job_name)
     return None
 
   instance_spec = _get_instance_spec(swarming_config, job)
   if instance_spec is None:
+    logs.warning(
+        '[Swarming] No matching spec in swarming config for job.',
+        job_name=job_name)
     return None
 
   swarming_realm = swarming_config.get('swarming_realm',)

src/clusterfuzz/_internal/swarming/service.py

@@ -107,19 +107,25 @@ def create_utask_main_jobs(self,
     for i, task in enumerate(remote_tasks):
       if not swarming.is_swarming_job(task.job_type):
         unscheduled_tasks.append(task)
+        logs.debug(
+            '[Swarming] Task is not suitable for swarming.',
+            job_name=task.job_type)
         continue
 
       task_req = swarming.create_new_task_request(task.command, task.job_type,
                                                   task.argument)
       if not task_req:
         unscheduled_tasks.append(task)
+        logs.debug(
+            '[Swarming] Failed to create task request.', job_name=task.job_type)
         continue
 
       os_val = self._get_dimension(task_req, 'os')
       pool_val = self._get_dimension(task_req, 'pool')
       if not os_val or not pool_val:
-        logs.error(f'[Swarming] Failed to find required dimension for job '
-                   f'{task.job_type}. Discarding task')
+        logs.error(
+            '[Swarming] Missing dimensions for task request. Skipping task.',
+            job_name=task.job_type)
         continue
 
       # Since there are multiple concurrent scheduling sessions/bots, it is