diff --git a/.github/workflows/ft-afs-auth-debug.yml b/.github/workflows/ft-afs-auth-debug.yml index 90e40f5..4637f74 100644 --- a/.github/workflows/ft-afs-auth-debug.yml +++ b/.github/workflows/ft-afs-auth-debug.yml @@ -84,3 +84,43 @@ jobs: http=row.get("httpRequest") or {} print(row.get("timestamp"), "status=", http.get("status"), "size=", http.get("responseSize"), "latency=", http.get("latency"), "ua=", http.get("userAgent"), "protocol=", http.get("protocol")) PY + # App reason codes / exceptions around the latest AFS attempt (no secrets). + last_ts="$(python3 - <<'PY' + import json + rows=json.load(open("/tmp/logs.json")) + print((rows[0].get("timestamp") if rows else "") or "") + PY + )" + if [ -n "${last_ts}" ]; then + echo "app_log_window_around=${last_ts}" + app_filter='resource.type="cloud_run_revision" AND resource.labels.service_name="'"${svc}"'" AND timestamp>="'"${last_ts}"'"' + # Expand: 30s before via python + app_filter="$(LAST_TS="${last_ts}" SVC="${svc}" python3 - <<'PY' + import datetime as dt, os + ts = os.environ["LAST_TS"].replace("Z", "+00:00") + when = dt.datetime.fromisoformat(ts) + start = (when - dt.timedelta(seconds=30)).isoformat().replace("+00:00", "Z") + end = (when + dt.timedelta(seconds=30)).isoformat().replace("+00:00", "Z") + svc = os.environ["SVC"] + print( + f'resource.type="cloud_run_revision" AND resource.labels.service_name="{svc}" ' + f'AND timestamp>="{start}" AND timestamp<="{end}"' + ) + PY + )" + gcloud logging read "${app_filter}" --project="${GCP_PROJECT_ID}" --limit=40 --format=json > /tmp/app-logs.json + python3 - <<'PY' + import json + rows=json.load(open("/tmp/app-logs.json")) + print("app_log_count", len(rows)) + for row in rows: + http=row.get("httpRequest") or {} + sev=row.get("severity") + text=row.get("textPayload") or "" + jp=row.get("jsonPayload") or {} + msg=text or jp.get("message") or jp.get("msg") or "" + if not msg and not http: + continue + print(row.get("timestamp"), "sev=", sev, "status=", http.get("status"), "msg=", str(msg)[:400]) + PY + fi diff --git a/.github/workflows/invoke-cloud-run.yml b/.github/workflows/invoke-cloud-run.yml index c760424..849b8a7 100644 --- a/.github/workflows/invoke-cloud-run.yml +++ b/.github/workflows/invoke-cloud-run.yml @@ -513,7 +513,7 @@ jobs: if [ "${attempt_seen}" = "true" ]; then if [ "${{ steps.service.outputs.path }}" = "/account-facts-sync" ]; then - filter="resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"${CLOUD_RUN_SERVICE}\" AND timestamp>=\"${started_at}\" AND (httpRequest.requestUrl:\"/account-facts-sync\" OR textPayload:\"account_facts\" OR textPayload:\"Firstrade\" OR textPayload:\"unauthorized\" OR jsonPayload.message:\"account_facts\")" + filter="resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"${CLOUD_RUN_SERVICE}\" AND timestamp>=\"${started_at}\" AND (httpRequest.requestUrl:\"/account-facts-sync\" OR textPayload:\"account_facts\" OR textPayload:\"account_facts_sync\" OR textPayload:\"reason=\" OR textPayload:\"exc_type=\" OR textPayload:\"Firstrade\" OR textPayload:\"unauthorized\" OR jsonPayload.message:\"account_facts\")" gcloud logging read "${filter}" --project="${GCP_PROJECT_ID}" --limit=40 --format=json > /tmp/afs-logs.json || true python3 - <<'PY' import json diff --git a/main.py b/main.py index 4b46314..d7773ed 100644 --- a/main.py +++ b/main.py @@ -939,10 +939,23 @@ def account_facts_sync(): account_scope=str(runtime_target.account_scope), ) except AccountFactsUnavailable as exc: + app.logger.warning("account_facts_sync collect failed reason=%s", exc.reason_code) return jsonify({"ok": False, "error": exc.reason_code}), 503 except AccountFactsPublishError as exc: + app.logger.warning("account_facts_sync build failed reason=%s", exc.reason_code) return jsonify({"ok": False, "error": exc.reason_code}), 503 - except Exception: + except FirstradePlatformError as exc: + # connect_read_only / cached session failures land here (not AccountFactsUnavailable). + app.logger.warning( + "account_facts_sync client failed exc_type=%s", + type(exc).__name__, + ) + return jsonify({"ok": False, "error": "readonly_client_unavailable"}), 503 + except Exception as exc: + app.logger.warning( + "account_facts_sync unavailable exc_type=%s", + type(exc).__name__, + ) return jsonify({"ok": False, "error": "account_facts_unavailable"}), 503 finally: if client is not None: @@ -954,8 +967,10 @@ def account_facts_sync(): try: publish_firstrade_account_snapshot(payload, config) except AccountFactsPublishError as exc: + app.logger.warning("account_facts_sync publish failed reason=%s", exc.reason_code) status = 502 if exc.reason_code.startswith("sync_") else 503 return jsonify({"ok": False, "error": exc.reason_code}), status + app.logger.info("account_facts_sync published ok") return jsonify({"ok": True, "stored": True}), 200