diff --git a/.claude/routines/desk.md b/.claude/routines/desk.md index 8c0f8b140d..85c5c77ce1 100644 --- a/.claude/routines/desk.md +++ b/.claude/routines/desk.md @@ -78,9 +78,9 @@ Done when: the PR body starts with this run's "Desk last ran" line, or the new ` ### 8. Clean the queue -In QUEUE, remove every queue item whose worklist item ended `actioned` or `dismissed`, or `deferred` as a needs-maintainer result or verdict or a legacy suspect: those wait on the maintainer, whose answer on the PR comes back through step 5. Use `python utils/desk_queue.py` (see its `--help`) or edit the JSON as `QUEUE/README.md` describes, and delete each file left with no items. Leads, items whose subagent failed and items not reached stay queued. Write `ledger/pending.md` from step 7, or delete it once its text is in the PR. Commit these changes as one commit and push them following "Pushing" with `git -C QUEUE push origin HEAD:desk-queue`. +In QUEUE, remove every queue item whose worklist item ended `actioned` or `dismissed`, or `deferred` as a needs-maintainer result or verdict or a legacy suspect: those wait on the maintainer, whose answer on the PR comes back through step 5. Remove them with `python utils/desk_queue.py remove QUEUE ...`, which also records each id as handled, so the writers do not queue the same finding again. Leads, items whose subagent failed and items not reached stay queued. Write `ledger/pending.md` from step 7, or delete it once its text is in the PR. Commit these changes as one commit and push them following "Pushing" with `git -C QUEUE push origin HEAD:desk-queue`. -Done when: after `git -C QUEUE fetch origin desk-queue`, `origin/desk-queue` holds none of the removed item ids and its `ledger/pending.md` matches step 7. +Done when: after `git -C QUEUE fetch origin desk-queue`, no `items` list on `origin/desk-queue` holds a removed id, each removed id is in its file's `handled` list, and `ledger/pending.md` matches step 7. ### 9. Report diff --git a/.github/actions/desk-queue-write/action.yml b/.github/actions/desk-queue-write/action.yml new file mode 100644 index 0000000000..acbc004d17 --- /dev/null +++ b/.github/actions/desk-queue-write/action.yml @@ -0,0 +1,89 @@ +name: Write to the desk queue +description: >- + Add JSON-line records to the desk-queue branch with utils/desk_queue.py, + creating the branch when it does not exist. Run it after actions/checkout, + from the repository root, with contents: write. + +inputs: + records: + description: File of queue records, one JSON object per line. A missing or empty file writes nothing. + required: true + message: + description: Commit message for the queue commit. + required: true + +outputs: + added: + description: Number of items the push added to the queue. + value: ${{ steps.write.outputs.added }} + +runs: + using: composite + steps: + - id: write + shell: bash + env: + RECORDS: ${{ inputs.records }} + MESSAGE: ${{ inputs.message }} + BRANCH: desk-queue + INIT_BRANCH: desk-queue-init + run: | + set -euo pipefail + cd "$GITHUB_WORKSPACE" + + if [ ! -s "$RECORDS" ]; then + echo "No records to queue" + echo "added=0" >> "$GITHUB_OUTPUT" + exit 0 + fi + + SCRIPT="$GITHUB_WORKSPACE/utils/desk_queue.py" + QUEUE="$RUNNER_TEMP/desk-queue" + RESULTS="$RUNNER_TEMP/desk-queue-results.txt" + + # The queue only grows by appends keyed on stable ids, so a rejected + # push is retried from a fresh copy of the branch; never forced. + for attempt in 1 2 3 4 5 6 7 8; do + git worktree remove --force "$QUEUE" 2>/dev/null || rm -rf "$QUEUE" + git worktree prune + git branch -D "$INIT_BRANCH" 2>/dev/null || true + + if git ls-remote --exit-code origin "refs/heads/$BRANCH" > /dev/null; then + git fetch --quiet --depth=1 origin "$BRANCH" + git worktree add --quiet --detach "$QUEUE" FETCH_HEAD + else + echo "::notice title=Desk queue::Creating the $BRANCH branch" + git worktree add --quiet --detach "$QUEUE" + git -C "$QUEUE" checkout --quiet --orphan "$INIT_BRANCH" + git -C "$QUEUE" rm -rfq . + fi + + # Rejected records and broken queue files are reported as warnings; + # the valid records are still written. + python3 "$SCRIPT" add "$QUEUE" --jsonl "$RECORDS" \ + 2> >(sed 's/^desk_queue: /::warning title=Desk queue::/' >&2) | tee "$RESULTS" + git -C "$QUEUE" add -A + if git -C "$QUEUE" diff --cached --quiet; then + echo "Nothing new to queue" + echo "added=0" >> "$GITHUB_OUTPUT" + exit 0 + fi + + git -C "$QUEUE" \ + -c user.name="github-actions[bot]" \ + -c user.email="github-actions[bot]@users.noreply.github.com" \ + commit --quiet -m "$MESSAGE" + if git -C "$QUEUE" push --quiet origin "HEAD:refs/heads/$BRANCH"; then + ADDED=$(grep -c '^added ' "$RESULTS" || true) + echo "added=$ADDED" >> "$GITHUB_OUTPUT" + echo "::notice title=Desk queue::Queued $ADDED item(s)" + exit 0 + fi + + WAIT=$(( attempt * 2 + RANDOM % 5 )) + echo "Push rejected (attempt $attempt), retrying in ${WAIT}s" + sleep "$WAIT" + done + + echo "::error title=Desk queue::Could not push to $BRANCH after 8 attempts" + exit 1 diff --git a/.github/workflows/canonical-source-update.yml b/.github/workflows/canonical-source-update.yml index 665f74ac32..c289f5c63d 100644 --- a/.github/workflows/canonical-source-update.yml +++ b/.github/workflows/canonical-source-update.yml @@ -29,6 +29,8 @@ concurrency: cancel-in-progress: false permissions: + # actions: write lets the job start the TBA enrichment run + actions: write contents: write pull-requests: write @@ -125,6 +127,9 @@ jobs: id: merge env: SOURCE: ${{ steps.source.outputs.source }} + # Questions the merge would ask a person are answered with their + # default here and written to this file for the desk to check. + DESK_QUEUE_OUT: ${{ runner.temp }}/desk-records.jsonl run: | echo "::group::Running merge for source: $SOURCE" @@ -150,6 +155,15 @@ jobs: # Capture which source was processed for commit message echo "source_label=$SOURCE" >> $GITHUB_OUTPUT + # A queue problem must not hold back the data merge. + - name: Queue the merge's open questions for the desk + if: inputs.dry_run != true + continue-on-error: true + uses: ./.github/actions/desk-queue-write + with: + records: ${{ runner.temp }}/desk-records.jsonl + message: "Queue merge questions from canonical source ${{ steps.source.outputs.source }}" + - name: Verify no conference data loss run: | # An automated merge must never delete or rename existing conferences. @@ -302,7 +316,7 @@ jobs: GH_TOKEN: ${{ github.token }} run: | echo "::notice title=Triggering TBA Enrichment::Starting TBA conference enrichment workflow" - gh workflow run enrich-tba-conferences.yml -f enrichment_level=full || { + gh workflow run enrich-tba-conferences.yml || { echo "::warning title=Trigger Failed::Could not trigger TBA enrichment workflow" } diff --git a/.github/workflows/check-conference-update.yml b/.github/workflows/check-conference-update.yml index b243bcf246..0a1cccd596 100644 --- a/.github/workflows/check-conference-update.yml +++ b/.github/workflows/check-conference-update.yml @@ -1,5 +1,9 @@ name: Check Conference Update +# A changedetection.io watch saw a conference page change. The change goes to +# the desk queue as it is: the desk checks it against the data and the live +# site on its next run, so nothing is filtered or edited here. + on: # Triggered by changedetection.io webhook repository_dispatch: @@ -13,1320 +17,113 @@ on: required: true type: string conference_name: - description: 'Conference name (e.g., "PyCon US")' + description: 'Conference name (e.g., "PyCon US 2027")' required: true type: string - skip_triage: - description: 'Skip triage and go straight to full analysis' - required: false - type: boolean - default: false - force_zone: - description: 'Override zone detection (normal/slow/dead)' + note: + description: 'What to check, for the desk (optional)' required: false type: string default: '' -# Bug #3 Fix: Per-conference concurrency to prevent duplicate triggers -# Different conferences can run in parallel, but the same conference serializes -# This prevents race conditions when changedetection.io sends multiple notifications -concurrency: - group: conference-update-${{ github.event.client_payload.watch_uuid || inputs.url || github.run_id }} - cancel-in-progress: false - permissions: contents: write - pull-requests: write - issues: write jobs: - # Stage 0: Determine monitoring zone BEFORE spending money on triage - # This is FREE - just reads YAML and does date math - zone-check: + queue: runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - zone: ${{ steps.detect.outputs.zone }} - skip: ${{ steps.detect.outputs.skip }} - triage_mode: ${{ steps.detect.outputs.triage_mode }} - url: ${{ steps.vars.outputs.url }} - conference: ${{ steps.vars.outputs.conference }} - diff_b64: ${{ steps.vars.outputs.diff_b64 }} - watch_uuid: ${{ steps.vars.outputs.watch_uuid }} - has_diff: ${{ steps.vars.outputs.has_diff }} - has_pending: ${{ steps.pending.outputs.has_pending }} - # Trigger context from scheduled extension checker - trigger_reason: ${{ steps.vars.outputs.trigger_reason }} - original_cfp_deadline: ${{ steps.vars.outputs.original_cfp_deadline }} + timeout-minutes: 10 steps: - name: Checkout repository uses: actions/checkout@v7 - - name: Validate payload - id: validate - env: - EVENT_NAME: ${{ github.event_name }} - PAYLOAD_URL: ${{ github.event.client_payload.url }} - PAYLOAD_TITLE: ${{ github.event.client_payload.title }} - PAYLOAD_UUID: ${{ github.event.client_payload.watch_uuid }} - INPUT_URL: ${{ inputs.url }} - INPUT_CONFERENCE: ${{ inputs.conference_name }} - run: | - # Payload validation: Fail fast with clear error messages if required fields are missing - # This helps diagnose issues like expired tokens or malformed webhook payloads - - echo "::group::Payload Validation" - echo "Event: $EVENT_NAME" - - if [ "$EVENT_NAME" = "repository_dispatch" ]; then - MISSING_FIELDS="" - - if [ -z "$PAYLOAD_URL" ]; then - MISSING_FIELDS="${MISSING_FIELDS}url, " - echo "::error title=Missing Field::client_payload.url is required for repository_dispatch" - fi - - if [ -z "$PAYLOAD_TITLE" ]; then - MISSING_FIELDS="${MISSING_FIELDS}title, " - echo "::error title=Missing Field::client_payload.title is required for repository_dispatch" - fi - - if [ -z "$PAYLOAD_UUID" ]; then - echo "::warning title=Missing Field::client_payload.watch_uuid is missing - concurrency may not work correctly" - fi - - if [ -n "$MISSING_FIELDS" ]; then - echo "::endgroup::" - echo "" - echo "## ❌ Payload Validation Failed" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Missing required fields: ${MISSING_FIELDS%, }" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "This usually indicates one of:" >> $GITHUB_STEP_SUMMARY - echo "- The GitHub PAT token used by changedetection.io is expired or has insufficient permissions" >> $GITHUB_STEP_SUMMARY - echo "- The webhook payload is malformed" >> $GITHUB_STEP_SUMMARY - echo "- The repository_dispatch event was triggered with incomplete data" >> $GITHUB_STEP_SUMMARY - exit 1 - fi - - echo "✓ All required fields present for repository_dispatch" - else - # workflow_dispatch validation - if [ -z "$INPUT_URL" ]; then - echo "::error title=Missing Input::url input is required for manual trigger" - echo "::endgroup::" - exit 1 - fi - - if [ -z "$INPUT_CONFERENCE" ]; then - echo "::error title=Missing Input::conference_name input is required for manual trigger" - echo "::endgroup::" - exit 1 - fi - - echo "✓ All required inputs present for workflow_dispatch" - fi - echo "::endgroup::" - - - name: Set up variables - id: vars - env: - # Pass webhook payload via env to prevent injection - PAYLOAD_URL: ${{ github.event.client_payload.url }} - PAYLOAD_TITLE: ${{ github.event.client_payload.title }} - PAYLOAD_DIFF: ${{ toJson(github.event.client_payload.diff) }} - PAYLOAD_UUID: ${{ github.event.client_payload.watch_uuid }} - # Trigger context from scheduled extension checker - PAYLOAD_TRIGGER_REASON: ${{ github.event.client_payload.trigger_reason }} - PAYLOAD_ORIGINAL_CFP: ${{ github.event.client_payload.original_cfp_deadline }} - INPUT_URL: ${{ inputs.url }} - INPUT_CONFERENCE: ${{ inputs.conference_name }} - EVENT_NAME: ${{ github.event_name }} - run: | - # Bug #2 Fix: Use base64 encoding for diff content to prevent heredoc delimiter collision - # The diff can contain arbitrary website content including strings that look like delimiters - - if [ "$EVENT_NAME" = "repository_dispatch" ]; then - echo "url=$PAYLOAD_URL" >> $GITHUB_OUTPUT - echo "conference=$PAYLOAD_TITLE" >> $GITHUB_OUTPUT - # Base64 encode the diff to safely pass multi-line content with special characters - DIFF_B64=$(printf '%s' "$PAYLOAD_DIFF" | base64 -w 0) - echo "diff_b64=$DIFF_B64" >> $GITHUB_OUTPUT - echo "watch_uuid=$PAYLOAD_UUID" >> $GITHUB_OUTPUT - echo "has_diff=true" >> $GITHUB_OUTPUT - # Trigger context (empty if from changedetection.io, populated if from scheduled checker) - echo "trigger_reason=${PAYLOAD_TRIGGER_REASON:-website_change}" >> $GITHUB_OUTPUT - echo "original_cfp_deadline=${PAYLOAD_ORIGINAL_CFP:-}" >> $GITHUB_OUTPUT - else - echo "url=$INPUT_URL" >> $GITHUB_OUTPUT - echo "conference=$INPUT_CONFERENCE" >> $GITHUB_OUTPUT - echo "has_diff=false" >> $GITHUB_OUTPUT - echo "watch_uuid=" >> $GITHUB_OUTPUT - echo "diff_b64=" >> $GITHUB_OUTPUT - echo "trigger_reason=manual" >> $GITHUB_OUTPUT - echo "original_cfp_deadline=" >> $GITHUB_OUTPUT - fi - - - name: Check for pending updates - id: pending - env: - CONF_NAME: ${{ steps.vars.outputs.conference }} - CONF_URL: ${{ steps.vars.outputs.url }} - UPDATE_BRANCH: auto/conference-updates - GH_TOKEN: ${{ github.token }} - run: | - # Bug #1 Fix (Revised): Check if any open PR modifies the same conference FILE - # This is more precise than fuzzy title/branch matching - # Logic: Find PRs that modify _data/conferences.yml, then check if their diff includes this conference - - echo "::group::Checking for open PRs modifying ${CONF_NAME}" - - MATCHING_PR_NUM="" - TARGET_FILE="_data/conferences.yml" - - # Get all open PRs - OPEN_PRS=$(gh pr list --state open --json number,title --jq '.[].number' --limit 50 2>/dev/null || echo "") - - if [ -n "$OPEN_PRS" ]; then - echo "Found $(echo "$OPEN_PRS" | wc -w) open PRs, checking which ones modify $TARGET_FILE..." - - for PR_NUM in $OPEN_PRS; do - # Get files changed by this PR - CHANGED_FILES=$(gh pr view "$PR_NUM" --json files --jq '.files[].path' 2>/dev/null || echo "") - - # Check if this PR modifies the conferences file - if echo "$CHANGED_FILES" | grep -q "^${TARGET_FILE}$"; then - echo "PR #$PR_NUM modifies $TARGET_FILE, checking if it affects ${CONF_NAME}..." - - # Get the PR diff and search for this conference - # Use Python for safe string matching - PR_DIFF=$(gh pr diff "$PR_NUM" -- "$TARGET_FILE" 2>/dev/null || echo "") - - if [ -n "$PR_DIFF" ]; then - export PR_DIFF - FOUND=$(python3 << 'DIFFSEARCH' - import os - import re - - conf_name = os.environ.get('CONF_NAME', '').lower() - diff_content = os.environ.get('PR_DIFF', '').lower() - - # Normalize conference name for matching - # "PyCon US" -> also match "pycon us", "pycon-us", etc. - conf_slug = re.sub(r'[^a-z0-9]+', '-', conf_name).strip('-') - conf_compact = re.sub(r'[^a-z0-9]+', '', conf_name) - - # Check if this conference appears in the diff - if (conf_name in diff_content or - conf_slug in diff_content or - conf_compact in diff_content): - print("FOUND") - DIFFSEARCH - ) - - if [ "$FOUND" = "FOUND" ]; then - MATCHING_PR_NUM="$PR_NUM" - echo "PR #$PR_NUM modifies ${CONF_NAME}" - break - fi - fi - fi - done - - if [ -z "$MATCHING_PR_NUM" ]; then - echo "Checked all PRs - none modify ${CONF_NAME} in $TARGET_FILE" - fi - else - echo "No open PRs found" - fi - - if [ -n "$MATCHING_PR_NUM" ]; then - MATCHING_PR=$(gh pr view "$MATCHING_PR_NUM" --json title,url --jq '"\(.title) (\(.url))"' 2>/dev/null || echo "PR #$MATCHING_PR_NUM") - echo "has_pending=true" >> $GITHUB_OUTPUT - echo "pending_pr=$MATCHING_PR_NUM" >> $GITHUB_OUTPUT - echo "::notice title=Open PR Found::${CONF_NAME} already has an open PR: ${MATCHING_PR}" - echo "::endgroup::" - exit 0 - fi - - echo "::endgroup::" - - # Also check the accumulator branch for pending commits - echo "::group::Checking accumulator branch for pending commits" - if git ls-remote --heads origin "$UPDATE_BRANCH" | grep -q "$UPDATE_BRANCH"; then - git fetch origin "$UPDATE_BRANCH" --depth=50 - - # Look for commits mentioning this conference that aren't in main yet - if git log "origin/main..origin/$UPDATE_BRANCH" --oneline --grep="conf: ${CONF_NAME}" 2>/dev/null | grep -q .; then - PENDING_COMMIT=$(git log "origin/main..origin/$UPDATE_BRANCH" --oneline --grep="conf: ${CONF_NAME}" -1) - - # Check if there's an open PR from the update branch - OPEN_PR=$(gh pr list --head "$UPDATE_BRANCH" --state open --json number --jq '.[0].number' 2>/dev/null || echo "") - - if [ -n "$OPEN_PR" ]; then - echo "has_pending=true" >> $GITHUB_OUTPUT - echo "pending_pr=$OPEN_PR" >> $GITHUB_OUTPUT - echo "::notice title=Pending Update::${CONF_NAME} already has pending commit: ${PENDING_COMMIT} (PR #${OPEN_PR})" - else - echo "has_pending=false" >> $GITHUB_OUTPUT - echo "::notice title=Stale Branch::${CONF_NAME} has commit ${PENDING_COMMIT} but no open PR - proceeding with update" - fi - else - echo "has_pending=false" >> $GITHUB_OUTPUT - fi - else - echo "has_pending=false" >> $GITHUB_OUTPUT - fi - echo "::endgroup::" - - - name: Detect monitoring zone - id: detect - env: - CONF_NAME: ${{ steps.vars.outputs.conference }} - FORCE_ZONE: ${{ inputs.force_zone }} - run: | - # Allow manual override - if [ -n "$FORCE_ZONE" ]; then - echo "zone=$FORCE_ZONE" >> $GITHUB_OUTPUT - if [ "$FORCE_ZONE" = "dead" ]; then - echo "skip=true" >> $GITHUB_OUTPUT - else - echo "skip=false" >> $GITHUB_OUTPUT - fi - echo "triage_mode=$FORCE_ZONE" >> $GITHUB_OUTPUT - echo "::notice title=Zone Override::Manually set to $FORCE_ZONE" - exit 0 - fi - - # Run zone detection (safe - uses env vars, not interpolation) - python3 << 'ZONECHECK' > /tmp/zone_check.txt 2>&1 - import yaml - import os - import sys - from datetime import datetime, timedelta - - conf_name = os.environ.get('CONF_NAME', '') - today = datetime.now().date() - - # Load both conferences.yml and archive.yml - all_confs = [] - - try: - with open("_data/conferences.yml") as f: - confs = yaml.safe_load(f) or [] - all_confs.extend(confs) - except FileNotFoundError: - print("INFO: conferences.yml not found", file=sys.stderr) - except yaml.YAMLError as e: - print(f"WARNING: Failed to parse conferences.yml: {e}", file=sys.stderr) - - try: - with open("_data/archive.yml") as f: - archived = yaml.safe_load(f) or [] - all_confs.extend(archived) - except FileNotFoundError: - print("INFO: archive.yml not found", file=sys.stderr) - except yaml.YAMLError as e: - print(f"WARNING: Failed to parse archive.yml: {e}", file=sys.stderr) - - if not all_confs: - print("No conference data found - defaulting to NORMAL zone") - print("ZONE=normal") - print("SKIP=false") - print("TRIAGE_MODE=normal") - sys.exit(0) - - # Find the most recent entry for this conference (highest year) - matching = [c for c in all_confs if conf_name.lower() in c.get("conference", "").lower()] - conf = max(matching, key=lambda x: x.get("year", 0)) if matching else None - - if not conf: - print(f"Conference '{conf_name}' not found - NORMAL zone (discovery mode)") - print("ZONE=normal") - print("SKIP=false") - print("TRIAGE_MODE=normal") - sys.exit(0) - - # Get CFP date (use extension if available) - cfp_str = conf.get("cfp_ext") or conf.get("cfp", "TBA") - has_extension = conf.get("cfp_ext") is not None - - # Get conference end date - end_date = conf.get("end") - if isinstance(end_date, str): - end_date = datetime.strptime(end_date, "%Y-%m-%d").date() - - # No CFP yet = NORMAL zone (actively hunting for CFP) - if cfp_str in ("TBA", "Cancelled", "None"): - print(f"No CFP announced yet for {conf_name} - NORMAL zone (hunting mode)") - print("ZONE=normal") - print("SKIP=false") - print("TRIAGE_MODE=normal") - sys.exit(0) - - # Parse CFP date - try: - cfp_date = datetime.strptime(cfp_str[:10], "%Y-%m-%d").date() - except ValueError as e: - print(f"WARNING: Could not parse CFP date '{cfp_str}': {e}", file=sys.stderr) - print("ZONE=normal") - print("SKIP=false") - print("TRIAGE_MODE=normal") - sys.exit(0) - - dead_zone_end = end_date + timedelta(days=60) - - # CFP is still open = SLOW zone - if today <= cfp_date: - print(f"CFP open until {cfp_date} - SLOW zone (watching for extensions)") - print("ZONE=slow") - print("SKIP=false") - print("TRIAGE_MODE=slow") - sys.exit(0) - - # CFP just closed (within 3 days) and no extension yet - extension_window = cfp_date + timedelta(days=3) - if today <= extension_window and not has_extension: - print(f"CFP just closed ({cfp_date}) - extension_watch zone") - print("ZONE=extension_watch") - print("SKIP=false") - print("TRIAGE_MODE=slow") - sys.exit(0) - - # Between CFP close and conference end + 60 days = DEAD zone - if cfp_date < today <= dead_zone_end: - print(f"DEAD zone: CFP closed {cfp_date}, no updates until {dead_zone_end}") - print("ZONE=dead") - print("SKIP=true") - print("TRIAGE_MODE=dead") - sys.exit(0) - - # Past dead zone = NORMAL (hunting for next year's CFP) - print(f"Past dead zone ({dead_zone_end}) - NORMAL zone (hunting next year)") - print("ZONE=normal") - print("SKIP=false") - print("TRIAGE_MODE=normal") - ZONECHECK - - # Parse results - cat /tmp/zone_check.txt - - ZONE=$(grep "^ZONE=" /tmp/zone_check.txt | cut -d= -f2 || echo "normal") - SKIP=$(grep "^SKIP=" /tmp/zone_check.txt | cut -d= -f2 || echo "false") - TRIAGE_MODE=$(grep "^TRIAGE_MODE=" /tmp/zone_check.txt | cut -d= -f2 || echo "normal") - - echo "zone=${ZONE:-normal}" >> $GITHUB_OUTPUT - echo "skip=${SKIP:-false}" >> $GITHUB_OUTPUT - echo "triage_mode=${TRIAGE_MODE:-normal}" >> $GITHUB_OUTPUT - - echo "::group::Zone Detection Results" - echo "Conference: $CONF_NAME" - echo "Zone: $ZONE" - echo "Skip: $SKIP" - echo "Triage mode: $TRIAGE_MODE" - echo "::endgroup::" - - - name: Log zone detection - env: - CONFERENCE: ${{ steps.vars.outputs.conference }} - ZONE: ${{ steps.detect.outputs.zone }} - SKIP: ${{ steps.detect.outputs.skip }} - TRIAGE_MODE: ${{ steps.detect.outputs.triage_mode }} - run: | - echo "## Zone Detection" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Conference:** $CONFERENCE" >> $GITHUB_STEP_SUMMARY - echo "**Zone:** $ZONE" >> $GITHUB_STEP_SUMMARY - echo "**Skip:** $SKIP" >> $GITHUB_STEP_SUMMARY - echo "**Triage Mode:** $TRIAGE_MODE" >> $GITHUB_STEP_SUMMARY - - # Stage 1: Quick triage using DIRECT API call (no agent overhead) - # This is ~10x cheaper than using claude-code-action for simple classification - triage: - needs: zone-check - if: needs.zone-check.outputs.skip != 'true' && needs.zone-check.outputs.has_pending != 'true' - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - should_analyze: ${{ steps.parse.outputs.should_analyze }} - triage_reason: ${{ steps.parse.outputs.reason }} - suggested_filters: ${{ steps.parse.outputs.suggested_filters }} - is_new_year: ${{ steps.parse.outputs.is_new_year }} - detected_year: ${{ steps.parse.outputs.detected_year }} - - steps: - - name: Check if triage should be skipped - id: skip_check - env: - SKIP_TRIAGE: ${{ inputs.skip_triage }} - HAS_DIFF: ${{ needs.zone-check.outputs.has_diff }} - run: | - if [ "$SKIP_TRIAGE" = "true" ] || [ "$HAS_DIFF" = "false" ]; then - echo "skip=true" >> $GITHUB_OUTPUT - echo "::notice title=Triage Skipped::Manual trigger or no diff available" - else - echo "skip=false" >> $GITHUB_OUTPUT - fi - - # Direct API call - much cheaper than agent mode for simple classification - - name: Quick triage via API - id: triage_api - if: steps.skip_check.outputs.skip != 'true' + - name: Build the queue record + id: record env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - CONF_URL: ${{ needs.zone-check.outputs.url }} - CONF_NAME: ${{ needs.zone-check.outputs.conference }} - DIFF_B64: ${{ needs.zone-check.outputs.diff_b64 }} - TRIAGE_MODE: ${{ needs.zone-check.outputs.triage_mode }} - # Trigger context from scheduled extension checker - TRIGGER_REASON: ${{ needs.zone-check.outputs.trigger_reason }} - ORIGINAL_CFP_DEADLINE: ${{ needs.zone-check.outputs.original_cfp_deadline }} + RECORDS: ${{ runner.temp }}/desk-records.jsonl run: | - # Bug #2 Fix: Decode base64 diff content - # The diff is base64 encoded to safely pass multi-line content with special characters - if [ -n "$DIFF_B64" ]; then - DIFF_CONTENT=$(echo "$DIFF_B64" | base64 -d 2>/dev/null || echo "Failed to decode diff") - else - DIFF_CONTENT="No diff available" - fi - export DIFF_CONTENT - - # Build the appropriate prompt based on trigger context and triage mode - # Priority: extension_check trigger > slow zone > general website change - # Export so Python subprocess can access it - if [ "$TRIGGER_REASON" = "extension_check" ]; then - # Triggered by scheduled extension checker - CFP deadline has passed - export PROMPT="You are checking for a CFP EXTENSION. The CFP deadline (${ORIGINAL_CFP_DEADLINE}) has passed. Focus ONLY on: 1) Whether the CFP was extended (new deadline announced), 2) Any 'extended', 'deadline moved', 'last chance' announcements. If you find a new CFP deadline date, mark RELEVANT and include the new date in REASON. If NO extension is mentioned, mark NOISE with REASON 'No CFP extension found'. Do NOT mark relevant for general website changes, schedules, speakers, etc." - echo "::notice title=Trigger Context::Scheduled CFP extension check (deadline was ${ORIGINAL_CFP_DEADLINE})" - elif [ "$TRIAGE_MODE" = "slow" ]; then - export PROMPT="You are a STRICT triage filter. This conference already has a CFP deadline - we ONLY care about EXTENSIONS. RELEVANT (extension indicators): extended, extension, new deadline, deadline moved, pushed back, more time, last chance. NOISE (everything else): speaker announcements, schedule, sponsors, registration, venue info. Be STRICT - if not clearly about deadline extension, mark NOISE." - else - export PROMPT="You are a triage filter for a Python conference tracker. NEW YEAR indicators: future year (2025/2026/2027) WITH SPECIFIC DATES (exact days like 'September 15-17', not just month or season). RELEVANT: CFP announcements with specific deadlines (exact date), specific conference dates announced (exact days). NOISE: copyright year only, timestamps, counters, typos, cookies, nav changes, sponsors, speakers, schedule. ALSO NOISE: Vague date announcements ('sometime in September', 'early 2026', 'Fall 2026', 'dates TBD', 'coming soon') - these are not actionable until specific dates are published." - fi - - # Create the API request payload - cat > /tmp/request.json << 'PAYLOAD_END' - { - "model": "claude-haiku-4-5-20251001", - "max_tokens": 256, - "messages": [ - { - "role": "user", - "content": "PROMPT_PLACEHOLDER\n\nConference: CONF_PLACEHOLDER\nURL: URL_PLACEHOLDER\n\nDIFF:\n```\nDIFF_PLACEHOLDER\n```\n\nRespond with EXACTLY this format:\nDECISION: RELEVANT or NOISE\nNEW_YEAR: YES or NO\nDETECTED_YEAR: [year or none]\nREASON: [one sentence]\nCONFIDENCE: HIGH, MEDIUM, or LOW\nSUGGESTED_FILTERS: [CSS selectors like '.footer, #nav, .countdown' and/or text patterns like 'text:Copyright, text:Last updated' - or 'none' if no filters needed. Do NOT use natural language descriptions.]" - } - ] - } - PAYLOAD_END - - # Use Python to safely substitute values (prevents injection) - python3 << 'PYSUBST' + # The payload is read from the event file, never interpolated into + # the script: titles and diffs are text from conference websites. + python3 << 'BUILD' import json import os + import sys - with open('/tmp/request.json') as f: - data = json.load(f) - - prompt = os.environ.get('PROMPT', '') - conf_name = os.environ.get('CONF_NAME', '') - conf_url = os.environ.get('CONF_URL', '') - diff = os.environ.get('DIFF_CONTENT', '')[:8000] # Limit diff size - - content = data['messages'][0]['content'] - content = content.replace('PROMPT_PLACEHOLDER', prompt) - content = content.replace('CONF_PLACEHOLDER', conf_name) - content = content.replace('URL_PLACEHOLDER', conf_url) - content = content.replace('DIFF_PLACEHOLDER', diff) - data['messages'][0]['content'] = content - - with open('/tmp/request.json', 'w') as f: - json.dump(data, f) - PYSUBST - - # Make the API call with retry for transient errors - echo "Calling Claude API for triage..." - for attempt in 1 2 3; do - HTTP_CODE=$(curl -s -w "%{http_code}" -o /tmp/response.json \ - -X POST "https://api.anthropic.com/v1/messages" \ - -H "x-api-key: $ANTHROPIC_API_KEY" \ - -H "content-type: application/json" \ - -H "anthropic-version: 2023-06-01" \ - -d @/tmp/request.json) - - # Success - if [ "$HTTP_CODE" = "200" ]; then - break - fi - - # Retry on rate limit (429) or overloaded (529) - if [ "$HTTP_CODE" = "429" ] || [ "$HTTP_CODE" = "529" ]; then - echo "::warning title=API Rate Limited::HTTP $HTTP_CODE, attempt $attempt/3" - sleep $((attempt * 5)) - continue - fi - - # Other errors - don't retry - break - done - - if [ "$HTTP_CODE" != "200" ]; then - echo "::warning title=API Error::HTTP $HTTP_CODE - falling back to full analysis" - echo "success=false" >> $GITHUB_OUTPUT - cat /tmp/response.json >&2 - exit 0 - fi - - # Extract the response text - RESPONSE=$(jq -r '.content[0].text // ""' /tmp/response.json) - - if [ -z "$RESPONSE" ]; then - echo "::warning title=Empty Response::No content in API response" - echo "success=false" >> $GITHUB_OUTPUT - exit 0 - fi - - echo "success=true" >> $GITHUB_OUTPUT - - # Save response for parsing - DELIM="RESP_EOF_$(date +%s%N)_$$" - echo "response<<$DELIM" >> $GITHUB_OUTPUT - echo "$RESPONSE" >> $GITHUB_OUTPUT - echo "$DELIM" >> $GITHUB_OUTPUT - - # Log token usage (use awk instead of bc for portability) - INPUT_TOKENS=$(jq -r '.usage.input_tokens // 0' /tmp/response.json) - OUTPUT_TOKENS=$(jq -r '.usage.output_tokens // 0' /tmp/response.json) - echo "::group::API Usage" - echo "Input tokens: $INPUT_TOKENS" - echo "Output tokens: $OUTPUT_TOKENS" - COST=$(awk "BEGIN {printf \"%.6f\", $INPUT_TOKENS * 0.00000025 + $OUTPUT_TOKENS * 0.00000125}") - echo "Estimated cost: \$$COST" - echo "::endgroup::" - - # Cleanup temp files (security best practice) - rm -f /tmp/request.json - - - name: Parse triage result - id: parse - env: - SKIP_CHECK: ${{ steps.skip_check.outputs.skip }} - API_SUCCESS: ${{ steps.triage_api.outputs.success }} - TRIAGE_RESPONSE: ${{ steps.triage_api.outputs.response }} - TRIGGER_REASON: ${{ needs.zone-check.outputs.trigger_reason }} - ORIGINAL_CFP_DEADLINE: ${{ needs.zone-check.outputs.original_cfp_deadline }} - CONF_NAME: ${{ needs.zone-check.outputs.conference }} - run: | - # Default to analyzing if triage was skipped - if [ "$SKIP_CHECK" = "true" ]; then - echo "should_analyze=true" >> $GITHUB_OUTPUT - echo "reason=Triage skipped - manual trigger or no diff" >> $GITHUB_OUTPUT - echo "suggested_filters=" >> $GITHUB_OUTPUT - echo "is_new_year=false" >> $GITHUB_OUTPUT - echo "detected_year=" >> $GITHUB_OUTPUT - exit 0 - fi - - # Default to analyzing if API call failed - if [ "$API_SUCCESS" != "true" ]; then - echo "should_analyze=true" >> $GITHUB_OUTPUT - echo "reason=Triage API failed - analyzing as fallback" >> $GITHUB_OUTPUT - echo "suggested_filters=" >> $GITHUB_OUTPUT - echo "is_new_year=false" >> $GITHUB_OUTPUT - echo "detected_year=" >> $GITHUB_OUTPUT - exit 0 - fi - - echo "::group::Triage Response" - echo "$TRIAGE_RESPONSE" - echo "::endgroup::" - - # Parse DECISION (case-insensitive, whitespace-tolerant) - if echo "$TRIAGE_RESPONSE" | grep -iqE "DECISION:\s*RELEVANT"; then - echo "should_analyze=true" >> $GITHUB_OUTPUT - elif echo "$TRIAGE_RESPONSE" | grep -iqE "DECISION:\s*NOISE"; then - # Check confidence - if LOW, analyze anyway - if echo "$TRIAGE_RESPONSE" | grep -iqE "CONFIDENCE:\s*LOW"; then - echo "should_analyze=true" >> $GITHUB_OUTPUT - echo "::notice title=Low Confidence::Analyzing despite NOISE decision" - else - echo "should_analyze=false" >> $GITHUB_OUTPUT - # Log "no extension found" specifically for extension checks - if [ "$TRIGGER_REASON" = "extension_check" ]; then - echo "::notice title=No Extension Found::${CONF_NAME} - CFP deadline ${ORIGINAL_CFP_DEADLINE} was NOT extended" - echo "## ✓ No Extension Found" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Conference:** ${CONF_NAME}" >> $GITHUB_STEP_SUMMARY - echo "**Original CFP deadline:** ${ORIGINAL_CFP_DEADLINE}" >> $GITHUB_STEP_SUMMARY - echo "**Result:** No CFP extension announced - no PR needed" >> $GITHUB_STEP_SUMMARY - fi - fi - else - echo "should_analyze=true" >> $GITHUB_OUTPUT - echo "::warning title=Parse Error::Could not parse DECISION, defaulting to analyze" - fi - - # Parse NEW_YEAR - if echo "$TRIAGE_RESPONSE" | grep -iqE "NEW_YEAR:\s*YES"; then - echo "is_new_year=true" >> $GITHUB_OUTPUT - else - echo "is_new_year=false" >> $GITHUB_OUTPUT - fi - - # Parse DETECTED_YEAR - DETECTED_YEAR=$(echo "$TRIAGE_RESPONSE" | grep -iE "DETECTED_YEAR:" | sed 's/.*DETECTED_YEAR:\s*//i' | head -1 | tr -d ' ') - if [ "$DETECTED_YEAR" = "none" ] || [ -z "$DETECTED_YEAR" ]; then - echo "detected_year=" >> $GITHUB_OUTPUT - else - echo "detected_year=$DETECTED_YEAR" >> $GITHUB_OUTPUT - fi - - # Extract reason - REASON=$(echo "$TRIAGE_RESPONSE" | grep -iE "REASON:" | sed 's/.*REASON:\s*//i' | head -1) - echo "reason=${REASON:-Triage completed}" >> $GITHUB_OUTPUT - - # Extract suggested filters - FILTERS=$(echo "$TRIAGE_RESPONSE" | grep -iE "SUGGESTED_FILTERS:" | sed 's/.*SUGGESTED_FILTERS:\s*//i' | head -1) - echo "suggested_filters=${FILTERS:-}" >> $GITHUB_OUTPUT - - # Apply filters directly to changedetection.io - apply-filters: - needs: [zone-check, triage] - if: | - always() && - needs.triage.outputs.should_analyze == 'false' && - needs.triage.outputs.suggested_filters != 'none' && - needs.triage.outputs.suggested_filters != '' - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - issues: write - - steps: - - name: Apply filters via API - id: apply_api - if: vars.CHANGEDETECTION_URL != '' && needs.zone-check.outputs.watch_uuid != '' - env: - WATCH_UUID: ${{ needs.zone-check.outputs.watch_uuid }} - FILTERS: ${{ needs.triage.outputs.suggested_filters }} - CHANGEDETECTION_URL: ${{ vars.CHANGEDETECTION_URL }} - CHANGEDETECTION_KEY: ${{ secrets.CHANGEDETECTION_KEY }} - CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }} - CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }} - run: | - if [ -z "$FILTERS" ] || [ "$FILTERS" = "none" ]; then - echo "No filters to apply" - echo "success=skipped" >> $GITHUB_OUTPUT - exit 0 - fi - - # Validate filter format: must contain CSS selectors (starting with . # or tag names) - # or text: patterns. Reject natural language descriptions. - # Valid examples: ".footer, #nav, text:Copyright" - # Invalid: "Filter out pages discussing..." - if ! echo "$FILTERS" | grep -qE '^\s*([.#]|text:|[a-z]+\[|[a-z]+\s*,|[a-z]+\s*$)' ; then - # Check if it looks like natural language (contains common words) - if echo "$FILTERS" | grep -qiE '\b(filter|out|pages|discussing|announcements|containing|patterns|deadline)\b'; then - echo "::warning::Filters appear to be natural language, not CSS selectors. Skipping." - echo "success=skipped" >> $GITHUB_OUTPUT - exit 0 - fi - fi - - # Parse CSS selectors and text patterns - CSS_FILTERS=$(echo "$FILTERS" | tr ',' '\n' | grep -v 'text:' | sed 's/^ *//' | tr '\n' ',' | sed 's/,$//') - TEXT_PATTERNS=$(echo "$FILTERS" | tr ',' '\n' | grep 'text:' | sed 's/text://' | sed 's/^ *//' | jq -R -s -c 'split("\n") | map(select(length > 0))') - - echo "::group::Filter Details" - echo "CSS filters: $CSS_FILTERS" - echo "Text patterns: $TEXT_PATTERNS" - echo "::endgroup::" - - # Fetch current config - CURRENT=$(curl -sf \ - -H "x-api-key: $CHANGEDETECTION_KEY" \ - -H "CF-Access-Client-Id: $CF_ACCESS_CLIENT_ID" \ - -H "CF-Access-Client-Secret: $CF_ACCESS_CLIENT_SECRET" \ - "$CHANGEDETECTION_URL/api/v1/watch/$WATCH_UUID") || { - echo "::warning::Failed to fetch watch config" - echo "success=false" >> $GITHUB_OUTPUT - exit 0 - } - - # Merge filters - EXISTING_CSS=$(echo "$CURRENT" | jq -r '.subtractive_selectors // ""') - if [ -n "$EXISTING_CSS" ] && [ "$EXISTING_CSS" != "null" ] && [ -n "$CSS_FILTERS" ]; then - MERGED_CSS="$EXISTING_CSS, $CSS_FILTERS" - elif [ -n "$CSS_FILTERS" ]; then - MERGED_CSS="$CSS_FILTERS" - else - MERGED_CSS="$EXISTING_CSS" - fi - - EXISTING_TEXT=$(echo "$CURRENT" | jq -c '.ignore_text // []' 2>/dev/null) || EXISTING_TEXT="[]" - # Ensure EXISTING_TEXT is valid JSON array - if ! echo "$EXISTING_TEXT" | jq -e 'type == "array"' > /dev/null 2>&1; then - EXISTING_TEXT="[]" - fi - - # Ensure TEXT_PATTERNS is valid JSON array - if [ -z "$TEXT_PATTERNS" ] || ! echo "$TEXT_PATTERNS" | jq -e 'type == "array"' > /dev/null 2>&1; then - TEXT_PATTERNS="[]" - fi - - if [ "$TEXT_PATTERNS" != "[]" ]; then - MERGED_TEXT=$(echo "$EXISTING_TEXT $TEXT_PATTERNS" | jq -s 'add | unique' 2>/dev/null) || MERGED_TEXT="$EXISTING_TEXT" - else - MERGED_TEXT="$EXISTING_TEXT" - fi - - # Final validation: ensure MERGED_TEXT is valid JSON array - if ! echo "$MERGED_TEXT" | jq -e 'type == "array"' > /dev/null 2>&1; then - MERGED_TEXT="[]" - fi - - # Apply update - UPDATE_PAYLOAD=$(jq -n --arg css "$MERGED_CSS" --argjson text "$MERGED_TEXT" '{subtractive_selectors: $css, ignore_text: $text}') - - curl -sf -X PUT \ - -H "x-api-key: $CHANGEDETECTION_KEY" \ - -H "CF-Access-Client-Id: $CF_ACCESS_CLIENT_ID" \ - -H "CF-Access-Client-Secret: $CF_ACCESS_CLIENT_SECRET" \ - -H "Content-Type: application/json" \ - -d "$UPDATE_PAYLOAD" \ - "$CHANGEDETECTION_URL/api/v1/watch/$WATCH_UUID" && { - echo "success=true" >> $GITHUB_OUTPUT - echo "::notice::Filters applied successfully" - } || { - echo "success=false" >> $GITHUB_OUTPUT - echo "::warning::Failed to apply filters" + sys.path.insert(0, "utils") + from desk_queue import clean_watch_title + + event_name = os.environ["GITHUB_EVENT_NAME"] + with open(os.environ["GITHUB_EVENT_PATH"], encoding="utf-8") as f: + event = json.load(f) + + if event_name == "repository_dispatch": + payload = event.get("client_payload") or {} + url, title = payload.get("url"), payload.get("title") + diff = payload.get("diff") + watch_uuid = payload.get("watch_uuid") or None + trigger = "website_change" + else: + inputs = event.get("inputs") or {} + url, title = inputs.get("url"), inputs.get("conference_name") + diff = inputs.get("note") or "" + watch_uuid = None + trigger = "manual" + + missing = [ + field for field, value in (("url", url), ("title", title)) + if not isinstance(value, str) or not value.strip() + ] + if missing: + for field in missing: + print(f"::error title=Missing Field::{field} is required") + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: + summary.write("## Payload Validation Failed\n\n") + summary.write(f"Missing required fields: {', '.join(missing)}\n\n") + summary.write("This usually means the changedetection.io token expired or the payload is malformed.\n") + sys.exit(1) + if not watch_uuid and event_name == "repository_dispatch": + print("::warning title=Missing Field::client_payload.watch_uuid is missing") + + if diff is None: + diff = "" + elif not isinstance(diff, str): + diff = json.dumps(diff, ensure_ascii=False) + + name, year = clean_watch_title(title) + name = " ".join(name.split()) # one line, for the step output + record = { + "source": "changedetection", + "kind": "signal", + "title": title.strip(), + "url": url.strip(), + "year": year, + "payload": {"diff": diff, "watch_uuid": watch_uuid, "trigger": trigger}, } - - - name: Log unapplied filter suggestion - if: steps.apply_api.outputs.success == 'false' || vars.CHANGEDETECTION_URL == '' - env: - CONF_NAME: ${{ needs.zone-check.outputs.conference }} - TRIAGE_FILTERS: ${{ needs.triage.outputs.suggested_filters }} - run: | - echo "::warning title=Filter Not Applied::Could not apply filters for ${CONF_NAME}: ${TRIAGE_FILTERS}" - - # Stage 2: Full analysis with Claude Code Action (needs file editing tools) - analyze-update: - needs: [zone-check, triage] - if: | - needs.zone-check.outputs.skip != 'true' && - needs.zone-check.outputs.has_pending != 'true' && - needs.triage.outputs.should_analyze == 'true' && - needs.triage.outputs.is_new_year != 'true' - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: write - pull-requests: write - id-token: write - - env: - UPDATE_BRANCH: auto/conference-updates - - steps: - - name: Checkout repository - uses: actions/checkout@v7 + with open(os.environ["RECORDS"], "w", encoding="utf-8") as f: + f.write(json.dumps(record, ensure_ascii=False) + "\n") + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: + out.write(f"series={name}\n") + print(f"Queueing a {trigger} signal for {name or title}") + BUILD + + - name: Write to the desk queue + id: queue + uses: ./.github/actions/desk-queue-write with: - fetch-depth: 0 - token: ${{ github.token }} - persist-credentials: true - - - name: Configure git identity - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - - name: Setup accumulator branch - run: | - git remote set-url origin "https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git" - - if git ls-remote --heads origin $UPDATE_BRANCH | grep -q $UPDATE_BRANCH; then - git fetch origin $UPDATE_BRANCH - git checkout $UPDATE_BRANCH - # Try rebase first, then merge, fail if neither works - if ! git rebase origin/main; then - echo "::warning title=Rebase Failed::Attempting merge instead" - git rebase --abort 2>/dev/null || true - if ! git merge origin/main --no-edit; then - git merge --abort 2>/dev/null || true - echo "::warning title=Merge Conflict::Auto-resolving conflicts in favor of main" - if ! git merge origin/main -X theirs --no-edit; then - git merge --abort 2>/dev/null || true - echo "::error title=Branch Sync Failed::Could not sync with main" - exit 1 - fi - fi - fi - else - git checkout -b $UPDATE_BRANCH - fi - - - name: Snapshot conferences.yml - run: cp _data/conferences.yml /tmp/conferences_before.yml - - - name: Prepare content - id: content - env: - DIFF_B64: ${{ needs.zone-check.outputs.diff_b64 }} - CONF_URL: ${{ needs.zone-check.outputs.url }} - HAS_DIFF: ${{ needs.zone-check.outputs.has_diff }} - run: | - # Bug #2 Fix: Write content to a temp file instead of using heredoc - # This prevents delimiter collision when scraped content contains special characters - # Claude Code Action can read the file using its Read tool - - CONTENT_FILE="${RUNNER_TEMP}/website_content.txt" - - # Decode base64 diff if available - if [ "$HAS_DIFF" = "true" ] && [ -n "$DIFF_B64" ]; then - DIFF_CONTENT=$(echo "$DIFF_B64" | base64 -d 2>/dev/null || echo "") - DIFF_LENGTH=${#DIFF_CONTENT} - else - DIFF_CONTENT="" - DIFF_LENGTH=0 - fi - - if [ "$DIFF_LENGTH" -gt 100 ]; then - echo "source=diff" >> $GITHUB_OUTPUT - printf '%s' "$DIFF_CONTENT" > "$CONTENT_FILE" - else - echo "source=fetched" >> $GITHUB_OUTPUT - # Use lynx for clean text extraction (much fewer tokens than raw HTML) - sudo apt-get install -qq -y lynx > /dev/null 2>&1 || true - lynx -dump -nolist -nonumbers -width=200 "$CONF_URL" 2>/dev/null | head -c 6000 > "$CONTENT_FILE" || echo "Failed to fetch content" > "$CONTENT_FILE" - fi - - # Store the file path as output (safe - no special characters) - echo "content_file=$CONTENT_FILE" >> $GITHUB_OUTPUT - echo "::notice title=Content::Stored $(wc -c < "$CONTENT_FILE") bytes in $CONTENT_FILE" - - # Workaround for anthropics/claude-code-action#900: the action's - # checkHumanActor calls the GitHub API before checking allowed_bots, - # which 404s for bot actors like github-actions[bot]. Override - # GITHUB_ACTOR so the action sees a real user. - - name: Fix actor for bot-triggered runs - if: contains(github.actor, '[bot]') - run: echo "GITHUB_ACTOR=${{ github.repository_owner }}" >> "$GITHUB_ENV" - - - name: Run Claude Code Analysis - uses: anthropics/claude-code-action@v1.0.241 - with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - allowed_bots: 'github-actions[bot]' - prompt: | - Update pythondeadlin.es conference data. DO NOT use Task tool or fetch URLs. - - Conference: ${{ needs.zone-check.outputs.conference }} - URL: ${{ needs.zone-check.outputs.url }} - Zone: ${{ needs.zone-check.outputs.zone }} - Reason: ${{ needs.triage.outputs.triage_reason }} - - FIRST: Use the Read tool to read the website content from: ${{ steps.content.outputs.content_file }} - This file contains either the diff of recent changes or the fetched website content. - - Task: Read _data/conferences.yml, find this conference, update if needed. - If zone is slow/extension_watch: ONLY update cfp_ext for extensions. - If zone is normal: update CFP dates, conference dates, URLs. - - Schema: conference (no year), year (int), link, cfp ('YYYY-MM-DD HH:mm:ss' or 'TBA'), cfp_ext (extended deadline), place ("City, Country"), start/end (YYYY-MM-DD), sub (PY|SCIPY|DATA|WEB|BIZ|GEO) - - CFP Extension: Keep original cfp, set cfp_ext to new deadline. - - CRITICAL: Only update dates if you find SPECIFIC dates (e.g., "September 15-17, 2026"). If dates are vague ("sometime in September", "Fall 2026", "TBD"), do NOT update the entry. Report "No changes - specific dates not found" and make no edits. Never fabricate or approximate dates. - claude_args: | - --model claude-haiku-4-5-20251001 - --max-turns 5 - --allowedTools Read,Edit,Grep,Glob - - - name: Check for changes - id: check_changes - run: | - if ! diff -q _data/conferences.yml /tmp/conferences_before.yml > /dev/null 2>&1; then - echo "changed=true" >> $GITHUB_OUTPUT - else - echo "changed=false" >> $GITHUB_OUTPUT - fi - - - name: Setup Pixi - if: steps.check_changes.outputs.changed == 'true' - uses: prefix-dev/setup-pixi@v0.10.2 - - - name: Sort conferences - if: steps.check_changes.outputs.changed == 'true' - run: pixi run sort - - - name: Commit and push - if: steps.check_changes.outputs.changed == 'true' - id: commit - env: - CONF_NAME: ${{ needs.zone-check.outputs.conference }} - CONF_URL: ${{ needs.zone-check.outputs.url }} - CONF_ZONE: ${{ needs.zone-check.outputs.zone }} - run: | - git remote set-url origin "https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git" - git add _data/conferences.yml _data/archive.yml _data/legacy.yml 2>/dev/null || git add _data/conferences.yml - git commit -m "conf: ${CONF_NAME}" -m "Source: ${CONF_URL}" -m "Zone: ${CONF_ZONE}" - git push origin $UPDATE_BRANCH --force-with-lease - echo "commit_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Create or update PR - if: steps.check_changes.outputs.changed == 'true' - uses: actions/github-script@v9 - env: - UPDATE_BRANCH: ${{ env.UPDATE_BRANCH }} - CONF_NAME: ${{ needs.zone-check.outputs.conference }} - COMMIT_SHA: ${{ steps.commit.outputs.commit_sha }} - with: - script: | - const branch = process.env.UPDATE_BRANCH; - const conference = process.env.CONF_NAME; - const sha = process.env.COMMIT_SHA.substring(0, 7); - const date = new Date().toISOString().split('T')[0]; - - const { data: prs } = await github.rest.pulls.list({ - owner: context.repo.owner, - repo: context.repo.repo, - head: `${context.repo.owner}:${branch}`, - state: 'open' - }); - - const entry = `| ${conference} | [${sha}](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/commit/${process.env.COMMIT_SHA}) | ${date} |`; - - if (prs.length > 0) { - const pr = prs[0]; - let body = pr.body || ''; - body = body.replace(/()/, `${entry}\n$1`); - await github.rest.pulls.update({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: pr.number, - body - }); - } else { - const { data: newPr } = await github.rest.pulls.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title: '🐍 Conference updates', - head: branch, - base: 'main', - body: `## 🐍 Automated Conference Updates\n\n| Conference | Commit | Date |\n|------------|--------|------|\n${entry}\n` - }); - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: newPr.number, - labels: ['automated', 'conference-update'] - }); - } + records: ${{ runner.temp }}/desk-records.jsonl + message: "Queue a changedetection signal for ${{ steps.record.outputs.series }}" - name: Summary - if: always() env: - CONF_NAME: ${{ needs.zone-check.outputs.conference }} - CHANGED: ${{ steps.check_changes.outputs.changed }} - run: | - echo "## Update: $CONF_NAME" >> $GITHUB_STEP_SUMMARY - echo "Status: $([[ '$CHANGED' == 'true' ]] && echo '✅ Updated' || echo '✓ No changes')" >> $GITHUB_STEP_SUMMARY - - # Stage 2b: Comprehensive new year scraper - new-year-scraper: - needs: [zone-check, triage] - # Disabled: in practice this path re-edited existing editions instead of - # adding new ones. New editions are handled by the cfp-scout routine. - if: | - false && - needs.zone-check.outputs.skip != 'true' && - needs.zone-check.outputs.has_pending != 'true' && - needs.triage.outputs.should_analyze == 'true' && - needs.triage.outputs.is_new_year == 'true' - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: write - pull-requests: write - id-token: write - - env: - UPDATE_BRANCH: auto/conference-updates - - steps: - - name: Checkout repository - uses: actions/checkout@v7 - with: - fetch-depth: 0 - token: ${{ github.token }} - persist-credentials: true - - - name: Configure git identity - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - - name: Setup accumulator branch - run: | - git remote set-url origin "https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git" - - if git ls-remote --heads origin $UPDATE_BRANCH | grep -q $UPDATE_BRANCH; then - git fetch origin $UPDATE_BRANCH - git checkout $UPDATE_BRANCH - # Try rebase first, then merge, fail if neither works - if ! git rebase origin/main; then - echo "::warning title=Rebase Failed::Attempting merge instead" - git rebase --abort 2>/dev/null || true - if ! git merge origin/main --no-edit; then - git merge --abort 2>/dev/null || true - echo "::warning title=Merge Conflict::Auto-resolving conflicts in favor of main" - if ! git merge origin/main -X theirs --no-edit; then - git merge --abort 2>/dev/null || true - echo "::error title=Branch Sync Failed::Could not sync with main" - exit 1 - fi - fi - fi - else - git checkout -b $UPDATE_BRANCH - fi - - - name: Snapshot and get previous entry - id: prep - env: - CONF_NAME: ${{ needs.zone-check.outputs.conference }} - run: | - cp _data/conferences.yml /tmp/conferences_before.yml - - # Bug #2 Fix: Write previous entry to a file instead of using heredoc - # This prevents delimiter collision when content contains special characters - - PREV_FILE="${RUNNER_TEMP}/prev_entry.txt" - - # Get previous entry for reference - python3 << 'GETPREV' > "$PREV_FILE" - import yaml, os, sys - conf_name = os.environ.get('CONF_NAME', '') - all_confs = [] - for f in ['_data/conferences.yml', '_data/archive.yml']: - try: - with open(f) as fh: all_confs.extend(yaml.safe_load(fh) or []) - except: pass - matching = [c for c in all_confs if conf_name.lower() in c.get('conference','').lower()] - if matching: - c = max(matching, key=lambda x: x.get('year', 0)) - for k in ['year','conference','sub','place','link','timezone','twitter','mastodon']: - if c.get(k): print(f"{k}: {c[k]}") - GETPREV - - # Store file path as output (safe - no special characters) - echo "prev_file=$PREV_FILE" >> $GITHUB_OUTPUT - echo "::notice title=Previous Entry::Found $(wc -l < "$PREV_FILE") lines of reference data" - - - name: Pre-fetch conference pages - id: fetch - env: - CONF_URL: ${{ needs.zone-check.outputs.url }} - run: | - # Bug #2 Fix: Write content to files instead of using heredoc - # This prevents delimiter collision when scraped content contains special characters - # Claude Code Action can read the files using its Read tool - - # Install lynx for clean text extraction - sudo apt-get install -qq -y lynx > /dev/null 2>&1 || true - - CONTENT_FILE="${RUNNER_TEMP}/website_content.txt" - - # Fetch main page as text (much fewer tokens than HTML) - lynx -dump -nolist -nonumbers -width=200 "$CONF_URL" 2>/dev/null > /tmp/main_text.txt || echo "Failed to fetch main page" > /tmp/main_text.txt - - # Also fetch raw HTML to extract subpage links - curl -sL --max-time 30 "$CONF_URL" > /tmp/main.html 2>/dev/null || true - - # Find CFP/finaid/sponsor links - grep -oiE 'href="[^"]*(/cfp|/submit|/call|/speak|/finaid|/financial|/grant|/sponsor|/partner)[^"]*"' /tmp/main.html | \ - sed 's/href="//;s/"$//' | head -3 > /tmp/subpage_links.txt 2>/dev/null || true - - # Fetch subpages with lynx - echo "" > /tmp/subpages_text.txt - while read -r link; do - [ -z "$link" ] && continue - # Handle relative URLs - if [[ "$link" == /* ]]; then - BASE=$(echo "$CONF_URL" | sed 's|^\(https\?://[^/]*\).*|\1|') - link="${BASE}${link}" - fi - echo "Fetching: $link" - lynx -dump -nolist -nonumbers -width=200 "$link" 2>/dev/null | head -c 4000 >> /tmp/subpages_text.txt || true - echo -e "\n---\n" >> /tmp/subpages_text.txt - done < /tmp/subpage_links.txt - - # Combine content into a single file (avoids heredoc issues) - { - head -c 12000 /tmp/main_text.txt - echo -e "\n\n--- SUBPAGES ---\n" - head -c 8000 /tmp/subpages_text.txt - } > "$CONTENT_FILE" - - # Store file path as output (safe - no special characters) - echo "content_file=$CONTENT_FILE" >> $GITHUB_OUTPUT - echo "::notice title=Content::Stored $(wc -c < "$CONTENT_FILE") bytes in $CONTENT_FILE" - - # Workaround for anthropics/claude-code-action#900: the action's - # checkHumanActor calls the GitHub API before checking allowed_bots, - # which 404s for bot actors like github-actions[bot]. Override - # GITHUB_ACTOR so the action sees a real user. - - name: Fix actor for bot-triggered runs - if: contains(github.actor, '[bot]') - run: echo "GITHUB_ACTOR=${{ github.repository_owner }}" >> "$GITHUB_ENV" - - - name: Run Claude Code Analysis - uses: anthropics/claude-code-action@v1.0.241 - with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - allowed_bots: 'github-actions[bot]' - prompt: | - Create/update entry for NEW YEAR of Python conference. DO NOT fetch URLs - content provided in files. - - Conference: ${{ needs.zone-check.outputs.conference }} - URL: ${{ needs.zone-check.outputs.url }} - Year: ${{ needs.triage.outputs.detected_year }} - - FIRST: Read the following files using the Read tool: - 1. Previous entry reference: ${{ steps.prep.outputs.prev_file }} - 2. Website content (main page + subpages): ${{ steps.fetch.outputs.content_file }} - - Required: conference (no year), year, link, cfp (YYYY-MM-DD HH:mm:ss or TBA), place (City, Country), start, end, sub - Optional: cfp_link, timezone, finaid, sponsor, twitter, mastodon, workshop_deadline, tutorial_deadline - - Read _data/conferences.yml, add/update entry for year ${{ needs.triage.outputs.detected_year }}. - Keep conference name consistent with previous years. Match YAML formatting. - - CRITICAL: Conference entries REQUIRE specific start/end dates. Only create an entry if you find EXACT dates (e.g., "September 15-17, 2026"). If dates are vague ("sometime in September", "Fall 2026", "TBD", "dates coming soon"), do NOT create the entry - the conference is not yet ready to be added. Report "No changes - specific dates not announced yet" and make no edits. Never fabricate or approximate dates. - claude_args: | - --model claude-haiku-4-5-20251001 - --max-turns 5 - --allowedTools Read,Edit,Grep,Glob - - - name: Check for changes - id: check_changes + SERIES: ${{ steps.record.outputs.series }} + ADDED: ${{ steps.queue.outputs.added }} run: | - if ! diff -q _data/conferences.yml /tmp/conferences_before.yml > /dev/null 2>&1; then - echo "changed=true" >> $GITHUB_OUTPUT + echo "## Desk queue" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "**Series:** $SERIES" >> "$GITHUB_STEP_SUMMARY" + if [ "$ADDED" = "0" ]; then + echo "Already queued, nothing added." >> "$GITHUB_STEP_SUMMARY" else - echo "changed=false" >> $GITHUB_OUTPUT + echo "Queued for the desk's next run." >> "$GITHUB_STEP_SUMMARY" fi - - - name: Setup Pixi - if: steps.check_changes.outputs.changed == 'true' - uses: prefix-dev/setup-pixi@v0.10.2 - - - name: Sort conferences - if: steps.check_changes.outputs.changed == 'true' - run: pixi run sort - - - name: Commit and push - if: steps.check_changes.outputs.changed == 'true' - id: commit - env: - CONF_NAME: ${{ needs.zone-check.outputs.conference }} - CONF_URL: ${{ needs.zone-check.outputs.url }} - DETECTED_YEAR: ${{ needs.triage.outputs.detected_year }} - run: | - git remote set-url origin "https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git" - git add _data/conferences.yml _data/archive.yml _data/legacy.yml 2>/dev/null || git add _data/conferences.yml - git commit -m "conf: ${CONF_NAME} ${DETECTED_YEAR} 🆕" -m "New year detected" -m "Source: ${CONF_URL}" - git push origin $UPDATE_BRANCH --force-with-lease - echo "commit_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Create or update PR - if: steps.check_changes.outputs.changed == 'true' - uses: actions/github-script@v9 - env: - UPDATE_BRANCH: ${{ env.UPDATE_BRANCH }} - CONF_NAME: ${{ needs.zone-check.outputs.conference }} - DETECTED_YEAR: ${{ needs.triage.outputs.detected_year }} - COMMIT_SHA: ${{ steps.commit.outputs.commit_sha }} - with: - script: | - const branch = process.env.UPDATE_BRANCH; - const conference = process.env.CONF_NAME; - const year = process.env.DETECTED_YEAR; - const sha = process.env.COMMIT_SHA.substring(0, 7); - const date = new Date().toISOString().split('T')[0]; - - const { data: prs } = await github.rest.pulls.list({ - owner: context.repo.owner, - repo: context.repo.repo, - head: `${context.repo.owner}:${branch}`, - state: 'open' - }); - - const entry = `| 🆕 ${conference} ${year} | [${sha}](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/commit/${process.env.COMMIT_SHA}) | ${date} |`; - - if (prs.length > 0) { - let body = prs[0].body || ''; - body = body.replace(/()/, `${entry}\n$1`); - await github.rest.pulls.update({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prs[0].number, - body - }); - } else { - const { data: newPr } = await github.rest.pulls.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title: '🐍 Conference updates', - head: branch, - base: 'main', - body: `## 🐍 Automated Conference Updates\n\n| Conference | Commit | Date |\n|------------|--------|------|\n${entry}\n` - }); - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: newPr.number, - labels: ['automated', 'conference-update', 'new-year'] - }); - } - - - name: Summary - if: always() - env: - CONF_NAME: ${{ needs.zone-check.outputs.conference }} - YEAR: ${{ needs.triage.outputs.detected_year }} - CHANGED: ${{ steps.check_changes.outputs.changed }} - run: | - echo "## 🆕 New Year: $CONF_NAME $YEAR" >> $GITHUB_STEP_SUMMARY - echo "Status: $([[ '$CHANGED' == 'true' ]] && echo '✅ Created' || echo '⚠️ No changes')" >> $GITHUB_STEP_SUMMARY - - # Log skipped conferences - log-skipped: - needs: zone-check - if: needs.zone-check.outputs.skip == 'true' - runs-on: ubuntu-latest - timeout-minutes: 2 - steps: - - name: Log skip - env: - CONF_NAME: ${{ needs.zone-check.outputs.conference }} - CONF_ZONE: ${{ needs.zone-check.outputs.zone }} - run: | - echo "::notice title=Skipped::$CONF_NAME in $CONF_ZONE zone" - echo "## ⏭️ Skipped: $CONF_NAME" >> $GITHUB_STEP_SUMMARY - echo "Zone: $CONF_ZONE (no useful updates expected)" >> $GITHUB_STEP_SUMMARY - - # Log conferences with pending updates - log-pending: - needs: zone-check - if: needs.zone-check.outputs.skip != 'true' && needs.zone-check.outputs.has_pending == 'true' - runs-on: ubuntu-latest - timeout-minutes: 2 - steps: - - name: Log pending - env: - CONF_NAME: ${{ needs.zone-check.outputs.conference }} - run: | - echo "::notice title=Pending::$CONF_NAME already has pending update in PR" - echo "## ⏸️ Skipped: $CONF_NAME" >> $GITHUB_STEP_SUMMARY - echo "Already has pending update in accumulator branch. Merge the PR to process new changes." >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/enrich-tba-conferences.yml b/.github/workflows/enrich-tba-conferences.yml index 27ceee7d5c..39f9e7e75f 100644 --- a/.github/workflows/enrich-tba-conferences.yml +++ b/.github/workflows/enrich-tba-conferences.yml @@ -4,18 +4,13 @@ on: # Manual trigger or called by canonical-source-update workflow_dispatch: inputs: - enrichment_level: - description: 'quick=deterministic links only (no API), full=links + dates via Claude' - type: choice - options: ['full', 'quick'] - default: 'full' dry_run: description: 'Dry run (no commit/PR)' required: false type: boolean default: false - # Daily run - quick mode, except Tuesday which does full (with Claude API) + # Daily run schedule: - cron: '0 6 * * *' # Daily 6 AM UTC @@ -37,29 +32,6 @@ jobs: timeout-minutes: 20 steps: - - name: Determine enrichment level - id: config - env: - EVENT_NAME: ${{ github.event_name }} - INPUT_LEVEL: ${{ inputs.enrichment_level }} - run: | - if [ "$EVENT_NAME" = "schedule" ]; then - DAY_OF_WEEK=$(date -u +%u) # 1=Monday, 2=Tuesday, ... - - if [ "$DAY_OF_WEEK" = "2" ]; then - # Tuesday = weekly full run with Claude API - echo "level=full" >> $GITHUB_OUTPUT - echo "::notice title=Tuesday Full Run::Using full enrichment mode (with Claude API)" - else - # Other days = quick deterministic run - echo "level=quick" >> $GITHUB_OUTPUT - echo "::notice title=Daily Quick Run::Using quick enrichment mode (deterministic only)" - fi - else - echo "level=${INPUT_LEVEL:-full}" >> $GITHUB_OUTPUT - echo "::notice title=Manual Run::Using ${INPUT_LEVEL:-full} enrichment mode" - fi - - name: Checkout repository uses: actions/checkout@v7 with: @@ -106,7 +78,7 @@ jobs: - name: Setup Pixi uses: prefix-dev/setup-pixi@v0.10.2 - - name: Install lynx for text extraction + - name: Install lynx for link extraction run: sudo apt-get install -qq -y lynx > /dev/null 2>&1 || true - name: Find TBA conferences @@ -140,14 +112,12 @@ jobs: id: enrich if: steps.find-tba.outputs.tba_count != '0' env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - ENRICHMENT_LEVEL: ${{ steps.config.outputs.level }} DRY_RUN: ${{ inputs.dry_run }} run: | echo "::group::Running enrichment" # Build command - CMD="pixi run python utils/enrich_tba.py --level $ENRICHMENT_LEVEL" + CMD="pixi run python utils/enrich_tba.py" if [ "$DRY_RUN" = "true" ]; then CMD="$CMD --dry-run" fi @@ -178,7 +148,7 @@ jobs: echo "changed=false" >> $GITHUB_OUTPUT echo "::notice title=No Changes::No updates from TBA enrichment" echo "## No Changes" >> $GITHUB_STEP_SUMMARY - echo "TBA enrichment did not find any new CFP data to update." >> $GITHUB_STEP_SUMMARY + echo "TBA enrichment did not find any new links to add." >> $GITHUB_STEP_SUMMARY fi - name: Validate and sort @@ -188,8 +158,6 @@ jobs: - name: Commit and push if: steps.check_changes.outputs.changed == 'true' && inputs.dry_run != true id: commit - env: - ENRICHMENT_LEVEL: ${{ steps.config.outputs.level }} run: | git remote set-url origin "https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git" @@ -197,9 +165,9 @@ jobs: git add _data/conferences.yml _data/archive.yml _data/legacy.yml 2>/dev/null || git add _data/conferences.yml # Create descriptive commit message - COMMIT_MSG="chore: enrich TBA conferences with CFP data" + COMMIT_MSG="chore: enrich TBA conferences with links" - git commit -m "$COMMIT_MSG" -m "Enrichment level: $ENRICHMENT_LEVEL" -m "Triggered: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + git commit -m "$COMMIT_MSG" -m "Triggered: $(date -u +%Y-%m-%dT%H:%M:%SZ)" git push origin $UPDATE_BRANCH --force-with-lease echo "commit_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT @@ -224,14 +192,12 @@ jobs: env: UPDATE_BRANCH: ${{ env.UPDATE_BRANCH }} COMMIT_SHA: ${{ steps.commit.outputs.commit_sha }} - ENRICHMENT_LEVEL: ${{ steps.config.outputs.level }} HAS_REVIEW: ${{ steps.review.outputs.has_review }} REVIEW_CONTENT: ${{ steps.review.outputs.review_content }} with: script: | const branch = process.env.UPDATE_BRANCH; const sha = process.env.COMMIT_SHA.substring(0, 7); - const level = process.env.ENRICHMENT_LEVEL; const date = new Date().toISOString().split('T')[0]; const hasReview = process.env.HAS_REVIEW === 'true'; const reviewContent = process.env.REVIEW_CONTENT || ''; @@ -245,7 +211,7 @@ jobs: }); // Use same table format as other workflows - const entry = `| TBA Enrichment (${level}) | [${sha}](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/commit/${process.env.COMMIT_SHA}) | ${date} |`; + const entry = `| TBA Enrichment | [${sha}](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/commit/${process.env.COMMIT_SHA}) | ${date} |`; if (prs.length > 0) { // Update existing PR - append to table @@ -317,7 +283,6 @@ jobs: TBA_COUNT: ${{ steps.find-tba.outputs.tba_count }} CHANGED: ${{ steps.check_changes.outputs.changed }} DRY_RUN: ${{ inputs.dry_run }} - LEVEL: ${{ steps.config.outputs.level }} run: | echo "" >> $GITHUB_STEP_SUMMARY echo "## Result" >> $GITHUB_STEP_SUMMARY @@ -331,8 +296,7 @@ jobs: echo "- Status: Changes committed and PR updated" >> $GITHUB_STEP_SUMMARY fi else - echo "- Status: No new CFP data found" >> $GITHUB_STEP_SUMMARY + echo "- Status: No new links found" >> $GITHUB_STEP_SUMMARY fi - echo "- Enrichment level: $LEVEL" >> $GITHUB_STEP_SUMMARY echo "- TBA conferences scanned: $TBA_COUNT" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/redirect-scan.yml b/.github/workflows/redirect-scan.yml new file mode 100644 index 0000000000..cd6dd811d8 --- /dev/null +++ b/.github/workflows/redirect-scan.yml @@ -0,0 +1,67 @@ +name: Redirect Scan + +# Weekly and report-only. Runs the link check over the data and queues every +# link that now redirects to a different site, for the desk to judge: a series +# that moved to a new domain gets its link updated, a lapsed or hijacked domain +# gets an archived copy. A 200 from another site is not a healthy link, and +# lychee in link-check.yml cannot tell the two apart. +# +# Nothing is committed here. The link check's other edits (archived copies of +# dead links, same-site redirects) stay with the data writers. As it always +# does, the check also asks the Wayback Machine to save the pages of upcoming +# conferences. + +on: + schedule: + - cron: '0 3 * * 0' # Sundays 03:00 UTC + workflow_dispatch: + +concurrency: + group: redirect-scan + cancel-in-progress: false + +permissions: + contents: write + +jobs: + scan: + runs-on: ubuntu-latest + timeout-minutes: 180 + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup Pixi + uses: prefix-dev/setup-pixi@v0.10.2 + + # Below the job timeout, so a slow scan still queues what it found. + - name: Check links + timeout-minutes: 150 + env: + DESK_QUEUE_OUT: ${{ runner.temp }}/desk-records.jsonl + run: pixi run links + + # Redirects found before a failure are still worth queueing. + - name: Write to the desk queue + id: queue + if: ${{ !cancelled() }} + uses: ./.github/actions/desk-queue-write + with: + records: ${{ runner.temp }}/desk-records.jsonl + message: Queue links that redirect to other sites + + - name: Summary + if: ${{ !cancelled() }} + env: + RECORDS: ${{ runner.temp }}/desk-records.jsonl + ADDED: ${{ steps.queue.outputs.added }} + run: | + FOUND=0 + if [ -s "$RECORDS" ]; then + FOUND=$(wc -l < "$RECORDS") + fi + echo "## Redirect scan" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "- Links redirecting to another site: $FOUND" >> "$GITHUB_STEP_SUMMARY" + echo "- New in the desk queue: ${ADDED:-0}" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/scheduled-extension-checker.yml b/.github/workflows/scheduled-extension-checker.yml deleted file mode 100644 index 7d1959af48..0000000000 --- a/.github/workflows/scheduled-extension-checker.yml +++ /dev/null @@ -1,242 +0,0 @@ -name: Scheduled Extension Checker - -on: - # Run daily at 6 AM UTC - schedule: - - cron: '0 6 * * *' - - # Manual trigger for testing - workflow_dispatch: - inputs: - days_ahead: - description: 'Check CFPs closing within N days' - required: false - type: number - default: 3 - dry_run: - description: 'Dry run (show what would be checked without triggering)' - required: false - type: boolean - default: false - -permissions: - contents: read - -jobs: - find-closing-cfps: - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - conferences: ${{ steps.find.outputs.conferences }} - count: ${{ steps.find.outputs.count }} - - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Find CFPs closing soon - id: find - env: - # Pass inputs via env to prevent injection - DAYS_AHEAD: ${{ inputs.days_ahead || 3 }} - run: | - TODAY=$(date +%Y-%m-%d) - - # Use env vars safely in Python - python3 << 'FINDER' - import yaml - import json - import os - import sys - from datetime import datetime, timedelta - - days_ahead = int(os.environ.get('DAYS_AHEAD', '3')) - today = datetime.now().date() - check_until = today + timedelta(days=days_ahead) - - try: - with open("_data/conferences.yml") as f: - confs = yaml.safe_load(f) or [] - except Exception as e: - print(f"ERROR: Could not load conferences.yml: {e}", file=sys.stderr) - confs = [] - - closing_soon = [] - - for conf in confs: - name = conf.get("conference", "Unknown") - - # Get the effective CFP deadline - cfp_str = conf.get("cfp_ext") or conf.get("cfp", "TBA") - has_extension = conf.get("cfp_ext") is not None - - if cfp_str in ("TBA", "Cancelled", "None"): - continue - - try: - cfp_date = datetime.strptime(cfp_str[:10], "%Y-%m-%d").date() - except ValueError: - continue - - # Include CFPs closing within window or just closed (extension check) - extension_window_start = today - timedelta(days=3) - - if extension_window_start <= cfp_date <= check_until: - url = str(conf.get("link", "")) - - if cfp_date < today: - check_type = "extension_check" - reason = f"CFP closed {cfp_date}, checking for extension" - elif cfp_date == today: - check_type = "closing_today" - reason = "CFP closes today!" - else: - days_left = (cfp_date - today).days - check_type = "closing_soon" - reason = f"CFP closes in {days_left} days ({cfp_date})" - - # Skip if already has extension and past deadline - if has_extension and cfp_date < today: - print(f"Skip: {name} - already extended, past deadline", file=sys.stderr) - continue - - cfp_link = str(conf.get("cfp_link", "")) - closing_soon.append({ - "conference": name, - "url": url, - "cfp_link": cfp_link, - "cfp_date": str(cfp_date), - "has_extension": has_extension, - "check_type": check_type, - "reason": reason - }) - print(f"✓ {name}: {reason}") - - print(f"\nFound {len(closing_soon)} conferences to check") - - with open("/tmp/closing_cfps.json", "w") as f: - json.dump(closing_soon, f) - FINDER - - # Read results safely - CONFERENCES=$(cat /tmp/closing_cfps.json) - COUNT=$(echo "$CONFERENCES" | jq 'length') - - echo "conferences=$(echo "$CONFERENCES" | jq -c '.')" >> $GITHUB_OUTPUT - echo "count=$COUNT" >> $GITHUB_OUTPUT - - # Summary - echo "## CFPs Closing Soon" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Checking CFPs closing within **$DAYS_AHEAD days**" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - if [ "$COUNT" -gt 0 ]; then - echo "| Conference | CFP Date | Status |" >> $GITHUB_STEP_SUMMARY - echo "|------------|----------|--------|" >> $GITHUB_STEP_SUMMARY - echo "$CONFERENCES" | jq -r '.[] | "| \(.conference) | \(.cfp_date) | \(.reason) |"' >> $GITHUB_STEP_SUMMARY - else - echo "No CFPs closing soon ✅" >> $GITHUB_STEP_SUMMARY - fi - - # Trigger checks for each conference - trigger-checks: - needs: find-closing-cfps - if: needs.find-closing-cfps.outputs.count != '0' && inputs.dry_run != true - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: write - strategy: - matrix: - conference: ${{ fromJson(needs.find-closing-cfps.outputs.conferences) }} - max-parallel: 1 # Conservative to avoid rate limits - fail-fast: false - - steps: - - name: Install lynx - run: sudo apt-get update && sudo apt-get install -y lynx - - - name: Fetch website content and trigger check - env: - # Pass matrix values via env for safety - CONF_NAME: ${{ matrix.conference.conference }} - CONF_URL: ${{ matrix.conference.url }} - CONF_CFP_LINK: ${{ matrix.conference.cfp_link }} - CONF_REASON: ${{ matrix.conference.reason }} - CONF_CFP_DATE: ${{ matrix.conference.cfp_date }} - CONF_CHECK_TYPE: ${{ matrix.conference.check_type }} - CONF_HAS_EXT: ${{ matrix.conference.has_extension }} - GH_TOKEN: ${{ github.token }} - run: | - echo "Triggering check for: $CONF_NAME" - echo "Main URL: $CONF_URL" - echo "CFP Link: $CONF_CFP_LINK" - echo "Reason: $CONF_REASON" - echo "CFP Date: $CONF_CFP_DATE" - echo "Check Type: $CONF_CHECK_TYPE" - - # Fetch main website content - echo "Fetching main website content..." - MAIN_CONTENT="" - if [ -n "$CONF_URL" ]; then - MAIN_CONTENT=$(lynx -dump -nolist "$CONF_URL" 2>/dev/null | head -500 || echo "Failed to fetch main website") - fi - - # Fetch CFP link content - echo "Fetching CFP link content..." - CFP_CONTENT="" - if [ -n "$CONF_CFP_LINK" ] && [ "$CONF_CFP_LINK" != "$CONF_URL" ]; then - CFP_CONTENT=$(lynx -dump -nolist "$CONF_CFP_LINK" 2>/dev/null | head -500 || echo "Failed to fetch CFP page") - fi - - # Combine content for analysis (use printf to avoid YAML parsing issues) - COMBINED_CONTENT=$(printf '%s\n\n%s\n%s\n\n%s\n%s' \ - "=== EXTENSION CHECK: $CONF_REASON ===" \ - "=== MAIN WEBSITE ($CONF_URL) ===" \ - "$MAIN_CONTENT" \ - "=== CFP PAGE ($CONF_CFP_LINK) ===" \ - "$CFP_CONTENT") - - # Truncate if too long (GitHub has payload limits) - COMBINED_CONTENT=$(echo "$COMBINED_CONTENT" | head -c 60000) - - # Use gh CLI for safe API call - # Pass trigger context so triage workflow can adjust prompt accordingly - gh api repos/${{ github.repository }}/dispatches \ - -f event_type=conference-change \ - -f "client_payload[url]=$CONF_URL" \ - -f "client_payload[title]=$CONF_NAME" \ - -f "client_payload[watch_uuid]=" \ - -f "client_payload[diff]=$COMBINED_CONTENT" \ - -f "client_payload[source]=scheduled-checker" \ - -f "client_payload[trigger_reason]=$CONF_CHECK_TYPE" \ - -f "client_payload[original_cfp_deadline]=$CONF_CFP_DATE" \ - -f "client_payload[has_extension]=$CONF_HAS_EXT" - - echo "✓ Triggered with website content" - - - name: Rate limit pause - run: sleep 10 # Generous pause between triggers - - # Summary - summary: - needs: [find-closing-cfps, trigger-checks] - if: always() - runs-on: ubuntu-latest - timeout-minutes: 2 - steps: - - name: Final summary - env: - COUNT: ${{ needs.find-closing-cfps.outputs.count }} - DRY_RUN: ${{ inputs.dry_run }} - run: | - echo "## Extension Checker Summary" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - if [ "$COUNT" = "0" ]; then - echo "✅ No CFPs closing soon" >> $GITHUB_STEP_SUMMARY - elif [ "$DRY_RUN" = "true" ]; then - echo "🔍 **Dry run** - found $COUNT conferences (not triggered)" >> $GITHUB_STEP_SUMMARY - else - echo "🚀 Triggered checks for **$COUNT** conferences" >> $GITHUB_STEP_SUMMARY - fi diff --git a/ruff.toml b/ruff.toml index f96ca32da2..e40bdda7c7 100644 --- a/ruff.toml +++ b/ruff.toml @@ -57,7 +57,7 @@ ignore = [ ] [lint.per-file-ignores] -# S603: subprocess with validated URL input for lynx text extraction +# S603: subprocess with validated URL input for lynx link extraction "utils/enrich_tba.py" = ["S603"] [lint.pydocstyle] diff --git a/tests/test_desk_queue.py b/tests/test_desk_queue.py new file mode 100644 index 0000000000..ec5c3c343a --- /dev/null +++ b/tests/test_desk_queue.py @@ -0,0 +1,403 @@ +"""Tests for desk_queue: the deadline desk's queue on the desk-queue branch.""" + +import json +import sys +from datetime import date +from datetime import datetime +from datetime import timezone +from pathlib import Path + +import pytest + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +from desk_queue import MAX_HANDLED_PER_FILE +from desk_queue import MAX_ITEMS_PER_FILE +from desk_queue import MAX_STRING_LENGTH +from desk_queue import README +from desk_queue import add_items +from desk_queue import init_queue +from desk_queue import item_id +from desk_queue import list_items +from desk_queue import main +from desk_queue import normalise_record +from desk_queue import record_names +from desk_queue import remove_items +from desk_queue import slugify +from desk_worklist import build_worklist +from desk_worklist import load_queue + +NOW = datetime(2026, 10, 14, 6, 0, tzinfo=timezone.utc) + + +def signal(title="PyCon Greece 2027 - Main", diff="CfP closes 1 March", **fields): + record = { + "source": "changedetection", + "kind": "signal", + "title": title, + "url": "https://pycon.gr/", + "year": 2027, + "payload": {"diff": diff, "watch_uuid": "uuid-1", "trigger": "website_change"}, + } + record.update(fields) + return record + + +def read(path): + return json.loads(Path(path).read_text(encoding="utf-8")) + + +class TestSlugify: + @pytest.mark.parametrize( + ("name", "expected"), + [ + ("PyCon Greece", "pycon-greece"), + ("PyDay Hurlingham", "pyday-hurlingham"), + ("PyCon Česko", "pycon-cesko"), + ("Python Pizza Istanbul!", "python-pizza-istanbul"), + ("FOSDEM: Python Devroom", "fosdem-python-devroom"), + ("--", "unknown"), + ("", "unknown"), + ], + ) + def test_slug(self, name, expected): + assert slugify(name) == expected + + +class TestRecordNames: + def test_watch_title_is_cleaned(self): + assert record_names(signal(title="[Conference] PyCon Greece 2027 - CFP")) == ["PyCon Greece"] + + def test_payload_conference_comes_before_title(self): + record = { + "kind": "redirect", + "title": "https://old.example/", + "payload": {"conference": "PyTennessee", "year": 2019}, + } + assert record_names(record) == ["PyTennessee", "https://old.example/"] + + def test_missing_edition_names_the_matched_series_first(self): + record = { + "kind": "candidate", + "title": "PyCon SK", + "payload": { + "classification": "MISSING_EDITION", + "matched": {"conference": "PyCon Slovakia", "file": "archive", "year": 2026, "score": 90}, + "fields": {"conference": "PyCon SK"}, + }, + } + assert record_names(record) == ["PyCon Slovakia", "PyCon SK"] + + def test_registry_names(self): + record = {"kind": "registry", "title": "registry", "payload": {"names": ["PyCon Nepal", "PyCon NP"]}} + assert record_names(record) == ["PyCon Nepal", "PyCon NP", "registry"] + + +class TestNormaliseRecord: + def test_sets_id_and_received(self): + item = normalise_record(signal(), NOW) + assert item["received"] == "2026-10-14T06:00:00Z" + assert item["id"] == item_id("changedetection", "signal", "https://pycon.gr/", item["payload"]) + assert sorted(item) == ["id", "kind", "payload", "received", "source", "title", "url", "year"] + + def test_id_ignores_received_and_a_supplied_id(self): + first = normalise_record(signal(received="2026-10-01T00:00:00Z", id="mine")) + second = normalise_record(signal(received="2026-10-14T00:00:00Z")) + assert first["id"] == second["id"] != "mine" + + def test_id_depends_on_the_payload(self): + assert normalise_record(signal(diff="a"), NOW)["id"] != normalise_record(signal(diff="b"), NOW)["id"] + + def test_redirect_id_ignores_status_and_query(self): + def redirect(final_url, status): + payload = {"conference": "PyTennessee", "year": 2019, "field": "sponsor"} + payload.update(final_url=final_url, status=status) + record = { + "source": "link-check", + "kind": "redirect", + "title": "PyTennessee", + "url": "https://pytennessee.org/p", + } + return normalise_record({**record, "payload": payload}, NOW) + + first = redirect("https://casino.example/landing?session=1#top", 200) + second = redirect("https://casino.example/landing?session=2", 403) + assert first["id"] == second["id"] + assert first["payload"]["final_url"] == "https://casino.example/landing?session=1#top" + assert redirect("https://other.example/landing", 200)["id"] != first["id"] + + def test_long_strings_are_cut_and_flagged(self): + item = normalise_record(signal(diff="x" * (MAX_STRING_LENGTH + 5)), NOW) + assert len(item["payload"]["diff"]) == MAX_STRING_LENGTH + assert item["payload"]["truncated"] is True + + def test_nested_long_strings_are_cut(self): + record = signal() + record["payload"] = {"diffs": {"cfp": {"data": "y" * (MAX_STRING_LENGTH + 1), "source": "z"}}} + item = normalise_record(record, NOW) + assert len(item["payload"]["diffs"]["cfp"]["data"]) == MAX_STRING_LENGTH + assert item["payload"]["truncated"] is True + + def test_short_payload_is_not_flagged(self): + assert "truncated" not in normalise_record(signal(), NOW)["payload"] + + @pytest.mark.parametrize( + ("change", "message"), + [ + ({"source": "somewhere"}, "unknown source"), + ({"kind": "rumour"}, "unknown kind"), + ({"title": ""}, "title"), + ({"title": None}, "title"), + ({"url": None}, "url"), + ({"year": "2027"}, "year"), + ({"year": True}, "year"), + ({"payload": "diff"}, "payload"), + ({"received": "yesterday"}, "received"), + ], + ) + def test_rejects_bad_fields(self, change, message): + with pytest.raises(ValueError, match=message): + normalise_record(signal(**change), NOW) + + def test_rejects_non_objects(self): + with pytest.raises(ValueError, match="JSON object"): + normalise_record(["not", "a", "record"], NOW) + + +class TestAddItems: + def test_creates_one_file_per_series(self, tmp_path): + results = add_items(tmp_path, [signal(), signal(title="PyCon Greece 2027 - CFP", diff="other")], NOW) + assert [result["status"] for result in results] == ["added", "added"] # same series, two findings + document = read(tmp_path / "queue" / "pycon-greece.json") + assert document["series"] == "PyCon Greece" + assert len(document["items"]) == 2 + + def test_existing_id_is_a_no_op(self, tmp_path): + add_items(tmp_path, [signal()], NOW) + before = (tmp_path / "queue" / "pycon-greece.json").read_text(encoding="utf-8") + results = add_items(tmp_path, [signal(received="2026-10-15T06:00:00Z")], NOW) + assert results[0]["status"] == "exists" + assert (tmp_path / "queue" / "pycon-greece.json").read_text(encoding="utf-8") == before + + def test_file_layout(self, tmp_path): + add_items(tmp_path, [signal(diff="Ελληνικά")], NOW) + text = (tmp_path / "queue" / "pycon-greece.json").read_text(encoding="utf-8") + assert text.endswith("}\n") + assert "Ελληνικά" in text + assert text == json.dumps(json.loads(text), indent=2, sort_keys=True, ensure_ascii=False) + "\n" + + def test_keeps_the_newest_items(self, tmp_path): + records = [ + signal(diff=f"change {n}", received=f"2026-10-{1 + n // 24:02d}T{n % 24:02d}:00:00Z") + for n in range(MAX_ITEMS_PER_FILE + 2) + ] + results = add_items(tmp_path, records, NOW) + assert [result["status"] for result in results[:2]] == ["dropped", "dropped"] + items = read(tmp_path / "queue" / "pycon-greece.json")["items"] + assert len(items) == MAX_ITEMS_PER_FILE + assert {item["payload"]["diff"] for item in items} == {f"change {n}" for n in range(2, MAX_ITEMS_PER_FILE + 2)} + + def test_bad_record_is_reported_and_others_kept(self, tmp_path): + results = add_items(tmp_path, [signal(kind="rumour"), signal()], NOW) + assert "error" in results[0] + assert results[1]["status"] == "added" + + def test_same_finding_under_two_titles_is_one_item(self, tmp_path): + """A watch on link and one on cfp_link with the same URL send the same diff.""" + results = add_items(tmp_path, [signal(), signal(title="PyCon Greece 2027 - CFP")], NOW) + assert [result["status"] for result in results] == ["added", "exists"] + + def test_unnamed_series_uses_the_title(self, tmp_path): + record = signal(title="[Conference] Some Brand New Conf 2027 - Main") + add_items(tmp_path, [record], NOW) + assert read(tmp_path / "queue" / "some-brand-new-conf.json")["series"] == "Some Brand New Conf" + + def test_hand_typed_year_is_not_part_of_the_series(self, tmp_path): + """The manual check-conference-update run takes a name like "PyCon US 2027".""" + record = signal(title="PyCon US 2027", year=None, url="https://us.pycon.org/2027/") + add_items(tmp_path, [record], NOW) + assert read(tmp_path / "queue" / "pycon-us.json")["series"] == "PyCon US" + + +class TestRemoveItems: + def test_removes_and_remembers(self, tmp_path): + italia = signal(title="PyCon Italia 2027 - Main", url="https://pycon.it/") + first, second = add_items(tmp_path, [signal(), italia], NOW) + removed = remove_items(tmp_path, [first["id"], "missing"]) + assert removed == {first["id"]} + document = read(tmp_path / "queue" / "pycon-greece.json") + assert document["items"] == [] + assert document["handled"] == [first["id"]] + assert read(tmp_path / "queue" / "pycon-italia.json")["handled"] == [] + assert [row["id"] for row in list_items(tmp_path)] == [second["id"]] + + def test_handled_finding_is_not_queued_again(self, tmp_path): + (first,) = add_items(tmp_path, [signal()], NOW) + remove_items(tmp_path, [first["id"]]) + (again,) = add_items(tmp_path, [signal(received="2026-10-21T06:00:00Z")], NOW) + assert again == {"file": "queue/pycon-greece.json", "id": first["id"], "status": "handled"} + assert read(tmp_path / "queue" / "pycon-greece.json")["items"] == [] + + def test_changed_finding_is_queued(self, tmp_path): + (first,) = add_items(tmp_path, [signal()], NOW) + remove_items(tmp_path, [first["id"]]) + (changed,) = add_items(tmp_path, [signal(diff="CfP extended to 15 March")], NOW) + assert changed["status"] == "added" + + def test_keeps_the_latest_handled_ids(self, tmp_path): + ids = [] + for batch in range(0, MAX_HANDLED_PER_FILE + 3, 40): + records = [signal(diff=f"change {n}") for n in range(batch, min(batch + 40, MAX_HANDLED_PER_FILE + 3))] + added = [result["id"] for result in add_items(tmp_path, records, NOW)] + remove_items(tmp_path, added) + ids.extend(added) + handled = read(tmp_path / "queue" / "pycon-greece.json")["handled"] + assert len(ids) == MAX_HANDLED_PER_FILE + 3 + assert handled == ids[-MAX_HANDLED_PER_FILE:] + + def test_keeps_other_items_in_the_file(self, tmp_path): + first, second = add_items(tmp_path, [signal(), signal(diff="other")], NOW) + remove_items(tmp_path, [first["id"]]) + assert [item["id"] for item in read(tmp_path / "queue" / "pycon-greece.json")["items"]] == [second["id"]] + + def test_no_queue_directory(self, tmp_path): + assert remove_items(tmp_path, ["x"]) == set() + assert list_items(tmp_path) == [] + + def test_broken_files_are_skipped(self, tmp_path, capsys): + (first,) = add_items(tmp_path, [signal(title="PyCon Italia 2027 - Main", url="https://pycon.it/")], NOW) + (tmp_path / "queue" / "pycon-greece.json").write_text('{"items": [},', encoding="utf-8") + assert [row["id"] for row in list_items(tmp_path)] == [first["id"]] + assert remove_items(tmp_path, [first["id"]]) == {first["id"]} + assert "skipping queue/pycon-greece.json" in capsys.readouterr().err + + +class TestBrokenFile: + def test_moved_aside_and_writers_carry_on(self, tmp_path): + add_items(tmp_path, [signal()], NOW) + broken = tmp_path / "queue" / "pycon-greece.json" + broken.write_text('{"items": [},', encoding="utf-8") + italia = signal(title="PyCon Italia 2027 - Main", url="https://pycon.it/") + results = add_items(tmp_path, [signal(diff="new"), italia], NOW) + assert "moved to queue/pycon-greece.json.bad" in results[0]["error"] + assert [result.get("status") for result in results[1:]] == ["added", "added"] + assert (tmp_path / "queue" / "pycon-greece.json.bad").read_text(encoding="utf-8") == '{"items": [},' + assert [item["payload"]["diff"] for item in read(broken)["items"]] == ["new"] + + def test_file_without_items_list(self, tmp_path): + (tmp_path / "queue").mkdir() + (tmp_path / "queue" / "pycon-greece.json").write_text('{"series": "PyCon Greece"}', encoding="utf-8") + results = add_items(tmp_path, [signal()], NOW) + assert "no items list" in results[0]["error"] + assert results[1]["status"] == "added" + + +class TestInit: + def test_writes_layout_and_refreshes_readme(self, tmp_path): + (tmp_path / "README.md").write_text("old", encoding="utf-8") + add_items(tmp_path, [signal()], NOW) + init_queue(tmp_path) + assert (tmp_path / "README.md").read_text(encoding="utf-8") == README + assert (tmp_path / "queue" / ".gitkeep").exists() + assert (tmp_path / "queue" / "pycon-greece.json").exists() + + +class TestWorklistRoundTrip: + def test_items_load_and_route(self, tmp_path): + add_items(tmp_path, [signal(title="[Conference] PyCon Greece 2027 - Main")], NOW) + queue_items = load_queue(tmp_path) + assert queue_items[0]["file"] == "queue/pycon-greece.json" + assert queue_items[0]["series"] == "PyCon Greece" + data = { + "conferences": [], + "archive": [ + { + "conference": "PyCon Greece", + "year": 2026, + "link": "https://2026.pycon.gr/", + "cfp": "2026-03-01 23:59:00", + "place": "Athens, Greece", + "start": date(2026, 6, 1), + "end": date(2026, 6, 2), + "sub": "PY", + }, + ], + "legacy": [], + } + worklist = build_worklist(data, queue_items, date(2026, 10, 14), scout=False) + assert [(item["skill"], item["conference"]) for item in worklist["items"]] == [ + ("new-year-checker", "PyCon Greece"), + ] + assert worklist["items"][0]["queue_refs"] == [{"file": "queue/pycon-greece.json", "id": queue_items[0]["id"]}] + + def test_redirect_record_from_the_link_checker(self, tmp_path, monkeypatch): + from tidy_conf import links + + out = tmp_path / "records.jsonl" + monkeypatch.setenv("DESK_QUEUE_OUT", str(out)) + monkeypatch.setattr(links, "REDIRECT_RECORDS", []) + links.record_redirect( + "https://pytennessee.org/prospectus", + "https://casino.example/", + 200, + conference="PyTennessee", + year=2019, + field="sponsor", + ) + assert main(["add", str(tmp_path / "queue-checkout"), "--jsonl", str(out)]) == 0 + items = load_queue(tmp_path / "queue-checkout") + assert [(item["file"], item["kind"], item["payload"]["final_url"]) for item in items] == [ + ("queue/pytennessee.json", "redirect", "https://casino.example/"), + ] + + +class TestCli: + def test_add_jsonl_skips_bad_lines(self, tmp_path, capsys): + records = tmp_path / "records.jsonl" + records.write_text(json.dumps(signal()) + "\n\nnot json\n" + json.dumps(signal(kind="rumour")) + "\n") + assert main(["add", str(tmp_path / "q"), "--jsonl", str(records)]) == 0 + captured = capsys.readouterr() + assert captured.out.startswith("added queue/pycon-greece.json ") + assert captured.err.count("desk_queue: record rejected: ") == 2 + assert (tmp_path / "q" / "README.md").exists() + + def test_add_from_options(self, tmp_path, capsys): + payload = tmp_path / "payload.json" + payload.write_text(json.dumps({"diff": "new dates", "watch_uuid": None, "trigger": "website_change"})) + argv = [ + "add", + str(tmp_path / "q"), + "--source", + "changedetection", + "--kind", + "signal", + "--title", + "PyCon Italia 2027 - Main", + "--url", + "https://pycon.it/", + "--payload", + str(payload), + ] + assert main(argv) == 0 + item = read(tmp_path / "q" / "queue" / "pycon-italia.json")["items"][0] + assert item["payload"]["diff"] == "new dates" + assert item["year"] is None + + def test_add_needs_a_record(self, tmp_path): + with pytest.raises(SystemExit): + main(["add", str(tmp_path)]) + + def test_jsonl_excludes_options(self, tmp_path): + with pytest.raises(SystemExit): + main(["add", str(tmp_path), "--jsonl", "-", "--title", "x"]) + + def test_remove_and_list(self, tmp_path, capsys): + first, _ = add_items(tmp_path, [signal(), signal(diff="other")], NOW) + assert main(["remove", str(tmp_path), first["id"], "nope"]) == 0 + captured = capsys.readouterr() + assert "removed 1" in captured.out + assert "no item nope" in captured.err + assert main(["list", str(tmp_path)]) == 0 + rows = [json.loads(line) for line in capsys.readouterr().out.splitlines()] + assert len(rows) == 1 + assert rows[0]["file"] == "queue/pycon-greece.json" diff --git a/tests/test_desk_worklist.py b/tests/test_desk_worklist.py index cb4aa4d881..b1c4af3c30 100644 --- a/tests/test_desk_worklist.py +++ b/tests/test_desk_worklist.py @@ -119,7 +119,10 @@ class TestCleanWatchTitle: ("[Conference] PyCon Greece - Main", ("PyCon Greece", None)), ("PyCon Greece 2027 - Main", ("PyCon Greece", 2027)), ("PyCon Greece", ("PyCon Greece", None)), - ("PyCon Greece 2027", ("PyCon Greece 2027", None)), + ("PyCon Greece 2027", ("PyCon Greece", 2027)), + ("PyCon US 2027 ", ("PyCon US", 2027)), + ("Python 3000", ("Python 3000", None)), + ("PyCon 2027 Online", ("PyCon 2027 Online", None)), ("[Conference] Python - Main Track 2027 - Main", ("Python - Main Track", 2027)), ], ) diff --git a/tests/test_enrich_tba.py b/tests/test_enrich_tba.py new file mode 100644 index 0000000000..950db74140 --- /dev/null +++ b/tests/test_enrich_tba.py @@ -0,0 +1,34 @@ +"""Tests for which conferences the TBA enrichment visits.""" + +import sys +from pathlib import Path + +import pytest +import yaml + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +from enrich_tba import find_tba_conferences + + +def write_conferences(path, cfps): + entries = [ + {"conference": f"Conf {index}", "year": 2027, "cfp": cfp, "link": "https://example.org/"} + for index, cfp in enumerate(cfps) + ] + path.write_text(yaml.safe_dump(entries), encoding="utf-8") + return path + + +@pytest.mark.parametrize("cfp", ["TBA", "tbd", "None", None, "n/a"]) +def test_conferences_without_a_dated_cfp_are_visited(tmp_path, cfp): + path = write_conferences(tmp_path / "conferences.yml", [cfp]) + + assert [conf["conference"] for conf in find_tba_conferences(path)] == ["Conf 0"] + + +@pytest.mark.parametrize("cfp", ["Cancelled", "cancelled", "2027-03-01 23:59:00"]) +def test_cancelled_and_dated_conferences_are_left_out(tmp_path, cfp): + path = write_conferences(tmp_path / "conferences.yml", [cfp]) + + assert find_tba_conferences(path) == [] diff --git a/tests/test_merge_questions.py b/tests/test_merge_questions.py new file mode 100644 index 0000000000..4c714f6ed0 --- /dev/null +++ b/tests/test_merge_questions.py @@ -0,0 +1,168 @@ +"""Tests for merge questions handed to the desk queue when no one can answer them. + +In CI the merge has no terminal, so query_yes_no keeps the default answer. When +the caller says what the question is about, the question is also recorded and, +with DESK_QUEUE_OUT set, written as a JSON line for the desk queue. +""" + +import io +import json +import sys +from pathlib import Path + +import pandas as pd +import pytest + +sys.path.insert(0, str(Path(__file__).parent)) +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +from desk_queue import normalise_record +from tidy_conf import utils as tidy_utils +from tidy_conf.interactive_merge import fuzzy_match +from tidy_conf.interactive_merge import merge_conferences +from tidy_conf.utils import query_yes_no + + +@pytest.fixture() +def no_terminal(monkeypatch, tmp_path): + """Run without a terminal, with a fresh record list and a queue file.""" + monkeypatch.setattr(tidy_utils.sys, "stdin", io.StringIO()) + monkeypatch.setattr(tidy_utils, "QUESTION_RECORDS", []) + out = tmp_path / "questions.jsonl" + monkeypatch.setenv("DESK_QUEUE_OUT", str(out)) + return out + + +def read_lines(path): + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + + +def frame(name, place, link="https://example-conf.org/"): + return pd.DataFrame( + { + "conference": [name], + "year": [2027], + "cfp": ["2027-01-06 23:59:00"], + "link": [link], + "place": [place], + "start": ["2027-06-01"], + "end": ["2027-06-03"], + }, + ) + + +SCHEMA = ["conference", "year", "cfp", "link", "place", "start", "end", "sub"] + + +class TestQueryYesNo: + def test_records_the_question_with_its_default(self, no_terminal): + context = { + "conference": "Example Conf", + "year": 2027, + "field": "place", + "yaml_value": "Vienna, Austria", + "remote_value": "Graz, Austria", + "source_name": "python.org", + "url": "https://example-conf.org/", + } + assert query_yes_no("Prefer Graz?", context=context) is False + (record,) = read_lines(no_terminal) + assert record == tidy_utils.QUESTION_RECORDS[0] + assert record["source"] == "canonical-import" + assert record["kind"] == "question" + assert record["title"] == "Example Conf" + assert record["year"] == 2027 + assert record["url"] == "https://example-conf.org/" + assert record["payload"] == { + "question": "Prefer Graz?", + "conference": "Example Conf", + "year": 2027, + "field": "place", + "yaml_value": "Vienna, Austria", + "remote_value": "Graz, Austria", + "default_answer": False, + "source_name": "python.org", + } + assert normalise_record(record)["id"] + + def test_yes_default_is_recorded(self, no_terminal): + assert query_yes_no("Go on?", default="yes", context={"conference": "Example Conf"}) is True + assert read_lines(no_terminal)[0]["payload"]["default_answer"] is True + + def test_without_context_nothing_is_recorded(self, no_terminal): + assert query_yes_no("Prefer Graz?") is False + assert read_lines(no_terminal) == [] + assert tidy_utils.QUESTION_RECORDS == [] + + def test_empty_values_become_null(self, no_terminal): + query_yes_no("Fill it?", context={"conference": "Example Conf", "yaml_value": float("nan")}) + payload = read_lines(no_terminal)[0]["payload"] + assert payload["yaml_value"] is None + assert payload["remote_value"] is None + + def test_without_queue_file_the_record_is_kept_in_memory(self, no_terminal, monkeypatch): + monkeypatch.delenv("DESK_QUEUE_OUT") + query_yes_no("Prefer Graz?", context={"conference": "Example Conf"}) + assert not no_terminal.exists() + assert len(tidy_utils.QUESTION_RECORDS) == 1 + + +class TestMergeQuestions: + def test_conflicting_place_is_queued_and_yaml_kept(self, no_terminal, mock_title_mappings): + df_yml = frame("Example Summit", "Vienna, Austria") + df_remote = frame("Example Summit", "Graz, Austria") + df_matched, df_remote_processed, _ = fuzzy_match(df_yml, df_remote, source_name="python.org") + with pytest.MonkeyPatch.context() as patcher: + patcher.setattr("tidy_conf.interactive_merge.get_schema", lambda: pd.DataFrame(columns=SCHEMA)) + result = merge_conferences(df_matched, df_remote_processed, source_name="python.org") + + assert result["place"].iloc[0] == "Vienna, Austria" + questions = [record for record in read_lines(no_terminal) if record["payload"]["field"] == "place"] + assert len(questions) == 1 + payload = questions[0]["payload"] + assert payload["conference"] == "Example Summit" + assert payload["year"] == 2027 + assert payload["yaml_value"] == "Vienna, Austria" + assert payload["remote_value"] == "Graz, Austria" + assert payload["source_name"] == "python.org" + assert questions[0]["url"] == "https://example-conf.org/" + + def test_conflicting_cfp_is_queued_once(self, no_terminal, mock_title_mappings): + """The follow-up extension question adds nothing the desk needs, so only the cfp one is queued.""" + df_yml = frame("Example Summit", "Vienna, Austria") + df_remote = frame("Example Summit", "Vienna, Austria") + df_remote["cfp"] = ["2027-01-20 23:59:00"] + df_matched, df_remote_processed, _ = fuzzy_match(df_yml, df_remote, source_name="python.org") + with pytest.MonkeyPatch.context() as patcher: + patcher.setattr("tidy_conf.interactive_merge.get_schema", lambda: pd.DataFrame(columns=SCHEMA)) + result = merge_conferences(df_matched, df_remote_processed, source_name="python.org") + + assert result["cfp"].iloc[0] == "2027-01-06 23:59:00" + (record,) = read_lines(no_terminal) + assert record["payload"]["field"] == "cfp" + assert record["payload"]["yaml_value"] == "2027-01-06 23:59:00" + assert record["payload"]["remote_value"] == "2027-01-20 23:59:00" + + def test_fuzzy_match_question_is_queued(self, no_terminal, mock_title_mappings): + df_yml = frame("Example Python Summit", "Vienna, Austria") + df_remote = frame("Example Python Sumit", "Vienna, Austria") + fuzzy_match(df_yml, df_remote, source_name="python-organizers") + + (record,) = read_lines(no_terminal) + assert record["payload"]["field"] == "conference" + assert record["payload"]["yaml_value"] == "Example Python Summit" + assert record["payload"]["remote_value"] == "Example Python Sumit" + assert record["payload"]["source_name"] == "python-organizers" + assert record["year"] == 2027 + + def test_agreeing_sources_ask_nothing(self, no_terminal, mock_title_mappings): + df_matched, df_remote_processed, _ = fuzzy_match( + frame("Example Summit", "Vienna, Austria"), + frame("Example Summit", "Vienna, Austria"), + ) + with pytest.MonkeyPatch.context() as patcher: + patcher.setattr("tidy_conf.interactive_merge.get_schema", lambda: pd.DataFrame(columns=SCHEMA)) + merge_conferences(df_matched, df_remote_processed) + assert read_lines(no_terminal) == [] diff --git a/utils/desk_queue.py b/utils/desk_queue.py new file mode 100644 index 0000000000..1e3b90735b --- /dev/null +++ b/utils/desk_queue.py @@ -0,0 +1,631 @@ +#!/usr/bin/env python3 +r"""Read and write the deadline desk's queue on the desk-queue branch. + +GitHub Actions jobs record what they find for the desk here: changedetection +signals, the canonical import's open questions, links that now redirect to +another site, and importer candidates. The desk reads the queue into its +worklist (desk_worklist.py) and removes the items it has handled. The branch +is an orphan and is never merged. + +A desk-queue checkout holds: + +- README.md: the format, written by ``init`` from README below; +- queue/.json: the items for one series, keyed by the slug of its name; +- ledger/pending.md: ledger text waiting for a rolling PR, written by the desk. + +This module uses only the standard library, so any job can run it without +installing the data tooling. + +Usage:: + + python utils/desk_queue.py init QUEUE + python utils/desk_queue.py add QUEUE --jsonl records.jsonl + python utils/desk_queue.py add QUEUE --source changedetection --kind signal \ + --title "PyCon Greece 2027 - Main" --url https://pycon.gr/ --payload payload.json + python utils/desk_queue.py remove QUEUE ID [ID ...] + python utils/desk_queue.py list QUEUE +""" + +import argparse +import hashlib +import json +import re +import sys +import unicodedata +from collections.abc import Iterable +from datetime import datetime +from datetime import timezone +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit +from urllib.parse import urlunsplit + +SOURCES = ( + "changedetection", + "canonical-import", + "link-check", + "pretalx", + "callingallpapers", + "developers-events", + "psf-ics-drift", + "registry", +) +KINDS = ("signal", "question", "redirect", "candidate", "drift", "registry") +MAX_ITEMS_PER_FILE = 50 +MAX_HANDLED_PER_FILE = 500 +MAX_STRING_LENGTH = 20_000 +RECEIVED_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +RECEIVED = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") +LIST_FIELDS = ("id", "source", "kind", "received", "title") + +WATCH_PREFIX = re.compile(r"^\s*\[conference\]\s*", re.IGNORECASE) +WATCH_SUFFIX = re.compile(r"\s+(?:(?P\d{4})\s+)?-\s+(?:main|cfp)\s*$", re.IGNORECASE) +# A trailing year with no watch suffix, as in a hand-typed "PyCon US 2027" +TRAILING_YEAR = re.compile(r"\s+(?P(?:19|20)\d{2})\s*$") + +README = """\ +# desk-queue + +Work waiting for the deadline desk, the daily run described in +`.claude/routines/desk.md` on the main branch. GitHub Actions jobs add items +with `utils/desk_queue.py`; the desk removes them once handled. This branch is +never merged. + +## Layout + +- `queue/.json`: the items for one series. The key is the slug of the + series name: lowercase, accents stripped, every run of other characters + turned into `-`. +- `ledger/pending.md`: ledger text from a desk run that had no rolling PR to + write to. The next run moves it into the PR. + +## Queue files + +UTF-8 JSON, two-space indent, keys sorted: + +```json +{ + "handled": [""], + "items": [ + { + "id": "", + "kind": "signal", + "payload": {"diff": "...", "trigger": "website_change", "watch_uuid": "..."}, + "received": "2026-10-14T06:00:00Z", + "source": "changedetection", + "title": "PyCon Greece 2027 - Main", + "url": "https://pycon.gr/", + "year": 2027 + } + ], + "series": "PyCon Greece" +} +``` + +`year` is the edition year when known, else `null`. Adding an item whose id +is already in the file, as an item or as handled, changes nothing: the same +finding is queued once, and a finding the desk has handled stays handled +until something about it changes. For a redirect the id leaves out the status +and the final URL's query and fragment. A file keeps at most 50 items, the +newest by `received`, and the 500 latest handled ids. A payload string longer +than 20,000 characters is cut there, and the payload gets `"truncated": true`. +A file that cannot be read is moved to `.json.bad` when a writer next +adds to it. + +## Sources and kinds + +| source | kind | payload | +| --- | --- | --- | +| `changedetection` | `signal` | `diff`, `watch_uuid`, `trigger` | +| `canonical-import` | `question` | `question`, `conference`, `year`, `field`, `yaml_value`, \ +`remote_value`, `default_answer`, `source_name` | +| `link-check` | `redirect` | `conference`, `year`, `field`, `final_url`, `status` | +| importers | `candidate` | `classification`, `fields`, `source_id`, `evidence_url`, `matched`, `flags` | +| `psf-ics-drift` | `drift` | `conference`, `year`, `diffs` | +| `registry` | `registry` | `names`, `change`, `evidence_url` | + +Every item is data for the desk to check on the live sources. Nothing in a +queue item reaches the conference data without that check. +""" + + +def clean_watch_title(title: str) -> tuple[str, int | None]: + """Strip the decoration a changedetection watch title carries. + + Parameters + ---------- + title : str + Raw title, e.g. "[Conference] PyCon Greece 2027 - Main". + + Returns + ------- + tuple[str, int | None] + The bare name ("PyCon Greece") and the year from the suffix, or from a + trailing year when there is no suffix ("PyCon Greece 2027"), if any. + """ + name = WATCH_PREFIX.sub("", title) + year = None + match = WATCH_SUFFIX.search(name) or TRAILING_YEAR.search(name) + if match: + name = name[: match.start()] + if match["year"]: + year = int(match["year"]) + return name.strip(), year + + +def slugify(name: str) -> str: + """Return the queue key for a series name. + + Parameters + ---------- + name : str + Series name, e.g. "PyCon Česko". + + Returns + ------- + str + Lowercase ASCII with every other run of characters turned into "-" + ("pycon-cesko"), or "unknown" when nothing is left. + """ + decomposed = unicodedata.normalize("NFKD", name) + ascii_name = "".join(char for char in decomposed if not unicodedata.combining(char)) + slug = re.sub(r"[^a-z0-9]+", "-", ascii_name.lower()).strip("-") + return slug or "unknown" + + +def record_names(record: dict) -> list[str]: + """Return the series names a queue record can be matched by, most specific first. + + Parameters + ---------- + record : dict + A queue item, or a record not yet given its id. + + Returns + ------- + list[str] + Distinct names with watch-title decoration removed: for a + MISSING_EDITION candidate the matched series first, then the + payload's conference, the candidate's fields, registry names, and the + title. + """ + payload = record.get("payload") if isinstance(record.get("payload"), dict) else {} + raw = [] + matched = payload.get("matched") + if record.get("kind") == "candidate" and payload.get("classification") == "MISSING_EDITION": + raw.append(matched.get("conference") if isinstance(matched, dict) else None) + raw.append(payload.get("conference")) + fields = payload.get("fields") + if isinstance(fields, dict): + raw.append(fields.get("conference")) + if isinstance(payload.get("names"), list): + raw.extend(payload["names"]) + raw.append(record.get("title")) + names = [] + for value in raw: + if not isinstance(value, str): + continue + name = clean_watch_title(value)[0] + if name and name not in names: + names.append(name) + return names + + +def _cap_strings(value: Any) -> tuple[Any, bool]: + """Return a copy of a payload value with long strings cut, and whether any was cut.""" + if isinstance(value, str): + if len(value) > MAX_STRING_LENGTH: + return value[:MAX_STRING_LENGTH], True + return value, False + if isinstance(value, dict): + capped, cut = {}, False + for key, inner in value.items(): + capped[key], inner_cut = _cap_strings(inner) + cut = cut or inner_cut + return capped, cut + if isinstance(value, list): + capped_list, cut = [], False + for inner in value: + capped_inner, inner_cut = _cap_strings(inner) + capped_list.append(capped_inner) + cut = cut or inner_cut + return capped_list, cut + return value, False + + +def item_id(source: str, kind: str, url: str, payload: dict) -> str: + """Return the stable id of a queue item. + + Parameters + ---------- + source, kind, url : str + The item's fields of the same names. + payload : dict + The item's payload, after capping. + + Returns + ------- + str + SHA-1 hex of the four, so the same finding always gets the same id. + For a redirect, the status and the final URL's query and fragment are + left out: they can change from one check to the next while the + finding, a link that lands on another site, stays the same. + """ + if kind == "redirect": + payload = {key: value for key, value in payload.items() if key != "status"} + if isinstance(payload.get("final_url"), str): + parts = urlsplit(payload["final_url"]) + payload["final_url"] = urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + payload_text = json.dumps(payload, sort_keys=True, ensure_ascii=False) + digest = hashlib.sha1(payload_text.encode("utf-8"), usedforsecurity=False).hexdigest() + return hashlib.sha1(f"{source}|{kind}|{url}|{digest}".encode(), usedforsecurity=False).hexdigest() + + +def normalise_record(record: Any, now: datetime | None = None) -> dict: + """Check a record and turn it into a queue item. + + Parameters + ---------- + record : Any + A dict with "source", "kind", "title", "url" and "payload", and + optionally "year" and "received". An "id" it carries is replaced. + now : datetime | None + Time to use when the record has no "received" (default: now, UTC). + + Returns + ------- + dict + The queue item, with its id set and long payload strings cut. + + Raises + ------ + ValueError + When a field is missing or of the wrong type. + """ + if not isinstance(record, dict): + raise ValueError("a record must be a JSON object") + source, kind = record.get("source"), record.get("kind") + if source not in SOURCES: + raise ValueError(f"unknown source {source!r}") + if kind not in KINDS: + raise ValueError(f"unknown kind {kind!r}") + title, url = record.get("title"), record.get("url") + if not isinstance(title, str) or not title.strip(): + raise ValueError("title must be a non-empty string") + if not isinstance(url, str): + raise ValueError("url must be a string") + year = record.get("year") + if year is not None and (isinstance(year, bool) or not isinstance(year, int)): + raise ValueError("year must be an integer or null") + payload = record.get("payload") + if not isinstance(payload, dict): + raise ValueError("payload must be a JSON object") + received = record.get("received") + if received is None: + received = (now or datetime.now(tz=timezone.utc)).strftime(RECEIVED_FORMAT) + elif not isinstance(received, str) or not RECEIVED.match(received): + raise ValueError("received must look like 2026-10-14T06:00:00Z") + + payload, cut = _cap_strings(payload) + if cut: + payload["truncated"] = True + title = title[:MAX_STRING_LENGTH] + url = url[:MAX_STRING_LENGTH] + return { + "id": item_id(source, kind, url, payload), + "kind": kind, + "payload": payload, + "received": received, + "source": source, + "title": title, + "url": url, + "year": year, + } + + +def queue_root(queue_dir: Path) -> Path: + """Return the queue/ directory of a desk-queue checkout.""" + return Path(queue_dir) / "queue" + + +def _read_file(path: Path) -> dict: + """Read a queue file. + + Raises + ------ + ValueError + When the file is not valid JSON or has no items list. + """ + document = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(document, dict) or not isinstance(document.get("items"), list): + raise ValueError("no items list") + if not isinstance(document.get("handled", []), list): + raise ValueError("handled is not a list") + return document + + +def _open_file(path: Path, series: str) -> tuple[dict, str | None]: + """Read a queue file for adding, or start one for the series. + + A file that cannot be read is moved to .bad and a new one started, + so one broken file never stops the writers; the text is kept for a person. + + Parameters + ---------- + path : Path + The queue file. + series : str + Series name for a new file. + + Returns + ------- + tuple[dict, str | None] + The document, and a message when the file had to be moved aside. + """ + if not path.exists(): + return {"handled": [], "items": [], "series": series}, None + try: + document = _read_file(path) + except ValueError as error: + bad = path.with_name(path.name + ".bad") + path.replace(bad) + message = f"queue/{path.name} unreadable ({error}), moved to queue/{bad.name}" + return {"handled": [], "items": [], "series": series}, message + document.setdefault("handled", []) + return document, None + + +def _write_file(path: Path, document: dict) -> None: + """Write a queue file in the canonical layout.""" + path.parent.mkdir(parents=True, exist_ok=True) + text = json.dumps(document, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + path.write_text(text, encoding="utf-8", newline="\n") + + +def init_queue(queue_dir: Path) -> None: + """Create the layout of a desk-queue checkout, refreshing its README. + + Parameters + ---------- + queue_dir : Path + The checkout. Existing queue files are left alone. + """ + root = queue_root(queue_dir) + root.mkdir(parents=True, exist_ok=True) + (root / ".gitkeep").touch() + (Path(queue_dir) / "README.md").write_text(README, encoding="utf-8", newline="\n") + + +def add_items(queue_dir: Path, records: Iterable[Any], now: datetime | None = None) -> list[dict]: + """Add records to the queue. + + Parameters + ---------- + queue_dir : Path + The desk-queue checkout. + records : Iterable[Any] + Records as normalise_record takes them. + now : datetime | None + Time for records without "received" (default: now, UTC). + + Returns + ------- + list[dict] + One result per record, in order: {"file", "id", "status"} with status + "added", "exists", "handled" (the desk has handled this finding), or + "dropped" (older than the 50 items its file keeps), or {"error"} for a + record that failed its checks. A file moved aside adds an {"error"}. + """ + results = [] + documents: dict[Path, dict] = {} + for record in records: + try: + item = normalise_record(record, now) + except ValueError as error: + results.append({"error": f"record rejected: {error}"}) + continue + names = record_names(item) + series = names[0] if names else item["title"] + path = queue_root(queue_dir) / f"{slugify(series)}.json" + if path not in documents: + documents[path], moved = _open_file(path, series) + if moved: + results.append({"error": moved}) + document = documents[path] + relative = f"queue/{path.name}" + if item["id"] in document["handled"]: + results.append({"file": relative, "id": item["id"], "status": "handled"}) + continue + if any(isinstance(existing, dict) and existing.get("id") == item["id"] for existing in document["items"]): + results.append({"file": relative, "id": item["id"], "status": "exists"}) + continue + document["items"].append(item) + results.append({"file": relative, "id": item["id"], "status": "added"}) + + for path, document in documents.items(): + document["handled"] = document["handled"][-MAX_HANDLED_PER_FILE:] + items = document["items"] + if len(items) > MAX_ITEMS_PER_FILE: + newest = sorted(items, key=lambda entry: str(entry.get("received", "")), reverse=True) + keep = {id(entry) for entry in newest[:MAX_ITEMS_PER_FILE]} + document["items"] = [entry for entry in items if id(entry) in keep] + _write_file(path, document) + + kept_ids = { + (f"queue/{path.name}", item.get("id")) for path, document in documents.items() for item in document["items"] + } + for result in results: + if result.get("status") == "added" and (result["file"], result["id"]) not in kept_ids: + result["status"] = "dropped" + return results + + +def _read_or_report(path: Path) -> dict | None: + """Read a queue file, or report on stderr why it cannot be read.""" + try: + return _read_file(path) + except ValueError as error: + print(f"desk_queue: skipping queue/{path.name}: {error}", file=sys.stderr) + return None + + +def _readable_files(queue_dir: Path) -> Iterable[tuple[Path, dict]]: + """Yield each readable queue file with its document, skipping broken ones.""" + for path in sorted(queue_root(queue_dir).glob("*.json")): + document = _read_or_report(path) + if document is not None: + yield path, document + + +def remove_items(queue_dir: Path, ids: Iterable[str]) -> set[str]: + """Remove handled items, remembering their ids. + + A removed id is kept in its file's "handled" list, so a writer that finds + the same thing again does not queue it again. The desk removes every item + it has decided; items it has not reached stay. + + Parameters + ---------- + queue_dir : Path + The desk-queue checkout. + ids : Iterable[str] + Item ids to remove. + + Returns + ------- + set[str] + The ids that were found and removed. + """ + wanted = set(ids) + removed: set[str] = set() + for path, document in _readable_files(queue_dir): + items = document["items"] + found = [item["id"] for item in items if isinstance(item, dict) and item.get("id") in wanted] + if not found: + continue + removed.update(found) + document["items"] = [item for item in items if not (isinstance(item, dict) and item.get("id") in wanted)] + handled = document.setdefault("handled", []) + handled.extend(item_id for item_id in found if item_id not in handled) + document["handled"] = handled[-MAX_HANDLED_PER_FILE:] + _write_file(path, document) + return removed + + +def list_items(queue_dir: Path) -> list[dict]: + """Return every queue item with the file it is in. + + Parameters + ---------- + queue_dir : Path + The desk-queue checkout. + + Returns + ------- + list[dict] + {"file", "id", "source", "kind", "received", "title"} per item, by file and received. + """ + rows = [] + for path, document in _readable_files(queue_dir): + rows.extend( + {"file": f"queue/{path.name}", **{key: item.get(key) for key in LIST_FIELDS}} + for item in document["items"] + if isinstance(item, dict) + ) + return sorted(rows, key=lambda row: (row["file"], str(row["received"]), str(row["id"]))) + + +def _read_jsonl(path: str) -> list[Any]: + """Read JSON lines from a file, or from stdin for "-"; a bad line becomes a ValueError marker.""" + text = sys.stdin.read() if path == "-" else Path(path).read_text(encoding="utf-8") + records: list[Any] = [] + for number, line in enumerate(text.splitlines(), start=1): + if not line.strip(): + continue + try: + records.append(json.loads(line)) + except ValueError as error: + records.append(ValueError(f"line {number}: {error}")) + return records + + +def main(argv: list[str] | None = None) -> int: + """Run the command line interface.""" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + + init_parser = commands.add_parser("init", help="create queue/ and write README.md") + init_parser.add_argument("queue_dir", type=Path) + + add_parser = commands.add_parser("add", help="add records, from JSON lines or from options") + add_parser.add_argument("queue_dir", type=Path) + add_parser.add_argument("--jsonl", help="file of JSON records, one per line ('-' for stdin)") + add_parser.add_argument("--source", choices=SOURCES) + add_parser.add_argument("--kind", choices=KINDS) + add_parser.add_argument("--title") + add_parser.add_argument("--url", default="") + add_parser.add_argument("--year", type=int) + add_parser.add_argument("--payload", help="file holding the payload JSON object ('-' for stdin)") + + remove_parser = commands.add_parser("remove", help="remove handled items by id, remembering the ids") + remove_parser.add_argument("queue_dir", type=Path) + remove_parser.add_argument("ids", nargs="+") + + list_parser = commands.add_parser("list", help="print every item as a JSON line") + list_parser.add_argument("queue_dir", type=Path) + + args = parser.parse_args(argv) + + if args.command == "init": + init_queue(args.queue_dir) + return 0 + + if args.command == "list": + for row in list_items(args.queue_dir): + print(json.dumps(row, ensure_ascii=False)) + return 0 + + if args.command == "remove": + removed = remove_items(args.queue_dir, args.ids) + for missing in sorted(set(args.ids) - removed): + print(f"desk_queue: no item {missing}", file=sys.stderr) + print(f"removed {len(removed)}") + return 0 + + if args.jsonl: + if args.source or args.kind or args.title or args.payload: + parser.error("--jsonl cannot be combined with --source, --kind, --title or --payload") + records = _read_jsonl(args.jsonl) + else: + if not (args.source and args.kind and args.title): + parser.error("add needs --jsonl, or --source, --kind and --title") + payload: Any = {} + if args.payload: + text = sys.stdin.read() if args.payload == "-" else Path(args.payload).read_text(encoding="utf-8") + payload = json.loads(text) + records = [ + { + "source": args.source, + "kind": args.kind, + "title": args.title, + "url": args.url, + "year": args.year, + "payload": payload, + }, + ] + + parsed = [record for record in records if not isinstance(record, ValueError)] + for bad in (record for record in records if isinstance(record, ValueError)): + print(f"desk_queue: record rejected: {bad}", file=sys.stderr) + init_queue(args.queue_dir) + results = add_items(args.queue_dir, parsed) + for result in results: + if "error" in result: + print(f"desk_queue: {result['error']}", file=sys.stderr) + else: + print(f"{result['status']} {result['file']} {result['id']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/utils/desk_worklist.py b/utils/desk_worklist.py index a985e9996c..8ba50f9d4e 100644 --- a/utils/desk_worklist.py +++ b/utils/desk_worklist.py @@ -54,6 +54,8 @@ from cfp_candidates import find_candidates from cfp_candidates import load_data from cfp_candidates import series_key + from desk_queue import clean_watch_title + from desk_queue import record_names from sort_yaml import DATEFORMAT from sort_yaml import DEFAULT_CFP_TIME from tidy_conf.titles import normalize_conference_name @@ -63,6 +65,8 @@ from .cfp_candidates import find_candidates from .cfp_candidates import load_data from .cfp_candidates import series_key + from .desk_queue import clean_watch_title + from .desk_queue import record_names from .sort_yaml import DATEFORMAT from .sort_yaml import DEFAULT_CFP_TIME from .tidy_conf.titles import normalize_conference_name @@ -94,35 +98,10 @@ ROTATION_DAYS = 7 AOE = timezone(timedelta(hours=-12), "AoE") -WATCH_PREFIX = re.compile(r"^\s*\[conference\]\s*", re.IGNORECASE) -WATCH_SUFFIX = re.compile(r"\s+(?:(?P\d{4})\s+)?-\s+(?:main|cfp)\s*$", re.IGNORECASE) DATED = re.compile(r"^\d{4}-\d{2}-\d{2}( \d{2}:\d{2}:\d{2})?$") UTC_OFFSET = re.compile(r"^UTC([+-])(\d{1,2})$") -def clean_watch_title(title: str) -> tuple[str, int | None]: - """Strip the decoration a changedetection watch title carries. - - Parameters - ---------- - title : str - Raw title, e.g. "[Conference] PyCon Greece 2027 - Main". - - Returns - ------- - tuple[str, int | None] - The bare name ("PyCon Greece") and the year from the suffix, if any. - """ - name = WATCH_PREFIX.sub("", title) - year = None - match = WATCH_SUFFIX.search(name) - if match: - name = name[: match.start()] - if match["year"]: - year = int(match["year"]) - return name.strip(), year - - def rotation_slot(text: str) -> int: """Return the weekday (Monday is 0) on which a rotating item is due. @@ -431,24 +410,12 @@ def _index(data: dict[str, list[dict]], known_mappings: dict) -> tuple[dict, dic return editions, lookup -def _queue_names(item: dict, payload: dict) -> list[str]: +def _queue_names(item: dict) -> list[str]: """Return the names a queue item can be matched by, most specific first.""" - raw = [] - matched = payload.get("matched") - if item.get("kind") == "candidate" and payload.get("classification") == "MISSING_EDITION": - raw.append(matched.get("conference") if isinstance(matched, dict) else None) - raw.append(payload.get("conference")) - fields = payload.get("fields") - if isinstance(fields, dict): - raw.append(fields.get("conference")) - if isinstance(payload.get("names"), list): - raw.extend(payload["names"]) - raw.extend((item.get("title"), item.get("series"))) - names = [] - for value in raw: - if not isinstance(value, str): - continue - name = clean_watch_title(value)[0] + names = record_names(item) + series = item.get("series") + if isinstance(series, str): + name = clean_watch_title(series)[0] if name and name not in names: names.append(name) if not names and isinstance(item.get("file"), str): @@ -521,7 +488,7 @@ def _route_queue_item(item: dict, editions: dict, lookup: dict, known_mappings: payload = item.get("payload") if isinstance(item.get("payload"), dict) else {} reason = kind if kind in QUEUE_KINDS else "signal" year = _queue_year(item, payload) - name, matches = _match(_queue_names(item, payload), lookup, known_mappings) + name, matches = _match(_queue_names(item), lookup, known_mappings) upcoming = {key: [e for path, e in editions[key] if path == DATA_PATHS["conferences"]] for key in matches} if not matches: diff --git a/utils/enrich_tba.py b/utils/enrich_tba.py index 93a24a6e10..aa7561f004 100644 --- a/utils/enrich_tba.py +++ b/utils/enrich_tba.py @@ -1,21 +1,21 @@ #!/usr/bin/env python3 -"""Enrich TBA conferences with CFP data from their websites. +"""Enrich TBA conferences with links found on their websites. -This module provides two enrichment modes: -- Quick: Deterministic extraction only (no API) - finds social links, sponsor/finaid URLs -- Full: Deterministic + Claude API - also extracts CFP dates, deadlines, and timezones +A TBA conference is one whose cfp has no date: TBA, TBD, or None for an +edition with no public CfP, whose links are still worth filling. A cancelled +edition is left out. + +The enrichment is deterministic. It lists the links on each conference's +homepage with lynx and matches them against known patterns: -The deterministic extraction handles: - Bluesky profiles (bsky.app/profile/...) +- YouTube channels (youtube.com and youtu.be links) - Mastodon profiles (/@username patterns) -- Sponsor pages (URLs with sponsor keywords) -- Financial aid pages (URLs with finaid/scholarship keywords) -- CFP links (URLs with cfp/submit/proposal keywords) - -The Claude API extraction handles: -- CFP deadlines (date parsing from various formats) -- Workshop/tutorial deadlines -- Conference timezones (IANA format) +- Sponsor pages (same-domain URLs with sponsor keywords) +- Financial aid pages (same-domain URLs with finaid or scholarship keywords) + +A found link fills a field that is empty or TBA. A link that differs from a +value already set becomes a review item, and the existing value is kept. """ from __future__ import annotations @@ -24,16 +24,14 @@ import json import re import shutil -import subprocess # noqa: S404 - subprocess is required for lynx text extraction +import subprocess # noqa: S404 - subprocess is required for lynx link extraction import sys -import zoneinfo from dataclasses import dataclass from dataclasses import field from pathlib import Path from typing import Any from urllib.parse import urlparse -import requests import yaml try: @@ -44,23 +42,9 @@ # Constants TBA_WORDS = ["tba", "tbd", "cancelled", "none", "na", "n/a", "nan", "n.a."] -CONFIDENCE_THRESHOLD_AUTO = 0.8 -CONFIDENCE_THRESHOLD_REVIEW = 0.5 -ANTHROPIC_API_URL = "https://api.anthropic.com/v1/messages" -MAX_CONTENT_LENGTH = 15000 # Max characters per conference website # Field type categorization for validation -URL_FIELDS = {"sponsor", "finaid", "mastodon", "bluesky", "youtube", "cfp_link"} -DATE_FIELDS = {"cfp", "workshop_deadline", "tutorial_deadline"} -TIMEZONE_FIELD = "timezone" - - -@dataclass -class FieldUpdate: - """Represents a single field update with confidence.""" - - value: str - confidence: float +URL_FIELDS = {"sponsor", "finaid", "mastodon", "bluesky", "youtube"} @dataclass @@ -69,18 +53,14 @@ class ConferenceUpdate: conference: str year: int - status: str # found | partial | not_announced | error - confidence: float - fields: dict[str, FieldUpdate] = field(default_factory=dict) - notes: str = "" + fields: dict[str, str] = field(default_factory=dict) @dataclass class EnrichmentResult: - """Complete enrichment result from Claude API.""" + """Enrichment results for all conferences.""" conferences: list[ConferenceUpdate] = field(default_factory=list) - summary: dict[str, int] = field(default_factory=dict) def validate_field_value(field_name: str, value: str) -> tuple[bool, str]: @@ -108,27 +88,12 @@ def validate_field_value(field_name: str, value: str) -> tuple[bool, str]: return False, f"URL must start with https:// or http://, got: {value[:50]}" return True, "" - if field_name in DATE_FIELDS: - if not re.match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$", value): - return False, f"Date must be YYYY-MM-DD HH:mm:ss format, got: {value}" - return True, "" - - if field_name == TIMEZONE_FIELD: - # Must be valid IANA timezone with slash (reject abbreviations) - if "/" not in value: - return False, f"Timezone must be IANA format with slash (e.g., America/Chicago), got: {value}" - try: - zoneinfo.ZoneInfo(value) - return True, "" - except zoneinfo.ZoneInfoNotFoundError: - return False, f"Invalid IANA timezone: {value}" - # Unknown field type - allow it return True, "" def find_tba_conferences(data_path: Path | None = None) -> list[dict[str, Any]]: - """Find all conferences with TBA CFP deadlines. + """Find all conferences with TBA CFP deadlines, leaving out cancelled editions. Parameters ---------- @@ -155,7 +120,8 @@ def find_tba_conferences(data_path: Path | None = None) -> list[dict[str, Any]]: tba_conferences = [] for conf in conferences: cfp = str(conf.get("cfp", "")).lower().strip() - if cfp in TBA_WORDS: + # A cancelled edition has nothing to enrich + if cfp in TBA_WORDS and cfp != "cancelled": tba_conferences.append(conf) logger.debug(f"Found TBA conference: {conf.get('conference')} {conf.get('year')}") @@ -163,63 +129,6 @@ def find_tba_conferences(data_path: Path | None = None) -> list[dict[str, Any]]: return tba_conferences -def prefetch_website(url: str, timeout: int = 30) -> str: - """Fetch website content as clean text. - - Parameters - ---------- - url : str - Conference website URL - timeout : int - Request timeout in seconds - - Returns - ------- - str - Cleaned text content from the website - """ - logger = get_logger() - - # Validate URL before processing - parsed = urlparse(url) - if parsed.scheme not in ("http", "https") or not parsed.netloc: - logger.warning(f"Invalid URL scheme or missing host: {url}") - return f"Invalid URL: {url}" - - # Try using lynx for clean text extraction (if available) - lynx_path = shutil.which("lynx") - if lynx_path: - try: - result = subprocess.run( - [lynx_path, "-dump", "-nolist", "-nonumbers", "-width=200", url], - capture_output=True, - text=True, - timeout=timeout, - ) - if result.returncode == 0 and result.stdout.strip(): - return result.stdout[:MAX_CONTENT_LENGTH] - except subprocess.TimeoutExpired: - pass - - # Fallback to requests + basic HTML stripping - try: - response = requests.get(url, timeout=timeout, headers={"User-Agent": "python-deadlines-bot/1.0"}) - response.raise_for_status() - - # Basic HTML to text conversion - import re - - text = response.text - text = re.sub(r"]*>.*?]*>", "", text, flags=re.DOTALL | re.IGNORECASE) - text = re.sub(r"]*>.*?]*>", "", text, flags=re.DOTALL | re.IGNORECASE) - text = re.sub(r"<[^>]+>", " ", text) - text = re.sub(r"\s+", " ", text) - return text.strip()[:MAX_CONTENT_LENGTH] - except requests.RequestException as e: - logger.warning(f"Failed to fetch {url}: {e}") - return f"Error fetching website: {e}" - - def get_all_links(url: str) -> list[str]: """Get all links from a page using lynx. @@ -297,8 +206,8 @@ def _domain_matches(domain: str, hosts: tuple[str, ...]) -> bool: def extract_links_from_url(url: str) -> dict[str, str]: """Extract known field values from page links deterministically. - This function finds social media links, sponsor pages, financial aid pages, - and CFP links without using any AI - purely pattern matching. + This function finds social media links, sponsor pages and financial aid + pages by pattern matching. Parameters ---------- @@ -403,8 +312,8 @@ def extract_links_from_url(url: str) -> dict[str, str]: def run_deterministic_extraction(conferences: list[dict[str, Any]]) -> EnrichmentResult: """Run deterministic link extraction on all conferences. - This extracts social media links and sponsor/finaid pages without using AI - - purely pattern matching. CFP links are validated by Claude in full mode. + This extracts social media links and sponsor and financial aid pages by + pattern matching. Parameters ---------- @@ -414,7 +323,7 @@ def run_deterministic_extraction(conferences: list[dict[str, Any]]) -> Enrichmen Returns ------- EnrichmentResult - Result with deterministically-found fields (confidence 1.0) + Result with the fields found for each conference """ logger = get_logger() result = EnrichmentResult() @@ -430,19 +339,11 @@ def run_deterministic_extraction(conferences: list[dict[str, Any]]) -> Enrichmen logger.debug(f"Deterministic extraction for {name} {year}") extracted = extract_links_from_url(url) - # Note: cfp_link is validated by Claude in full mode, not here - if extracted: - # Create ConferenceUpdate with deterministic fields - fields = {field_name: FieldUpdate(value=value, confidence=1.0) for field_name, value in extracted.items()} - update = ConferenceUpdate( conference=name, year=year, - status="found" if fields else "not_announced", - confidence=1.0, - fields=fields, - notes="Deterministic extraction (no AI)", + fields=extracted, ) result.conferences.append(update) logger.info(f" {name} {year}: found {list(extracted.keys())}") @@ -450,567 +351,12 @@ def run_deterministic_extraction(conferences: list[dict[str, Any]]) -> Enrichmen return result -def merge_enrichment_results(deterministic: EnrichmentResult, ai_result: EnrichmentResult) -> EnrichmentResult: - """Merge deterministic results with AI results. - - Deterministic results take precedence since they're more reliable. - - Parameters - ---------- - deterministic : EnrichmentResult - Results from deterministic extraction - ai_result : EnrichmentResult - Results from Claude API - - Returns - ------- - EnrichmentResult - Merged results - """ - # Create lookup for deterministic results - det_lookup: dict[str, ConferenceUpdate] = {} - for conf in deterministic.conferences: - key = f"{conf.conference}_{conf.year}" - det_lookup[key] = conf - - # Merge AI results with deterministic - merged_conferences = [] - - for ai_conf in ai_result.conferences: - key = f"{ai_conf.conference}_{ai_conf.year}" - det_conf = det_lookup.pop(key, None) - - if det_conf: - # Merge fields - deterministic takes precedence for same field - merged_fields = dict(ai_conf.fields) - for field_name, field_update in det_conf.fields.items(): - # Only add deterministic if AI didn't find it or has lower confidence - if field_name not in merged_fields or merged_fields[field_name].confidence < 1.0: - merged_fields[field_name] = field_update - - merged_conf = ConferenceUpdate( - conference=ai_conf.conference, - year=ai_conf.year, - status=ai_conf.status, - confidence=ai_conf.confidence, - fields=merged_fields, - notes=ai_conf.notes, - ) - merged_conferences.append(merged_conf) - else: - merged_conferences.append(ai_conf) - - # Add any remaining deterministic-only results - merged_conferences.extend(det_lookup.values()) - - return EnrichmentResult(conferences=merged_conferences, summary=ai_result.summary) - - -def find_cfp_links(url: str) -> list[str]: - """Find CFP-related links on a page using lynx. - - Parameters - ---------- - url : str - URL to extract links from - - Returns - ------- - list[str] - List of absolute URLs to CFP-related pages - """ - # Keywords that suggest a CFP/speaker page (multilingual) - cfp_keywords = [ - # English - core - "cfp", - "call-for", - "callfor", - "call_for", - "speaker", - "proposal", - "submit", - "submission", - "contribute", - "talk", - "present", - # English - academic/extended - "abstract", - "paper", - "papers", - "apply", - "application", - "participate", - "deadline", - "deadlines", - "important-dates", - "dates", - "lightning", - "poster", - "tutorial", - "workshop", - "session", - # Spanish (convocatoria=call, ponente=speaker, propuesta=proposal, enviar=submit, - # fechas=dates, participar=participate) - "convocatoria", - "ponente", - "propuesta", - "enviar", - "fechas", - "participar", - # German (einreichung=submission, vortrag=talk, sprecher=speaker, beitrag=contribution, - # aufruf=call, teilnehmen=participate, termine=dates, wichtige-termine=important-dates) - "einreichung", - "vortrag", - "sprecher", - "beitrag", - "aufruf", - "teilnehmen", - "termine", - "wichtige-termine", - # Portuguese (chamada=call, palestrante=speaker, proposta=proposal, - # submissão=submission, participar=participate, datas=dates) - "chamada", - "palestrante", - "proposta", - "submissao", - "submissão", - "participar", - "datas", - # French (appel=call, orateur=speaker, soumission=submission, conférencier=lecturer, - # participer=participate, dates-importantes=important-dates) - "appel", - "orateur", - "soumission", - "conferencier", - "conférencier", - "participer", - "dates-importantes", - # Japanese romanized (happyo=発表=presentation, oubo=応募=application, - # teishutsu=提出=submission, kouen=講演=lecture, boshu=募集=recruitment) - "happyo", - "oubo", - "teishutsu", - "kouen", - "boshu", - # Chinese romanized (zhenggao=征稿=call-for-papers, yanjiang=演讲=speech, - # tijiao=提交=submit, tougao=投稿=contribute) - "zhenggao", - "yanjiang", - "tijiao", - "tougao", - # Dutch (inzending=submission, spreker=speaker, voorstel=proposal, deelnemen=participate) - "inzending", - "spreker", - "voorstel", - "deelnemen", - # Italian (proposta=proposal, relatore=speaker, invio=submission, partecipare=participate) - "proposta", - "relatore", - "invio", - "partecipare", - # Polish (zgłoszenie=submission, prelegent=speaker, referat=paper, wystąpienie=presentation) - "zgloszenie", - "prelegent", - "referat", - "wystapienie", - # Russian romanized (doklad=доклад=report/talk, zayavka=заявка=application, - # vystuplenie=выступление=presentation, uchastie=участие=participation) - "doklad", - "zayavka", - "vystuplenie", - "uchastie", - ] - - links = get_all_links(url) - if not links: - return [] - - cfp_links = [] - seen = set() - - for link in links: - link_lower = link.lower() - - # Check if link contains CFP-related keywords and not already seen - if any(kw in link_lower for kw in cfp_keywords) and link not in seen: - seen.add(link) - cfp_links.append(link) - - return cfp_links[:3] # Limit to 3 CFP pages max - - -def prefetch_websites( - conferences: list[dict[str, Any]], -) -> tuple[dict[str, str], dict[str, str]]: - """Pre-fetch website content for multiple conferences. - - Also fetches CFP-related subpages if found on the main page, and tracks - which cfp_link was used for each conference (for validation). - - Parameters - ---------- - conferences : list[dict[str, Any]] - List of conference dicts - - Returns - ------- - tuple[dict[str, str], dict[str, str]] - Tuple of (content_map, cfp_link_map): - - content_map: Dict mapping conference key (name_year) to website content - - cfp_link_map: Dict mapping conference key to the cfp_link URL that was fetched - """ - logger = get_logger() - content_map: dict[str, str] = {} - cfp_link_map: dict[str, str] = {} - - for conf in conferences: - name = conf.get("conference", "Unknown") - year = conf.get("year", 0) - url = conf.get("link", "") - cfp_link = conf.get("cfp_link", "") # Use explicit cfp_link if available - key = f"{name}_{year}" - - if not url: - logger.warning(f"No URL for {key}") - content_map[key] = "No website URL available" - continue - - logger.info(f"Fetching: {url}") - - # Get main page content (uses lynx) - main_content = prefetch_website(url) - - # Find and fetch CFP-related subpages - additional_content = [] - - # If cfp_link is explicitly set, prioritize it - cfp_links_to_fetch = [] - if cfp_link: - cfp_links_to_fetch.append(cfp_link) - - # Also look for CFP links using lynx -listonly - found_links = find_cfp_links(url) - for link in found_links: - if link not in cfp_links_to_fetch and link != url: - cfp_links_to_fetch.append(link) - - # Track the first valid cfp_link we fetch (for later validation) - fetched_cfp_link = None - - # Fetch CFP subpages (limit to 2 to avoid too much content) - for cfp_url in cfp_links_to_fetch[:2]: - logger.debug(f" Also fetching CFP page: {cfp_url}") - cfp_content = prefetch_website(cfp_url) - if cfp_content and not cfp_content.startswith("Error"): - additional_content.append(f"\n\n--- CFP Page ({cfp_url}) ---\n{cfp_content}") - # Track the first successfully fetched cfp_link - if fetched_cfp_link is None: - fetched_cfp_link = cfp_url - - # Store the cfp_link that was fetched (if any) - if fetched_cfp_link: - cfp_link_map[key] = fetched_cfp_link - - # Combine all content - combined = main_content + "".join(additional_content) - content_map[key] = combined[:MAX_CONTENT_LENGTH] - - return content_map, cfp_link_map - - -def build_enrichment_prompt( - conferences: list[dict[str, Any]], - content_map: dict[str, str], -) -> str: - """Build the Claude API prompt for CFP data extraction. - - Note: URL fields (bluesky, mastodon, sponsor, finaid) are handled - deterministically. cfp_link is validated implicitly - if Claude finds - a CFP deadline in content that includes cfp_link content, that link is valid. - - Parameters - ---------- - conferences : list[dict[str, Any]] - List of conference dicts with TBA CFP - content_map : dict[str, str] - Dict mapping conference key to website content - - Returns - ------- - str - Formatted prompt string for Claude API - """ - fields_to_extract = [ - "cfp", - "workshop_deadline", - "tutorial_deadline", - "timezone", - ] - field_instructions = """ -Extract the following fields if found: -- cfp: MAIN CFP CLOSING deadline for talks/papers/proposals - the date submissions CLOSE - (MUST be format 'YYYY-MM-DD HH:mm:ss', use 23:59:00 if no time) -- workshop_deadline: Workshop submission CLOSING deadline (MUST be format 'YYYY-MM-DD HH:mm:ss') -- tutorial_deadline: Tutorial submission CLOSING deadline (MUST be format 'YYYY-MM-DD HH:mm:ss') -- timezone: Conference timezone (MUST be IANA format with slash, e.g., 'America/Chicago', 'Europe/Berlin') - - NEVER use abbreviations like EST, CEST, PST, UTC - ONLY full IANA names with slash - -CRITICAL RULES: -- cfp MUST be the MAIN Call for Papers/Proposals CLOSING deadline (when submissions end) -- cfp MUST NOT be the date the CFP OPENS / starts / launches - only when it CLOSES -- cfp MUST NOT be: sponsors, volunteers, specialist tracks, financial aid, grants, reviewers -- If only "Call for Sponsors/Volunteers/Tracks" found, set status to "not_announced" -- If only the CFP OPENING date is announced (e.g., "CFP opens on X", "CFP will open", - "submissions open on X", "proposals open on X") with no closing date, set status to - "not_announced" and DO NOT populate cfp -- Date fields: MUST be exactly 'YYYY-MM-DD HH:mm:ss' format -- Timezone: MUST be IANA format with slash (America/New_York), NEVER abbreviations (EST, CEST) -- Leave field EMPTY if not found on the page - -Opening-vs-closing examples (DO NOT treat opening dates as deadlines): -- "CFP will open on February 1, 2026" → status "not_announced", cfp EMPTY -- "Call for Proposals opens March 15" → status "not_announced", cfp EMPTY -- "Submissions open: Feb 1 — Submissions close: Mar 15" → cfp="2026-03-15 23:59:00" -- "Deadline: March 15, 2026" → cfp="2026-03-15 23:59:00" -- "Submit proposals by March 15, 2026" → cfp="2026-03-15 23:59:00" - -Date conversion examples: -- "February 8, 2026" → "2026-02-08 23:59:00" -- "8th of February 2026 at 23:59 CET" → "2026-02-08 23:59:00" -- "Feb 8" (current year context) → "2026-02-08 23:59:00" -""" - - conference_sections = [] - for conf in conferences: - name = conf.get("conference", "Unknown") - year = conf.get("year", 0) - url = conf.get("link", "") - key = f"{name}_{year}" - content = content_map.get(key, "No content available") - - conference_sections.append( - f""" -### {name} {year} -URL: {url} - -Website Content: -``` -{content[:8000]} -``` -""", - ) - - prompt = f"""You are an expert at extracting conference information from websites. -Analyze the following Python conference websites and extract CFP (Call for Proposals) information. - -{field_instructions} - -IMPORTANT RULES: -1. Only extract dates that are EXPLICITLY stated on the website -2. Do NOT guess or approximate dates - if unsure, set status to "not_announced" -3. If a deadline says "TBA", "coming soon", or is not found, set status to "not_announced" -4. If only a CFP OPENING date is listed (no closing deadline yet), set status to - "not_announced" - an opening date is NOT a submission deadline -5. Use status "found" when you have extracted all requested date fields with clear values -6. Use status "partial" ONLY when some fields are found but others are missing -7. CONFIDENCE SCORING: - - 0.9-1.0: Date is clearly and unambiguously stated (even if in human-readable format) - - 0.7-0.9: Date is stated but requires interpretation (e.g., relative dates) - - 0.5-0.7: Date is implied or uncertain - - Below 0.5: Do not include - set status to "not_announced" instead -8. For URLs, only include if they appear to be valid absolute URLs - -Fields to extract: {', '.join(fields_to_extract)} - -CONFERENCES TO ANALYZE: -{''.join(conference_sections)} - -Respond with ONLY valid JSON in this exact format: -{{ - "conferences": [ - {{ - "conference": "Conference Name", - "year": 2026, - "status": "found | partial | not_announced | error", - "confidence": 0.0-1.0, - "fields": {{ - "cfp": {{ "value": "2026-02-15 23:59:00", "confidence": 0.95 }}, - ... other fields as applicable - }}, - "notes": "Optional notes about what was found" - }} - ], - "summary": {{ - "total": 5, - "found": 3, - "partial": 1, - "not_announced": 1, - "errors": 0 - }} -}} -""" - return prompt - - -def call_claude_api( - prompt: str, - api_key: str, - model: str = "claude-haiku-4-5-20251001", - max_tokens: int = 4096, -) -> dict[str, Any] | None: - """Call Claude API with the enrichment prompt. - - Parameters - ---------- - prompt : str - The formatted prompt - api_key : str - Anthropic API key - model : str - Model to use - max_tokens : int - Maximum tokens in response - - Returns - ------- - dict[str, Any] | None - Parsed JSON response or None on error - """ - logger = get_logger() - - headers = { - "x-api-key": api_key, - "content-type": "application/json", - "anthropic-version": "2023-06-01", - } - - payload = { - "model": model, - "max_tokens": max_tokens, - "messages": [{"role": "user", "content": prompt}], - } - - for attempt in range(3): - try: - response = requests.post( - ANTHROPIC_API_URL, - headers=headers, - json=payload, - timeout=120, - ) - - if response.status_code == 200: - result = response.json() - content = result.get("content", [{}])[0].get("text", "") - - # Debug: Log the raw API response - logger.debug("=" * 80) - logger.debug("RAW CLAUDE API RESPONSE:") - logger.debug("=" * 80) - logger.debug(content) - logger.debug("=" * 80) - - # Log token usage - usage = result.get("usage", {}) - logger.info( - f"API usage - Input: {usage.get('input_tokens', 0)}, Output: {usage.get('output_tokens', 0)}", - ) - - # Parse JSON from response - try: - # Try to find JSON in the response - json_start = content.find("{") - json_end = content.rfind("}") + 1 - if json_start >= 0 and json_end > json_start: - parsed_json = json.loads(content[json_start:json_end]) - logger.debug("Successfully parsed JSON from response") - return parsed_json - logger.error("No valid JSON found in response") - return None - except json.JSONDecodeError as e: - logger.error(f"Failed to parse JSON response: {e}") - return None - - if response.status_code in (429, 529): - logger.warning(f"Rate limited (HTTP {response.status_code}), attempt {attempt + 1}/3") - import time - - time.sleep((attempt + 1) * 5) - continue - - logger.error(f"API error: HTTP {response.status_code} - {response.text}") - return None - - except requests.RequestException as e: - logger.error(f"Request error: {e}") - return None - - return None - - -def parse_response(response: dict[str, Any]) -> EnrichmentResult: - """Parse Claude API response into structured result. - - Parameters - ---------- - response : dict[str, Any] - Raw JSON response from Claude API - - Returns - ------- - EnrichmentResult - EnrichmentResult with parsed updates - """ - logger = get_logger() - result = EnrichmentResult() - - if not response: - logger.debug("Empty response received") - return result - - result.summary = response.get("summary", {}) - logger.debug(f"Response summary: {result.summary}") - - for conf_data in response.get("conferences", []): - conf_name = conf_data.get("conference", "") - conf_year = conf_data.get("year", 0) - conf_status = conf_data.get("status", "error") - - logger.debug(f"Parsing conference: {conf_name} {conf_year} (status: {conf_status})") - - fields = {} - for field_name, field_data in conf_data.get("fields", {}).items(): - if isinstance(field_data, dict): - field_value = field_data.get("value", "") - field_confidence = field_data.get("confidence", 0.0) - fields[field_name] = FieldUpdate( - value=field_value, - confidence=field_confidence, - ) - logger.debug( - f" Parsed field: {field_name} = '{field_value}' (confidence: {field_confidence:.2f})", - ) - - update = ConferenceUpdate( - conference=conf_name, - year=conf_year, - status=conf_status, - confidence=conf_data.get("confidence", 0.0), - fields=fields, - notes=conf_data.get("notes", ""), - ) - result.conferences.append(update) - - logger.debug(f"Parsed {len(result.conferences)} conferences from response") - return result - - def apply_updates( data_path: Path, updates: EnrichmentResult, dry_run: bool = False, ) -> tuple[int, list[str]]: - """Apply high-confidence updates to conferences.yml. + """Apply found links to empty or TBA fields in conferences.yml. Parameters ---------- @@ -1035,15 +381,10 @@ def apply_updates( review_items = [] for update in updates.conferences: - if update.status not in ("found", "partial"): - continue - - # Find matching conference (strip year suffix if Claude included it) + # Find matching conference matching = None - update_name = update.conference.lower().replace(str(update.year), "").strip() for i, conf in enumerate(conferences): - conf_name = conf.get("conference", "").lower() - if (conf_name == update_name or conf_name == update.conference.lower()) and conf.get("year") == update.year: + if conf.get("conference") == update.conference and conf.get("year") == update.year: matching = i break @@ -1056,13 +397,13 @@ def apply_updates( logger.debug(f"Processing updates for: {update.conference} {update.year}") - for field_name, field_update in update.fields.items(): - if not field_update.value: + for field_name, value in update.fields.items(): + if not value: logger.debug(f" Skipping {field_name}: empty value") continue # Validate field value before applying - is_valid, validation_reason = validate_field_value(field_name, field_update.value) + is_valid, validation_reason = validate_field_value(field_name, value) if not is_valid: logger.warning( f" REJECTED {field_name} for {update.conference} {update.year}: " @@ -1070,55 +411,33 @@ def apply_updates( ) continue - logger.debug( - f" Field {field_name}: value='{field_update.value}', " - f"confidence={field_update.confidence:.2f}, valid={is_valid}", - ) - - if field_update.confidence >= CONFIDENCE_THRESHOLD_AUTO: - # Auto-apply high confidence updates - but ONLY if field is empty/TBA - old_value = conf.get(field_name) - old_value_str = str(old_value).lower().strip() if old_value else "" - - # Skip if field already has a valid (non-TBA) value - if old_value and old_value_str not in TBA_WORDS and old_value_str != "": - # Add to review if the values differ - might be worth checking - if old_value != field_update.value: - review_items.append( - f"⚠️ {update.conference} {update.year}: {field_name} - " - f"existing='{old_value}' vs found='{field_update.value}' " - f"(kept existing, confidence: {field_update.confidence:.2f})", - ) - logger.info( - f" REVIEW {field_name} for {update.conference} {update.year}: " - f"kept '{old_value}', found '{field_update.value}'", - ) - else: - logger.debug( - f" Skipping {field_name}: already has matching value '{old_value}'", - ) - continue - - if old_value != field_update.value: - conf[field_name] = field_update.value - changes_made.append(f"{field_name}: {old_value} -> {field_update.value}") + # Apply only if the field is empty or TBA + old_value = conf.get(field_name) + old_value_str = str(old_value).lower().strip() if old_value else "" + + # Skip if field already has a valid (non-TBA) value + if old_value and old_value_str not in TBA_WORDS and old_value_str != "": + # Add to review if the values differ - might be worth checking + if old_value != value: + review_items.append( + f"⚠️ {update.conference} {update.year}: {field_name} - " + f"existing='{old_value}' vs found='{value}' (kept existing)", + ) logger.info( - f"APPLIED {field_name} for {update.conference} {update.year}: " - f"'{old_value}' -> '{field_update.value}' " - f"(confidence: {field_update.confidence:.2f})", + f" REVIEW {field_name} for {update.conference} {update.year}: " + f"kept '{old_value}', found '{value}'", ) else: - logger.debug(f" Skipping {field_name}: value unchanged") - elif field_update.confidence >= CONFIDENCE_THRESHOLD_REVIEW: - # Add to review list - logger.debug(f" Adding {field_name} to review list (confidence below auto threshold)") - review_items.append( - f"{update.conference} {update.year}: " - f"{field_name}={field_update.value} " - f"(confidence: {field_update.confidence:.2f})", - ) - else: - logger.debug(f" Skipping {field_name}: confidence {field_update.confidence:.2f} too low") + logger.debug( + f" Skipping {field_name}: already has matching value '{old_value}'", + ) + continue + + conf[field_name] = value + changes_made.append(f"{field_name}: {old_value} -> {value}") + logger.info( + f"APPLIED {field_name} for {update.conference} {update.year}: '{old_value}' -> '{value}'", + ) if changes_made: conferences[matching] = conf @@ -1135,24 +454,17 @@ def apply_updates( def enrich_tba_conferences( - enrichment_level: str = "full", - api_key: str | None = None, dry_run: bool = False, data_path: Path | None = None, ) -> bool: """Main function to enrich TBA conferences. + Finds the conferences with a TBA CFP, extracts social links and sponsor + and financial aid pages from their websites, and fills the fields that + are empty or TBA. + Parameters ---------- - enrichment_level : str - Enrichment level: - - 'quick': Deterministic only (no API call) - extracts social links, - sponsor/finaid URLs via pattern matching. Fast and free. - - 'full': Deterministic + Claude API - also extracts CFP dates, - workshop/tutorial deadlines, and timezones from page content. - api_key : str | None - Anthropic API key (defaults to ANTHROPIC_API_KEY env var). - Only required for 'full' enrichment level. dry_run : bool If True, don't write changes data_path : Path | None @@ -1178,70 +490,13 @@ def enrich_tba_conferences( logger.info("No TBA conferences found") return True - # Step 2: Run deterministic extraction (no AI needed - always runs) + # Step 2: Extract links from the conference websites logger.info("Running deterministic link extraction...") - deterministic_result = run_deterministic_extraction(tba_conferences) - det_count = len([c for c in deterministic_result.conferences if c.fields]) + result = run_deterministic_extraction(tba_conferences) + det_count = len([c for c in result.conferences if c.fields]) logger.info(f"Deterministic: found links for {det_count} conferences") - # Step 3: If quick mode, skip AI and just use deterministic results - if enrichment_level == "quick": - logger.info("Quick mode - skipping Claude API, using deterministic results only") - result = deterministic_result - else: - # Full mode: also call Claude for dates/timezone - if api_key is None: - api_key = os.environ.get("ANTHROPIC_API_KEY") - if not api_key: - logger.error("ANTHROPIC_API_KEY not set (required for full enrichment)") - return False - - # Pre-fetch websites for AI processing - logger.info("Pre-fetching conference websites...") - content_map, cfp_link_map = prefetch_websites(tba_conferences) - - # Build and send prompt to Claude for date/timezone extraction - logger.info("Calling Claude API for date/timezone extraction...") - prompt = build_enrichment_prompt(tba_conferences, content_map) - - # Debug: Log the full prompt for debugging - logger.debug("=" * 80) - logger.debug("FULL PROMPT SENT TO CLAUDE API:") - logger.debug("=" * 80) - logger.debug(prompt) - logger.debug("=" * 80) - - response = call_claude_api(prompt, api_key) - - if not response: - logger.error("Failed to get response from Claude API") - return False - - # Parse response - logger.info("Parsing response...") - ai_result = parse_response(response) - - logger.info( - f"AI Results: {ai_result.summary.get('found', 0)} found, " - f"{ai_result.summary.get('partial', 0)} partial, " - f"{ai_result.summary.get('not_announced', 0)} not announced", - ) - - # Add cfp_link to conferences where Claude found a CFP deadline - # This validates the cfp_link - if Claude found CFP info in the content - # that includes the cfp_link page, that link is valid - for conf_update in ai_result.conferences: - key = f"{conf_update.conference}_{conf_update.year}" - if "cfp" in conf_update.fields and key in cfp_link_map: - cfp_link_url = cfp_link_map[key] - conf_update.fields["cfp_link"] = FieldUpdate(value=cfp_link_url, confidence=1.0) - logger.info(f" {key}: validated cfp_link {cfp_link_url}") - - # Merge deterministic and AI results - logger.info("Merging deterministic and AI results...") - result = merge_enrichment_results(deterministic_result, ai_result) - - # Step 4: Apply updates + # Step 3: Apply updates if dry_run: logger.info("Dry run - not applying changes") @@ -1257,7 +512,7 @@ def enrich_tba_conferences( # Build markdown content for PR description review_md = "## 👀 Items for Manual Review\n\n" - review_md += "The following items had moderate confidence and need verification:\n\n" + review_md += "The following links differ from values already set. The existing values were kept:\n\n" for item in review_items: review_md += f"- [ ] {item}\n" review_md += "\nPlease verify these values before merging.\n" @@ -1289,13 +544,7 @@ def enrich_tba_conferences( def main() -> None: """CLI entry point.""" - parser = argparse.ArgumentParser(description="Enrich TBA conferences with CFP data") - parser.add_argument( - "--level", - choices=["quick", "full"], - default="full", - help="quick=deterministic links only (no API), full=links + dates via Claude API", - ) + parser = argparse.ArgumentParser(description="Enrich TBA conferences with links from their websites") parser.add_argument( "--dry-run", action="store_true", @@ -1322,7 +571,6 @@ def main() -> None: setup_logging(level=args.log_level) success = enrich_tba_conferences( - enrichment_level=args.level, dry_run=args.dry_run, data_path=args.data_path, ) diff --git a/utils/import_python_official.py b/utils/import_python_official.py index 6966ae811d..c59fe15d6e 100644 --- a/utils/import_python_official.py +++ b/utils/import_python_official.py @@ -36,6 +36,9 @@ logger = get_tqdm_logger(__name__) +# Recorded with merge questions handed to the desk queue +SOURCE_NAME = "python.org" + def fill_links_from_history(df_ics: pd.DataFrame, df_yml: pd.DataFrame) -> pd.DataFrame: """Fill missing links in ICS data from historical conference data. @@ -380,6 +383,7 @@ def main(year=None, base="") -> bool: df_merged, df_remote, merge_report = fuzzy_match( df_yml[df_yml["year"] == y], df_ics.loc[df_ics["year"] == y], + source_name=SOURCE_NAME, ) logger.info( f"Merge report: {merge_report.exact_matches} exact, " @@ -391,7 +395,7 @@ def main(year=None, base="") -> bool: df_merged = df_merged.drop(["conference"], axis=1) df_merged = deduplicate(df_merged) df_remote = deduplicate(df_remote) - df_merged = merge_conferences(df_merged, df_remote) + df_merged = merge_conferences(df_merged, df_remote, source_name=SOURCE_NAME) # Concatenate the new data with the existing data df_new = pd.concat([df_new, df_merged], ignore_index=True) diff --git a/utils/import_python_organizers.py b/utils/import_python_organizers.py index cc1f4ca0ff..78f1608984 100644 --- a/utils/import_python_organizers.py +++ b/utils/import_python_organizers.py @@ -31,6 +31,9 @@ from .tidy_conf.yaml import load_title_mappings from .tidy_conf.yaml import write_df_yaml +# Recorded with merge questions handed to the desk queue +SOURCE_NAME = "python-organizers" + def load_remote(year: int) -> pd.DataFrame: """Load conference data from GitHub CSV for a specific year. @@ -314,7 +317,7 @@ def main(year: int | None = None, base: str = "") -> None: f"Year {y}: df_yml_year shape: {df_yml_year.shape}, df_csv_year shape: {df_csv_year.shape}", ) - df_merged, df_remote, merge_report = fuzzy_match(df_yml_year, df_csv_year) + df_merged, df_remote, merge_report = fuzzy_match(df_yml_year, df_csv_year, source_name=SOURCE_NAME) logger.info( f"Merge report: {merge_report.exact_matches} exact, " f"{merge_report.fuzzy_matches} fuzzy, {merge_report.no_matches} no match", @@ -333,7 +336,7 @@ def main(year: int | None = None, base: str = "") -> None: f"After deduplication - df_merged: {df_merged.shape}, df_remote: {df_remote.shape}", ) - df_merged = merge_conferences(df_merged, df_remote) + df_merged = merge_conferences(df_merged, df_remote, source_name=SOURCE_NAME) logger.info( f"Merge conferences completed for year {y}. Final shape: {df_merged.shape}", ) diff --git a/utils/tidy_conf/interactive_merge.py b/utils/tidy_conf/interactive_merge.py index e26f3562de..f26113719c 100644 --- a/utils/tidy_conf/interactive_merge.py +++ b/utils/tidy_conf/interactive_merge.py @@ -101,6 +101,44 @@ def is_placeholder_value(value) -> bool: return stripped in ("TBA", "TBD", "NONE", "N/A", "") or not stripped +def _row_year(row) -> int | None: + """Return a merge row's edition year as an int, or None.""" + for key in ("year", "year_x", "year_y"): + value = row.get(key) if key in row.index else None + if value is None or pd.isna(value): + continue + try: + return int(value) + except (TypeError, ValueError): + continue + return None + + +def _row_link(row) -> str: + """Return a merge row's main link, preferring the YAML side, or "".""" + for key in ("link", "link_x", "link_y"): + value = row.get(key) if key in row.index else None + if isinstance(value, str) and value: + return value + return "" + + +# The text of every query_yes_no prompt below, values included, goes into the id +# of the queue item it becomes (tidy_conf.utils.record_question). Rewording a +# prompt therefore queues every standing question again, once. +def _question_context(row, conference, field, yaml_value, remote_value, source_name) -> dict: + """Describe a merge question for the desk (see tidy_conf.utils.record_question).""" + return { + "conference": conference, + "year": _row_year(row), + "field": field, + "yaml_value": yaml_value, + "remote_value": remote_value, + "source_name": source_name, + "url": _row_link(row), + } + + def resolve_conflict( yaml_val, remote_val, @@ -206,6 +244,7 @@ def fuzzy_match( df_yml: pd.DataFrame, df_remote: pd.DataFrame, report: MergeReport | None = None, + source_name: str | None = None, ) -> tuple[pd.DataFrame, pd.DataFrame, MergeReport]: """Fuzzy merge conferences from two pandas dataframes on title. @@ -224,6 +263,9 @@ def fuzzy_match( Remote source DataFrame (CSV or ICS) report : MergeReport, optional Merge report for tracking operations + source_name : str, optional + Name of the remote source, recorded with questions asked where no + one can answer them Returns ------- @@ -397,6 +439,7 @@ def best_match(title_match): # Prompt user for fuzzy matches that aren't excluded elif not query_yes_no( f"Do '{row['conference']}' and '{title}' match? (y/n): ", + context=_question_context(row, conference_name, "conference", row["conference"], title, source_name), ): logger.info( f"Fuzzy match rejected: '{conference_name}' vs '{title}' (score: {prob})", @@ -476,6 +519,7 @@ def merge_conferences( df_yml: pd.DataFrame, df_remote: pd.DataFrame, report: MergeReport | None = None, + source_name: str | None = None, ) -> pd.DataFrame: """Merge two dataframes on title and interactively resolve conflicts. @@ -493,6 +537,9 @@ def merge_conferences( Remote source DataFrame report : MergeReport, optional Merge report for tracking operations + source_name : str, optional + Name of the remote source, recorded with questions asked where no + one can answer them Returns ------- @@ -633,6 +680,7 @@ def merge_conferences( else: if query_yes_no( f"For {i} in column '{column}' would you prefer '{ry}' or keep '{rx}'?", + context=_question_context(row, conference_name, column, rx, ry, source_name), ): df_new.loc[i, column] = ry else: @@ -681,10 +729,20 @@ def merge_conferences( f"For {i} in column '{column}' would you prefer " f"'{ry + cfp_time_y}' or keep '{rx + cfp_time_x}'?" ), + context=_question_context( + row, + conference_name, + column, + rx + cfp_time_x, + ry + cfp_time_y, + source_name, + ), ): df_new.loc[i, column] = ry + cfp_time_y else: # Check if it's an extension of the deadline and update both + # No context: the cfp question above already queued + # both deadlines for the desk. if query_yes_no("Is this an extension?"): rrx, rry = ( int(rx.replace("-", "").split(" ")[0]), @@ -722,6 +780,7 @@ def merge_conferences( else: if query_yes_no( f"For {i} in column '{column}' would you prefer '{ryy}' or keep '{rxx}'?", + context=_question_context(row, conference_name, column, rxx, ryy, source_name), ): df_new.loc[i, column] = ryy else: @@ -730,6 +789,7 @@ def merge_conferences( # For everything else give a choice if query_yes_no( f"For {i} in column '{column}' would you prefer '{ry}' or keep '{rx}'?", + context=_question_context(row, conference_name, column, rx, ry, source_name), ): df_new.loc[i, column] = ry else: diff --git a/utils/tidy_conf/links.py b/utils/tidy_conf/links.py index b125f7e607..f4c9092860 100644 --- a/utils/tidy_conf/links.py +++ b/utils/tidy_conf/links.py @@ -1,5 +1,3 @@ -import json -import os import re import time from datetime import datetime @@ -11,6 +9,15 @@ import requests from tqdm import tqdm +try: + from tidy_conf.queue_out import QUEUE_OUT_ENV + from tidy_conf.queue_out import append_queue_record + from tidy_conf.queue_out import utc_now +except ImportError: + from .queue_out import QUEUE_OUT_ENV + from .queue_out import append_queue_record + from .queue_out import utc_now + # Sub-page link fields that must point somewhere other than the main link. # A cfp_link/sponsor/finaid that is just the homepage again carries no # information (e.g. "cfp_link: https://2027.pycon.de/" on the PyCon DE entry @@ -73,11 +80,8 @@ REDIRECT_RECORDS: list[dict] = [] # When this environment variable names a file, every redirect record is also -# appended to it as one JSON object per line, for a later job to pick up. The -# lines are not complete queue items: the queue writer assigns each item its -# "id" and the series key of the file it goes in when it adds the line, so -# both are defined in one place. -REDIRECT_RECORDS_ENV = "DESK_QUEUE_OUT" +# appended to it as one JSON line for the desk queue (see queue_out.py). +REDIRECT_RECORDS_ENV = QUEUE_OUT_ENV def normalize_url_pointer(url: str) -> tuple: @@ -260,7 +264,7 @@ def record_redirect( record = { "source": "link-check", "kind": "redirect", - "received": datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "received": utc_now(), "title": conference or url, "url": url, "year": year, @@ -273,13 +277,7 @@ def record_redirect( }, } REDIRECT_RECORDS.append(record) - - out_path = os.environ.get(REDIRECT_RECORDS_ENV) - if out_path: - out_file = Path(out_path) - out_file.parent.mkdir(parents=True, exist_ok=True) - with out_file.open("a", encoding="utf-8", newline="\n") as f: - f.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + append_queue_record(record) return record diff --git a/utils/tidy_conf/queue_out.py b/utils/tidy_conf/queue_out.py new file mode 100644 index 0000000000..f6f0284a83 --- /dev/null +++ b/utils/tidy_conf/queue_out.py @@ -0,0 +1,46 @@ +"""Hand findings to the deadline desk's queue as JSON lines. + +Jobs that run the data tooling in GitHub Actions set ``DESK_QUEUE_OUT`` to a +file path. Code that finds something only a person or the desk can judge (a +link that now redirects to another site, a merge question with no one to +answer it) appends a record to that file, and the job adds the file to the +desk-queue branch with ``utils/desk_queue.py add --jsonl``. The records lack +"id": the queue writer assigns it, and the series key of the file each item +goes in, so both are defined in one place. +""" + +import json +import os +from datetime import datetime +from datetime import timezone +from pathlib import Path + +QUEUE_OUT_ENV = "DESK_QUEUE_OUT" + + +def utc_now() -> str: + """Return the current time in the queue's "received" format.""" + return datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def append_queue_record(record: dict) -> bool: + """Append a record to the file ``DESK_QUEUE_OUT`` names, if it names one. + + Parameters + ---------- + record : dict + A queue item without "id". + + Returns + ------- + bool + True when the record was written. + """ + out_path = os.environ.get(QUEUE_OUT_ENV) + if not out_path: + return False + out_file = Path(out_path) + out_file.parent.mkdir(parents=True, exist_ok=True) + with out_file.open("a", encoding="utf-8", newline="\n") as f: + f.write(json.dumps(record, ensure_ascii=False, sort_keys=True, default=str) + "\n") + return True diff --git a/utils/tidy_conf/utils.py b/utils/tidy_conf/utils.py index bfa7dfcbb8..4f0565dc66 100644 --- a/utils/tidy_conf/utils.py +++ b/utils/tidy_conf/utils.py @@ -12,6 +12,13 @@ from yaml.representer import SafeRepresenter +try: + from tidy_conf.queue_out import append_queue_record + from tidy_conf.queue_out import utc_now +except ImportError: + from .queue_out import append_queue_record + from .queue_out import utc_now + _mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG @@ -77,8 +84,70 @@ def pretty_print(header, conf, tba=None, expired=None) -> None: print("\n") +# Merge questions asked where no one could answer them (CI). Each is also +# handed to the desk queue when DESK_QUEUE_OUT names a file (queue_out.py), so +# the desk checks the live site; the merge itself keeps the default answer. +QUESTION_RECORDS: list[dict] = [] + + +def _queue_text(value) -> str | None: + """Return a merge value as text for a queue record, or None when it is empty.""" + if value is None: + return None + try: + if pd.isna(value): + return None + except (TypeError, ValueError): + pass + return str(value) + + +def record_question(question: str, default_answer: bool, context: dict) -> dict: + """Record a merge question that was answered with its default. + + Parameters + ---------- + question : str + The question as it would have been asked. + default_answer : bool + The answer the merge went on with. + context : dict + What the question is about: "conference", "year" (int or None), + "field", "yaml_value", "remote_value", "source_name" and "url". + + Returns + ------- + dict + The record, a queue item without "id" with source "canonical-import" + and kind "question". + """ + year = context.get("year") + payload = { + "question": question, + "conference": context.get("conference"), + "year": year, + "field": context.get("field"), + "yaml_value": _queue_text(context.get("yaml_value")), + "remote_value": _queue_text(context.get("remote_value")), + "default_answer": default_answer, + "source_name": context.get("source_name"), + } + record = { + "source": "canonical-import", + "kind": "question", + "received": utc_now(), + "title": payload["conference"] or question, + "url": context.get("url") or "", + "year": year, + "payload": payload, + } + QUESTION_RECORDS.append(record) + append_queue_record(record) + return record + + # Helper function for yes no questions -def query_yes_no(question, default="no"): +def query_yes_no(question, default="no", context=None): """Ask a yes/no question via input() and return their answer. "question" is a string that is presented to the user. "default" is the presumed @@ -87,16 +156,19 @@ def query_yes_no(question, default="no"): The "answer" return value is True for "yes" or False for "no". - In non-interactive environments (e.g., CI), returns the default value. + In non-interactive environments (e.g., CI), returns the default value. When + "context" describes what the question is about (see record_question), the + unanswered question is recorded for the desk. """ valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} # In non-interactive environments, use the default value if not sys.stdin.isatty(): - if default is not None: - return valid[default] # If no default and non-interactive, default to "no" for safety - return False + answer = valid[default] if default is not None else False + if context is not None: + record_question(question, answer, context) + return answer match default: case "yes":