dev: generated checklist of DF defect mitigations for release review

ci/list-df-mitigations.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""List DF defect mitigations that should be re-reviewed on a new DF release.
+
+DFHack works around a number of defects in Dwarf Fortress itself. When Bay 12
+releases a new version of DF, some of those mitigations may need to be adjusted
+or removed. This script generates the review checklist so the release
+coordinator does not have to track the mitigations by hand.
+
+Mitigations are marked where they live in the code with a comment containing
+the ``DF-MITIGATION:`` marker, e.g.::
+
+    // DF-MITIGATION: site_id is not assigned on reclaim until the first save
+
+In addition, every ``fix/*`` script in the scripts repo is a mitigation by
+definition, so they are listed automatically without needing a marker. Their
+descriptions come from the ``:summary:`` field of their documentation.
+
+The checklist is printed to stdout. If the GITHUB_STEP_SUMMARY environment
+variable is set (i.e. when running in a GitHub Actions job), it is also
+appended to the job summary so it is visible on the workflow run page.
+"""
+
+import os
+import re
+
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+SELF = relpath_self = os.path.relpath(os.path.abspath(__file__),
+                                      REPO_ROOT).replace(os.sep, '/')
+
+MARKER_RE = re.compile(r'DF-MITIGATION:?\s*(.*?)\s*(?:\*+/)?\s*$')
+SUMMARY_RE = re.compile(r'^\s*:summary:\s*(.*)$')
+
+# directories that never contain our own code
+SKIP_DIRS = {'.git', 'build', 'depends', 'package'}
+
+SOURCE_EXTS = {
+    '.c', '.cc', '.cpp', '.cxx', '.h', '.hh', '.hpp',
+    '.lua', '.py', '.rb', '.js', '.ts', '.sh', '.ps1',
+}
+
+
+def iter_source_files(root):
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames
+                       if d not in SKIP_DIRS and not d.startswith('.')]
+        for name in filenames:
+            if os.path.splitext(name)[1].lower() in SOURCE_EXTS:
+                yield os.path.join(dirpath, name)
+
+
+def relpath(path):
+    return os.path.relpath(path, REPO_ROOT).replace(os.sep, '/')
+
+
+def collect_markers():
+    entries = []
+    for path in iter_source_files(REPO_ROOT):
+        if relpath(path) == SELF:
+            continue
+        try:
+            with open(path, encoding='utf-8', errors='replace') as f:
+                for lineno, line in enumerate(f, 1):
+                    match = MARKER_RE.search(line)
+                    if match:
+                        entries.append((relpath(path), lineno,
+                                        match.group(1) or '(no description)'))
+        except OSError:
+            continue
+    return sorted(entries)
+
+
+def doc_summary(script_path):
+    """Look up the :summary: field in the script's documentation."""
+    rel = relpath(script_path)
+    name = os.path.splitext(os.path.basename(rel))[0]
+    for doc in (os.path.join(REPO_ROOT, 'scripts', 'docs', 'fix',
+                             name + '.rst'),
+                os.path.join(REPO_ROOT, 'scripts', 'docs', name + '.rst')):
+        try:
+            with open(doc, encoding='utf-8', errors='replace') as f:
+                for line in f:
+                    match = SUMMARY_RE.match(line)
+                    if match:
+                        return match.group(1).strip()
+        except OSError:
+            continue
+    return None
+
+
+def script_description(path):
+    """Fall back to the first substantive comment line of the script."""
+    try:
+        with open(path, encoding='utf-8', errors='replace') as f:
+            for line in f:
+                line = line.strip()
+                if line.startswith('--'):
+                    desc = line.lstrip('-').strip()
+                    if desc and not desc.startswith('@'):
+                        return desc
+                elif line:
+                    return None
+    except OSError:
+        pass
+    return None
+
+
+def collect_fix_scripts():
+    """fix/* scripts (and the top-level fix* scripts) are all DF bug
+    mitigations."""
+    entries = []
+    scripts_dir = os.path.join(REPO_ROOT, 'scripts')
+    if not os.path.isdir(scripts_dir):
+        return entries
+    for dirpath, _dirnames, filenames in os.walk(scripts_dir):
+        if relpath(dirpath) not in ('scripts', 'scripts/fix'):
+            continue
+        for name in filenames:
+            if not name.endswith('.lua'):
+                continue
+            if relpath(dirpath) == 'scripts' and not name.startswith('fix'):
+                continue
+            path = os.path.join(dirpath, name)
+            desc = doc_summary(path) or script_description(path) \
+                or '(no description)'
+            entries.append((relpath(path), desc))
+    return sorted(entries)
+
+
+def main():
+    lines = [
+        '## DF defect mitigations',
+        '',
+        'A new DF release may have fixed some of the defects that DFHack works',
+        'around. Review each entry and adjust or remove mitigations (and their',
+        '`DF-MITIGATION` markers) that are no longer needed.',
+        '',
+        '### Code sites',
+        '',
+    ]
+    markers = collect_markers()
+    if markers:
+        for path, lineno, desc in markers:
+            lines.append(f'- [ ] `{path}:{lineno}`: {desc}')
+    else:
+        lines.append('- (none found)')
+
+    lines += ['', '### `fix/*` scripts', '']
+    fixes = collect_fix_scripts()
+    if fixes:
+        for path, desc in fixes:
+            lines.append(f'- [ ] `{path}`: {desc}')
+    else:
+        lines.append('- (scripts repo not checked out)')
+
+    lines.append('')
+    output = '\n'.join(lines)
+    print(output)
+
+    summary_path = os.environ.get('GITHUB_STEP_SUMMARY')
+    if summary_path:
+        with open(summary_path, 'a', encoding='utf-8') as f:
+            f.write(output)
+
+
+if __name__ == '__main__':
+    main()

docs/dev/Contributing.rst

@@ -79,6 +79,24 @@ General C++ code guidelines
 * Prefer range for loops to traditional for loops when iterating over a container.
 * Avoid macros when possible; prefer ``constexpr`` variables for constants and functions or templates for code generation.
 
+Marking mitigations for DF bugs
+-------------------------------
+DFHack works around a number of defects in Dwarf Fortress itself. When Bay 12
+fixes one of these defects in a new DF release, the corresponding mitigation
+may need to be adjusted or removed. To keep track of these mitigations, mark
+them with a ``DF-MITIGATION:`` comment where they live in the code, e.g.::
+
+    // DF-MITIGATION: site_id is not assigned on reclaim until the first save
+
+The marker is also appropriate for code whose behavior depends on a DF defect
+without working around it, e.g. comments explaining why a field we set has no
+effect. Briefly describe the defect and include a reference to the DF bug or
+DFHack issue/PR when one exists. The markers are collected by
+:source:`ci/list-df-mitigations.py` into a checklist that is reviewed on every
+new DF release; see `release-process-df-mitigations`. The ``fix/*`` scripts are
+all DF bug mitigations by definition and are listed automatically, so they do
+not need markers.
+
 .. _contributing-pr-guidelines:
 
 Pull request guidelines

docs/dev/release-process.rst

@@ -6,6 +6,29 @@ This page details the process we follow for beta and stable releases.
 For documentation on the related GitHub workflows, see
 `workflows-release-automation`.
 
+.. _release-process-df-mitigations:
+
+New DF releases
+---------------
+
+When Bay 12 releases a new version of DF, the mitigations we maintain for
+defects in DF itself may need to be adjusted or removed. These mitigations are
+marked with ``DF-MITIGATION:`` comments in the code (see
+`contributing` for the convention) and the ``fix/*`` scripts in
+the scripts repo are all mitigations by definition.
+
+To review them:
+
+1. Run ``ci/list-df-mitigations.py`` in a DFHack checkout (with submodules) to
+   generate the checklist.
+
+2. For each entry, determine whether the new DF version still exhibits the
+   defect. Some entries can be checked by code inspection; others need a save
+   that reproduces the defect.
+
+3. Remove or adjust mitigations that are no longer needed, and remove their
+   ``DF-MITIGATION`` markers. Keep entries that still apply.
+
 Beta release
 ------------
 
@@ -78,52 +101,54 @@ branch back into ``develop`` and remove the release branch to clean up.
     - https://github.com/DFHack/scripts/commits/master
     - https://github.com/DFHack/df-structures/commits/master
 
-4. Update version strings in :source:`CMakeLists.txt` as appropriate
+4. Check the `DF defect mitigation checklist <release-process-df-mitigations>`_ to see if any mitigations need to be updated
+
+5. Update version strings in :source:`CMakeLists.txt` as appropriate
 
     - Ensure the ``DFHACK_PRERELEASE`` flag is set to ``FALSE``.
     - Set ``RELEASE`` in your environment for the commands below (e.g. ``RELEASE=51.07-r1``)
 
-5. Replace "Future" with the version number and clean up changelog entries; add new "Future" section (with headers pre-populated from the template at the top of the file):
+6. Replace "Future" with the version number and clean up changelog entries; add new "Future" section (with headers pre-populated from the template at the top of the file):
 
     - ``docs/changelog.txt``
     - ``scripts/changelog.txt``
     - ``library/xml/changelog.txt``
     - ``plugins/stonesense/docs/changelog.txt``
 
-6. Do a top-level build to ensure the docs build cleanly
+7. Do a top-level build to ensure the docs build cleanly
 
-7. Commit/push changes to submodules and tag (``git tag -a $RELEASE -m "Bump to $RELEASE"; git push --tags origin master``)
+8. Commit/push changes to submodules and tag (``git tag -a $RELEASE -m "Bump to $RELEASE"; git push --tags origin master``)
 
     - ``scripts``
     - ``library/xml``
     - ``plugins/stonesense``
 
-8. Commit and push changes to ``develop``
+9. Commit and push changes to ``develop``
 
     - Ensure that any updates you pushed to submodules are tracked in the commit to ``DFHack/develop``
 
-9. Tag ``dfhack``: ``git tag -a $RELEASE -m "Bump to $RELEASE"; git push --tags origin develop``
+10. Tag ``dfhack``: ``git tag -a $RELEASE -m "Bump to $RELEASE"; git push --tags origin develop``
 
     - This will automatically trigger a `Deploy to Steam <https://github.com/DFHack/dfhack/actions/workflows/steam-deploy.yml>`_ GitHub action to the "staging" Steam branch and a `Deploy to GitHub <https://github.com/DFHack/dfhack/actions/workflows/github-release.yml>`_ GitHub action to create a draft `release <https://github.com/DFHack/dfhack/releases>`_ from a template and attach the built artifacts.
 
-10. Switch to the Steam ``staging`` release channel in the Steam client (password: ``stagingstagingstaging``) and download/test the update.
+11. Switch to the Steam ``staging`` release channel in the Steam client (password: ``stagingstagingstaging``) and download/test the update.
 
     - Ensure DFHack starts DF when run from the Steam client
     - Ensure the DFHack version string is accurate on the title page (should just be the release number, e.g. ``DFHack 51.07-r1``, with no git hash or warnings)
     - Run `devel/check-release`
     - If something goes wrong with this step, fix it, delete the tag (both from `GitHub <https://github.com/DFHack/dfhack/tags>`_ and locally (``git tag -d $RELEASE``)), re-tag, re-push, and re-test. Note that you do *not* need to remove the GitHub draft release -- the existing one will just get updated with the new tag and binaries. You *can* remove the draft release, though, if you want the release notes to get regenerated.
 
-11. Prep release on GitHub
+12. Prep release on GitHub
 
     - Go to the draft `release <https://github.com/DFHack/dfhack/releases>`_ on GitHub
     - Add announcements, highlights (with demo videos), etc. to the description
 
-12. Push develop to master (``git push origin develop:master``)
+13. Push develop to master (``git push origin develop:master``)
 
     - This will start the documentation build process and update the published "stable" docs
     - Note that if this is a -r1 release, you won't be able to complete this step until a classic build is available on the Bay 12 website so the DFHack Test workflow can pass, which is a prerequisite for being able to push to ``master``.
 
-13. Post release notes on Steam
+14. Post release notes on Steam
 
     - Go to the `announcement creation page <https://steamcommunity.com/games/2346660/partnerevents/create>`_
     - Select "A game update"
@@ -140,16 +165,16 @@ branch back into ``develop`` and remove the release branch to clean up.
     - Go to the Artwork tab, select "Previously uploaded images", and search for and double-click on STABLEannouncement6.png. Click "Upload" (even though it has already been uploaded).
     - Switch to the "Publish" tab and publish!
 
-14. Go to the `Steam builds page <https://partner.steamgames.com/apps/builds/2346660>`_ and promote the build to the "default" branch
+15. Go to the `Steam builds page <https://partner.steamgames.com/apps/builds/2346660>`_ and promote the build to the "default" branch
 
     - For the build that you just pushed to "staging", click the "-- Select an app branch --" drop-down and select "default"
     - Click on "Preview Change"
     - Commit the change (you may need to verify with 2FA)
     - If the release is newer than what's on the ``beta`` and/or ``testing`` branches, set it live on those branches as well
 
-15. Publish the prepped GitHub release
+16. Publish the prepped GitHub release
 
-16. Send out release announcements
+17. Send out release announcements
 
     - Announce new version in r/dwarffortress. Example: https://www.reddit.com/r/dwarffortress/comments/1i3l5xl/dfhack_5015r2_released_highlights_stonesense/
         - Create the post in the Reddit web interface; the mobile app is extremely painful to use for posting
@@ -163,9 +188,9 @@ branch back into ``develop`` and remove the release branch to clean up.
     - Announce in `#mod-releases <https://discord.com/channels/329272032778780672/1066180550114680853>`_ on Kitfox Discord
         - Change the name of the release thread on Kitfox Discord to match the release version (if you are not Myk, ping Myk for this)
 
-17. Monitor all announcement channels for feedback and respond to questions/complaints
+18. Monitor all announcement channels for feedback and respond to questions/complaints
 
-18. Create a `project <https://github.com/orgs/dfhack/projects>`_ on GitHub in the DFHack org for the next release
+19. Create a `project <https://github.com/orgs/dfhack/projects>`_ on GitHub in the DFHack org for the next release
 
     - Open the `project template <https://github.com/orgs/DFHack/projects/52>`_
     - Click "Use this template"
@@ -174,7 +199,7 @@ branch back into ``develop`` and remove the release branch to clean up.
     - Move any remaining To Do or In Progress items from last release project to next release project
     - Close project for last release
 
-19. If this is a -r2 release or later, go to https://readthedocs.org/projects/dfhack/versions/ and "Edit" previous DFHack releases for the same DF version and mark them "Hidden" (keep the "Active" flag set) so they no longer appear on the docs version selector.
+20. If this is a -r2 release or later, go to https://readthedocs.org/projects/dfhack/versions/ and "Edit" previous DFHack releases for the same DF version and mark them "Hidden" (keep the "Active" flag set) so they no longer appear on the docs version selector.
 
 .. _converting-markdown-to-bbcode:
 

library/modules/Buildings.cpp

@@ -128,7 +128,8 @@ static df::building_extents_type *getExtentTile(const df::building::T_room &room
 }
 
 /*
- * A monitor to work around this bug, in its application to buildings:
+ * DF-MITIGATION: monitor works around DF bug 1416 in its application to
+ * buildings:
  *
  * http://www.bay12games.com/dwarves/mantisbt/view.php?id=1416
  */

library/modules/Gui.cpp

@@ -2040,8 +2040,12 @@ void Gui::showPopupAnnouncement(std::string message, int color, bool bright)
 {
     df::popup_message *popup = new df::popup_message();
     popup->text = message;
-    popup->color = color; // Doesn't do anything anymore? Popups are always [C:7:0:0] gray text
-    popup->bright = bright; // See: https://dwarffortressbugtracker.com/view.php?id=12672
+    // DF-MITIGATION: DF ignores popup color/bright fields (bug 12672)
+    // Popups always render as [C:7:0:0] gray text; keep setting the fields so
+    // they take effect again if DF is fixed:
+    // https://dwarffortressbugtracker.com/view.php?id=12672
+    popup->color = color;
+    popup->bright = bright;
 
     auto &popups = world->status.popups;
     popups.push_back(popup);

library/modules/MapCache.cpp

@@ -531,7 +531,8 @@ void MapExtras::Block::ParseTiles(TileInfo *tiles)
 
                     tt = con->original_tile;
 
-                    // Ice under construction is buggy:
+                    // DF-MITIGATION: DF bug 6330 makes ice under constructions
+                    // behave incorrectly:
                     // http://www.bay12games.com/dwarves/mantisbt/view.php?id=6330
                     // Therefore we just pretend it wasn't there (if it isn't too late),
                     // and overwrite it if/when we write the base layer.

library/modules/Screen.cpp

@@ -845,6 +845,7 @@ void dfhack_viewscreen::logic()
     // Various stuff works poorly unless always repainting
     Screen::invalidate();
 
+    // DF-MITIGATION: DF can get stuck when a dismissed screen stays buried
     // if the DF screen immediately beneath the DFHack viewscreens is waiting to
     // be dismissed, raise it to the top so DF never gets stuck
     auto *p = parent;

library/modules/World.cpp

@@ -218,8 +218,8 @@ int32_t World::GetCurrentSiteId() {
     if (!plotinfo)
         return -1;
     if (isFortressMode()) {
-        // on a reclaimed fortress, site_id isn't assigned until the first
-        // save; fortress_site is set at embark, so use it as a fallback
+        // DF-MITIGATION: reclaimed forts lack site_id until first save (#5716)
+        // fortress_site is set at embark, so use it as a fallback
         if (plotinfo->site_id >= 0)
             return plotinfo->site_id;
         if (auto site = plotinfo->main.fortress_site)

plugins/lua/buildingplan/unlink_mechanisms.lua

@@ -224,7 +224,9 @@ function MechLinkOverlay:init()
             on_activate = self:callback("ask_unlink_all"),
             enabled = function() return next(self.links) end,
         },
-        widgets.Scrollbar --Work around for https://dwarffortressbugtracker.com/view.php?id=12721
+        -- DF-MITIGATION: extra scrollbar works around DF bug 12721
+        -- https://dwarffortressbugtracker.com/view.php?id=12721
+        widgets.Scrollbar
         {
             view_id = "scroll",
             frame = {t=0, r=0, h=24},