fix: request every releases page once and cache release listings

README.md

@@ -49,6 +49,7 @@ With DLoad, you can:
     - [Download Types](#download-types)
     - [Version Constraints](#version-constraints)
     - [Advanced Configuration Options](#advanced-configuration-options)
+    - [Caching Release Lists](#caching-release-lists)
 - [Building Custom RoadRunner](#building-custom-roadrunner)
     - [Build Action Configuration](#build-action-configuration)
     - [Velox Action Attributes](#velox-action-attributes)
@@ -350,6 +351,50 @@ Use Composer-style version constraints:
 </dload>
 ```
 
+### Caching Release Lists
+
+Resolving a version means asking GitHub or GitLab for the repository's release list. DLoad can keep
+those listings in a directory and reuse them, so repeated runs resolve the same versions without
+spending the API rate limit:
+
+```xml
+<dload temp-dir="./runtime" cache-dir="./runtime/dload-cache" cache-ttl="3600">
+    <actions>
+        <download software="rr" />
+    </actions>
+</dload>
+```
+
+| Attribute   | Environment variable | Default | Meaning                                                       |
+|-------------|----------------------|---------|---------------------------------------------------------------|
+| `cache-dir` | `DLOAD_CACHE_DIR`    | not set | Directory to store cached release listings in. Caching is off until it is set. |
+| `cache-ttl` | `DLOAD_CACHE_TTL`    | `600`   | Seconds a cached listing stays usable. `0` disables caching.  |
+
+> [!NOTE]
+> Only successful release listings are cached. Failed requests are never stored, so a rate limit
+> answer is not replayed after the limit is gone, and asset downloads do not go through the cache:
+> the directory holds listings only, never the downloaded binaries.
+
+In GitHub Actions the directory can be carried between jobs, so only the first job of a workflow run
+spends any rate limit on listings:
+
+```yaml
+- name: Cache DLoad release lists
+  uses: actions/cache@v4
+  with:
+    path: ./runtime/dload-cache
+    key: dload-cache-${{ github.run_id }}
+    restore-keys: dload-cache-
+
+- run: ./vendor/bin/dload get
+  env:
+    DLOAD_CACHE_DIR: ./runtime/dload-cache
+```
+
+The `github.run_id` in the key makes every workflow run write a fresh entry, while `restore-keys`
+lets the remaining jobs of that run restore it. A static key would never be written again and the
+cached listings would stay stale forever.
+
 ## Building Custom RoadRunner
 
 DLoad supports building custom RoadRunner binaries using the Velox build tool. This is useful when you need RoadRunner with custom plugin combinations that aren't available in pre-built releases.
@@ -600,6 +645,9 @@ Add to CI/CD environment variables for automated downloads.
 > 1,000 requests per hour across all jobs of the repository. With a large job matrix the limit may run out,
 > and downloads from other repositories may be rejected. Use a personal access token if that happens.
 
+Release listings can also be cached between runs, which removes them from the rate limit budget
+entirely: see [Caching Release Lists](#caching-release-lists).
+
 ## Failure Reporting
 
 `dload get` exits with a non-zero code when at least one requested package was not installed, and prints

dload.xsd

@@ -249,6 +249,16 @@
                     <xs:documentation>Temporary directory for downloads</xs:documentation>
                 </xs:annotation>
             </xs:attribute>
+            <xs:attribute name="cache-dir" type="xs:string">
+                <xs:annotation>
+                    <xs:documentation>Directory to cache release listings in; caching is disabled when not set</xs:documentation>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="cache-ttl" type="xs:nonNegativeInteger" default="600">
+                <xs:annotation>
+                    <xs:documentation>Number of seconds a cached release listing stays usable; 0 disables caching</xs:documentation>
+                </xs:annotation>
+            </xs:attribute>
         </xs:complexType>
     </xs:element>
 </xs:schema>

src/Bootstrap.php

@@ -8,10 +8,14 @@
 use Internal\Container\ObjectContainer;
 use Internal\DLoad\Module\Binary\BinaryProvider;
 use Internal\DLoad\Module\Binary\Internal\BinaryProviderImpl;
+use Internal\DLoad\Module\Cache\Internal\FileResponseCache;
+use Internal\DLoad\Module\Cache\Internal\NullResponseCache;
+use Internal\DLoad\Module\Cache\ResponseCache;
 use Internal\DLoad\Module\Common\Architecture;
 use Internal\DLoad\Module\Common\Internal\Injection\ConfigInflector;
 use Internal\DLoad\Module\Common\OperatingSystem;
 use Internal\DLoad\Module\Common\Stability;
+use Internal\DLoad\Module\Config\Schema\Cache as CacheConfig;
 use Internal\DLoad\Module\HttpClient\Factory;
 use Internal\DLoad\Module\HttpClient\Internal\NyholmFactoryImpl;
 use Internal\DLoad\Module\Repository\Internal\GitHub\Factory as GithubRepositoryFactory;
@@ -21,6 +25,7 @@
 use Internal\DLoad\Module\Velox\Builder;
 use Internal\DLoad\Module\Velox\Internal\Client\BuildRoadRunner;
 use Internal\DLoad\Module\Velox\Internal\VeloxBuilder;
+use Internal\DLoad\Service\Logger;
 
 /**
  * Bootstraps the application by configuring the dependency container.
@@ -113,6 +118,16 @@ public function withConfig(
                 ->addRepositoryFactory($container->get(GithubRepositoryFactory::class))
                 ->addRepositoryFactory($container->get(GitLabRepositoryFactory::class)),
         );
+        $this->container->bind(
+            ResponseCache::class,
+            static function (Container $container): ResponseCache {
+                $config = $container->get(CacheConfig::class);
+
+                return $config->dir === null || $config->ttl <= 0
+                    ? new NullResponseCache()
+                    : new FileResponseCache($config->dir, $config->ttl, $container->get(Logger::class));
+            },
+        );
         $this->container->bind(BinaryProvider::class, BinaryProviderImpl::class);
         $this->container->bind(Factory::class, NyholmFactoryImpl::class);
         $this->container->bind(Builder::class, VeloxBuilder::class);

src/Module/Cache/Internal/FileResponseCache.php

@@ -0,0 +1,134 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Internal\DLoad\Module\Cache\Internal;
+
+use Internal\DLoad\Module\Cache\ResponseCache;
+use Internal\DLoad\Service\Logger;
+use Nyholm\Psr7\Response;
+use Psr\Http\Message\ResponseInterface;
+
+/**
+ * @internal
+ * @psalm-internal Internal\DLoad
+ */
+final class FileResponseCache implements ResponseCache
+{
+    /**
+     * @param non-empty-string $directory
+     * @param int<1, max> $ttl
+     */
+    public function __construct(
+        private readonly string $directory,
+        private readonly int $ttl,
+        private readonly Logger $logger,
+    ) {}
+
+    public function remember(string $key, \Closure $fetch): ResponseInterface
+    {
+        $file = $this->fileOf($key);
+
+        $cached = $this->read($file);
+        if ($cached !== null) {
+            return $cached;
+        }
+
+        $response = $fetch();
+        $this->write($file, $response);
+
+        return $response;
+    }
+
+    private static function readBody(ResponseInterface $response): string
+    {
+        $stream = $response->getBody();
+
+        $stream->isSeekable() and $stream->rewind();
+        $body = $stream->getContents();
+        $stream->isSeekable() and $stream->rewind();
+
+        return $body;
+    }
+
+    private function read(string $file): ?ResponseInterface
+    {
+        if (!\is_file($file)) {
+            return null;
+        }
+
+        $content = @\file_get_contents($file);
+        if ($content === false) {
+            $this->discard($file);
+            return null;
+        }
+
+        try {
+            /** @var mixed $payload */
+            $payload = \json_decode($content, true, 512, JSON_THROW_ON_ERROR);
+
+            \is_array($payload)
+            && \is_int($payload['created_at'] ?? null)
+            && \is_int($payload['status'] ?? null)
+            && \is_array($payload['headers'] ?? null)
+            && \is_string($payload['body'] ?? null)
+                or throw new \UnexpectedValueException('Unexpected cache entry structure.');
+        } catch (\Throwable) {
+            $this->discard($file);
+            return null;
+        }
+
+        /** @var array{created_at: int, status: int, headers: array<string, list<string>>, body: string} $payload */
+
+        if (\time() - $payload['created_at'] > $this->ttl) {
+            return null;
+        }
+
+        return new Response($payload['status'], $payload['headers'], $payload['body']);
+    }
+
+    private function write(string $file, ResponseInterface $response): void
+    {
+        $status = $response->getStatusCode();
+
+        if ($status < 200 || $status > 299) {
+            return;
+        }
+
+        try {
+            $payload = \json_encode([
+                'created_at' => \time(),
+                'status' => $status,
+                'headers' => $response->getHeaders(),
+                'body' => self::readBody($response),
+            ], JSON_THROW_ON_ERROR);
+
+            if (!\is_dir($this->directory) && !@\mkdir($this->directory, 0777, true) && !\is_dir($this->directory)) {
+                throw new \RuntimeException(\sprintf('Failed to create cache directory `%s`.', $this->directory));
+            }
+
+            $temp = $file . '.' . (string) \getmypid() . '.tmp';
+
+            if (@\file_put_contents($temp, $payload) === false) {
+                throw new \RuntimeException(\sprintf('Failed to write cache entry `%s`.', $temp));
+            }
+
+            if (!@\rename($temp, $file)) {
+                @\unlink($temp);
+                throw new \RuntimeException(\sprintf('Failed to store cache entry `%s`.', $file));
+            }
+        } catch (\Throwable $e) {
+            $this->logger->exception($e, important: false);
+        }
+    }
+
+    private function discard(string $file): void
+    {
+        @\unlink($file);
+    }
+
+    private function fileOf(string $key): string
+    {
+        return $this->directory . \DIRECTORY_SEPARATOR . \hash('xxh128', $key) . '.json';
+    }
+}

src/Module/Cache/Internal/NullResponseCache.php

@@ -0,0 +1,20 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Internal\DLoad\Module\Cache\Internal;
+
+use Internal\DLoad\Module\Cache\ResponseCache;
+use Psr\Http\Message\ResponseInterface;
+
+/**
+ * @internal
+ * @psalm-internal Internal\DLoad
+ */
+final class NullResponseCache implements ResponseCache
+{
+    public function remember(string $key, \Closure $fetch): ResponseInterface
+    {
+        return $fetch();
+    }
+}

src/Module/Cache/ResponseCache.php

@@ -0,0 +1,18 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Internal\DLoad\Module\Cache;
+
+use Psr\Http\Message\ResponseInterface;
+
+/**
+ * @internal
+ */
+interface ResponseCache
+{
+    /**
+     * @param \Closure(): ResponseInterface $fetch
+     */
+    public function remember(string $key, \Closure $fetch): ResponseInterface;
+}

src/Module/Config/Schema/Cache.php

@@ -0,0 +1,25 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Internal\DLoad\Module\Config\Schema;
+
+use Internal\DLoad\Module\Common\Internal\Attribute\Env;
+use Internal\DLoad\Module\Common\Internal\Attribute\InflectableConfig;
+use Internal\DLoad\Module\Common\Internal\Attribute\XPath;
+
+/**
+ * @internal
+ */
+#[InflectableConfig]
+final class Cache
+{
+    /** @var non-empty-string|null $dir */
+    #[XPath('/dload/@cache-dir')]
+    #[Env('DLOAD_CACHE_DIR')]
+    public ?string $dir = null;
+
+    #[XPath('/dload/@cache-ttl')]
+    #[Env('DLOAD_CACHE_TTL')]
+    public int $ttl = 600;
+}

src/Module/Repository/Internal/GitHub/Api/RepositoryApi.php

@@ -4,6 +4,7 @@
 
 namespace Internal\DLoad\Module\Repository\Internal\GitHub\Api;
 
+use Internal\DLoad\Module\Cache\ResponseCache;
 use Internal\DLoad\Module\HttpClient\Factory as HttpFactory;
 use Internal\DLoad\Module\HttpClient\Method;
 use Internal\DLoad\Module\Repository\Exception\ApiException;
@@ -27,6 +28,7 @@ final class RepositoryApi
 {
     private const URL_REPOSITORY = 'https://api.github.com/repos/%s';
     private const URL_RELEASES = 'https://api.github.com/repos/%s/releases';
+    private const RELEASES_PER_PAGE = 100;
 
     /**
      * @var non-empty-string
@@ -43,6 +45,7 @@ public function __construct(
         string $owner,
         string $repo,
         private readonly Logger $logger,
+        private readonly ResponseCache $cache,
     ) {
         $this->repositoryPath = $owner . '/' . $repo;
     }
@@ -196,13 +199,12 @@ private function decodeReleasesResponse(ResponseInterface $response): array
      */
     private function releasesRequest(int $page): ResponseInterface
     {
-        return $this->request(
-            Method::Get,
-            $this->httpFactory->uri(
-                \sprintf(self::URL_RELEASES, $this->repositoryPath),
-                ['page' => $page],
-            ),
+        $uri = $this->httpFactory->uri(
+            \sprintf(self::URL_RELEASES, $this->repositoryPath),
+            ['page' => $page, 'per_page' => self::RELEASES_PER_PAGE],
         );
+
+        return $this->cache->remember((string) $uri, fn(): ResponseInterface => $this->request(Method::Get, $uri));
     }
 
     private function hasNextPage(ResponseInterface $response): bool