better cache

site/scripts/cache-clear.ts

@@ -0,0 +1,22 @@
+import "dotenv/config";
+
+import { createCacheFromEnv } from "../src/lib/cache";
+
+function log(msg: string) {
+  // eslint-disable-next-line no-console
+  console.log(`[cache-clear] ${msg}`);
+}
+
+async function main() {
+  const cache = await createCacheFromEnv(process.env, { namespace: "fast-website", log });
+  await cache.flush();
+  await cache.close();
+  log("ok");
+}
+
+main().catch((e) => {
+  // eslint-disable-next-line no-console
+  console.error(`[cache-clear] failed: ${String(e)}`);
+  process.exitCode = 1;
+});
+

site/scripts/fetch-content.ts

@@ -4,6 +4,8 @@ import { promises as fs } from "node:fs";
 import path from "node:path";
 
 import { getIngestConfigFromEnv } from "../src/lib/config";
+import { createCacheFromEnv } from "../src/lib/cache";
+import { cachedCompute } from "../src/lib/cache/memoize";
 import type { ContentCache, ContentItem } from "../src/lib/content/types";
 import { readInstagramEmbedPosts } from "../src/lib/ingest/instagram";
 import { fetchPodcastRss } from "../src/lib/ingest/podcast";
@@ -42,6 +44,11 @@ async function main() {
   const all: ContentItem[] = [];
   const outPath = path.join(process.cwd(), "content", "cache", "content.json");
 
+  const kv = await createCacheFromEnv(process.env, {
+    namespace: "fast-website",
+    log,
+  });
+
   // Read the existing cache so we can keep last-known-good sections if a source fails.
   let existing: ContentCache | undefined;
   try {
@@ -56,17 +63,29 @@ async function main() {
     log("YouTube: skipped (missing YOUTUBE_CHANNEL_ID)");
   } else if (cfg.youtubeApiKey) {
     try {
-      const items = await fetchYoutubeViaApi(cfg.youtubeChannelId, cfg.youtubeApiKey, 25);
+      const cacheKey = `youtube:api:${cfg.youtubeChannelId}:25`;
+      const { value: items, cached } = await cachedCompute(kv, cacheKey, () =>
+        fetchYoutubeViaApi(cfg.youtubeChannelId!, cfg.youtubeApiKey!, 25),
+      );
+      log(`YouTube: API ${cached ? "cache" : "live"} (${items.length} items)`);
       log(`YouTube: API ok (${items.length} items)`);
       all.push(...items);
     } catch (e) {
       log(`YouTube: API failed (${String(e)}), falling back to RSS`);
-      const items = await fetchYoutubeViaRss(cfg.youtubeChannelId, 25);
+      const cacheKey = `youtube:rss:${cfg.youtubeChannelId}:25`;
+      const { value: items, cached } = await cachedCompute(kv, cacheKey, () =>
+        fetchYoutubeViaRss(cfg.youtubeChannelId!, 25),
+      );
+      log(`YouTube: RSS ${cached ? "cache" : "live"} (${items.length} items)`);
       log(`YouTube: RSS ok (${items.length} items)`);
       all.push(...items);
     }
   } else {
-    const items = await fetchYoutubeViaRss(cfg.youtubeChannelId, 25);
+    const cacheKey = `youtube:rss:${cfg.youtubeChannelId}:25`;
+    const { value: items, cached } = await cachedCompute(kv, cacheKey, () =>
+      fetchYoutubeViaRss(cfg.youtubeChannelId!, 25),
+    );
+    log(`YouTube: RSS ${cached ? "cache" : "live"} (${items.length} items)`);
     log(`YouTube: RSS ok (${items.length} items)`);
     all.push(...items);
   }
@@ -76,7 +95,11 @@ async function main() {
     log("Podcast: skipped (missing PODCAST_RSS_URL)");
   } else {
     try {
-      const items = await fetchPodcastRss(cfg.podcastRssUrl, 50);
+      const cacheKey = `podcast:rss:${cfg.podcastRssUrl}:50`;
+      const { value: items, cached } = await cachedCompute(kv, cacheKey, () =>
+        fetchPodcastRss(cfg.podcastRssUrl!, 50),
+      );
+      log(`Podcast: RSS ${cached ? "cache" : "live"} (${items.length} items)`);
       log(`Podcast: RSS ok (${items.length} items)`);
       all.push(...items);
     } catch (e) {
@@ -103,11 +126,17 @@ async function main() {
     wordpress = existing?.wordpress || wordpress;
   } else {
     try {
-      const wp = await fetchWordpressContent({
-        baseUrl: cfg.wordpressBaseUrl,
-        username: cfg.wordpressUsername,
-        appPassword: cfg.wordpressAppPassword,
-      });
+      const cacheKey = `wp:content:${cfg.wordpressBaseUrl}`;
+      const { value: wp, cached } = await cachedCompute(kv, cacheKey, () =>
+        fetchWordpressContent({
+          baseUrl: cfg.wordpressBaseUrl!,
+          username: cfg.wordpressUsername,
+          appPassword: cfg.wordpressAppPassword,
+        }),
+      );
+      log(
+        `WordPress: wp-json ${cached ? "cache" : "live"} (${wp.posts.length} posts, ${wp.pages.length} pages, ${wp.categories.length} categories)`,
+      );
       wordpress = wp;
       log(
         `WordPress: wp-json ok (${wp.posts.length} posts, ${wp.pages.length} pages, ${wp.categories.length} categories)`,
@@ -119,14 +148,16 @@ async function main() {
     }
   }
 
-  const cache: ContentCache = {
+  const contentCache: ContentCache = {
     generatedAt,
     items: dedupe(all),
     wordpress,
   };
 
-  await writeAtomic(outPath, JSON.stringify(cache, null, 2));
-  log(`Wrote cache: ${outPath} (${cache.items.length} total items)`);
+  await writeAtomic(outPath, JSON.stringify(contentCache, null, 2));
+  log(`Wrote cache: ${outPath} (${contentCache.items.length} total items)`);
+
+  await kv.close();
 }
 
 main().catch((e) => {