better cache

This commit is contained in:
2026-02-10 01:20:58 -05:00
parent c773affbc8
commit f056e67eae
39 changed files with 830 additions and 17 deletions

View File

@@ -19,3 +19,17 @@ WORDPRESS_BASE_URL=
# Optional credentials (prefer an Application Password). Leave blank if your WP endpoints are public.
WORDPRESS_USERNAME=
WORDPRESS_APP_PASSWORD=
# Cache layer (optional; used by ingestion scripts)
# If unset, caching is disabled.
#
# Using docker-compose redis:
# CACHE_REDIS_URL=redis://localhost:6380/0
CACHE_REDIS_URL=
# Alternative config if you prefer host/port/db:
CACHE_REDIS_HOST=localhost
CACHE_REDIS_PORT=6380
CACHE_REDIS_DB=0
# Default cache TTL (seconds). 3600 = 1 hour.
CACHE_DEFAULT_TTL_SECONDS=3600

View File

@@ -1,5 +1,5 @@
{
"generatedAt": "2026-02-10T06:01:51.379Z",
"generatedAt": "2026-02-10T06:16:26.031Z",
"items": [
{
"id": "gPGbtfQdaw4",
@@ -31,7 +31,7 @@
"publishedAt": "2026-02-05T04:31:18.000Z",
"thumbnailUrl": "https://i.ytimg.com/vi/9t8cBpZLHUo/hqdefault.jpg",
"metrics": {
"views": 328
"views": 325
}
},
{

101
site/package-lock.json generated
View File

@@ -10,6 +10,7 @@
"dependencies": {
"@astrojs/sitemap": "^3.7.0",
"astro": "^5.17.1",
"redis": "^4.7.1",
"rss-parser": "^3.13.0",
"zod": "^3.25.76"
},
@@ -1241,6 +1242,65 @@
"integrity": "sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==",
"license": "MIT"
},
"node_modules/@redis/bloom": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
"integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/client": {
"version": "1.6.1",
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz",
"integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==",
"license": "MIT",
"dependencies": {
"cluster-key-slot": "1.1.2",
"generic-pool": "3.9.0",
"yallist": "4.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/@redis/graph": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz",
"integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/json": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz",
"integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/search": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz",
"integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/time-series": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz",
"integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@rollup/pluginutils": {
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.3.0.tgz",
@@ -2515,6 +2575,15 @@
"node": ">=6"
}
},
"node_modules/cluster-key-slot": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
"integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
"license": "Apache-2.0",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@@ -3090,6 +3159,15 @@
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/generic-pool": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
"integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
"license": "MIT",
"engines": {
"node": ">= 4"
}
},
"node_modules/get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
@@ -4687,6 +4765,23 @@
"url": "https://paulmillr.com/funding/"
}
},
"node_modules/redis": {
"version": "4.7.1",
"resolved": "https://registry.npmjs.org/redis/-/redis-4.7.1.tgz",
"integrity": "sha512-S1bJDnqLftzHXHP8JsT5II/CtHWQrASX5K96REjWjlmWKrviSOLWmM7QnRLstAWsu1VBBV1ffV6DzCvxNP0UJQ==",
"license": "MIT",
"workspaces": [
"./packages/*"
],
"dependencies": {
"@redis/bloom": "1.2.0",
"@redis/client": "1.6.1",
"@redis/graph": "1.1.1",
"@redis/json": "1.0.7",
"@redis/search": "1.2.0",
"@redis/time-series": "1.1.0"
}
},
"node_modules/regex": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/regex/-/regex-6.1.0.tgz",
@@ -6745,6 +6840,12 @@
"node": ">=10"
}
},
"node_modules/yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
"license": "ISC"
},
"node_modules/yaml": {
"version": "2.8.2",
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz",

View File

@@ -7,6 +7,7 @@
"build": "astro build",
"preview": "astro preview",
"fetch-content": "tsx scripts/fetch-content.ts",
"cache:clear": "tsx scripts/cache-clear.ts",
"verify:blog": "npm run build && tsx scripts/verify-blog-build.ts",
"typecheck": "astro check",
"format": "prettier -w .",
@@ -17,6 +18,7 @@
"dependencies": {
"@astrojs/sitemap": "^3.7.0",
"astro": "^5.17.1",
"redis": "^4.7.1",
"rss-parser": "^3.13.0",
"zod": "^3.25.76"
},

View File

@@ -0,0 +1,22 @@
import "dotenv/config";
import { createCacheFromEnv } from "../src/lib/cache";
function log(msg: string) {
// eslint-disable-next-line no-console
console.log(`[cache-clear] ${msg}`);
}
async function main() {
const cache = await createCacheFromEnv(process.env, { namespace: "fast-website", log });
await cache.flush();
await cache.close();
log("ok");
}
main().catch((e) => {
// eslint-disable-next-line no-console
console.error(`[cache-clear] failed: ${String(e)}`);
process.exitCode = 1;
});

View File

@@ -4,6 +4,8 @@ import { promises as fs } from "node:fs";
import path from "node:path";
import { getIngestConfigFromEnv } from "../src/lib/config";
import { createCacheFromEnv } from "../src/lib/cache";
import { cachedCompute } from "../src/lib/cache/memoize";
import type { ContentCache, ContentItem } from "../src/lib/content/types";
import { readInstagramEmbedPosts } from "../src/lib/ingest/instagram";
import { fetchPodcastRss } from "../src/lib/ingest/podcast";
@@ -42,6 +44,11 @@ async function main() {
const all: ContentItem[] = [];
const outPath = path.join(process.cwd(), "content", "cache", "content.json");
const kv = await createCacheFromEnv(process.env, {
namespace: "fast-website",
log,
});
// Read the existing cache so we can keep last-known-good sections if a source fails.
let existing: ContentCache | undefined;
try {
@@ -56,17 +63,29 @@ async function main() {
log("YouTube: skipped (missing YOUTUBE_CHANNEL_ID)");
} else if (cfg.youtubeApiKey) {
try {
const items = await fetchYoutubeViaApi(cfg.youtubeChannelId, cfg.youtubeApiKey, 25);
const cacheKey = `youtube:api:${cfg.youtubeChannelId}:25`;
const { value: items, cached } = await cachedCompute(kv, cacheKey, () =>
fetchYoutubeViaApi(cfg.youtubeChannelId!, cfg.youtubeApiKey!, 25),
);
log(`YouTube: API ${cached ? "cache" : "live"} (${items.length} items)`);
log(`YouTube: API ok (${items.length} items)`);
all.push(...items);
} catch (e) {
log(`YouTube: API failed (${String(e)}), falling back to RSS`);
const items = await fetchYoutubeViaRss(cfg.youtubeChannelId, 25);
const cacheKey = `youtube:rss:${cfg.youtubeChannelId}:25`;
const { value: items, cached } = await cachedCompute(kv, cacheKey, () =>
fetchYoutubeViaRss(cfg.youtubeChannelId!, 25),
);
log(`YouTube: RSS ${cached ? "cache" : "live"} (${items.length} items)`);
log(`YouTube: RSS ok (${items.length} items)`);
all.push(...items);
}
} else {
const items = await fetchYoutubeViaRss(cfg.youtubeChannelId, 25);
const cacheKey = `youtube:rss:${cfg.youtubeChannelId}:25`;
const { value: items, cached } = await cachedCompute(kv, cacheKey, () =>
fetchYoutubeViaRss(cfg.youtubeChannelId!, 25),
);
log(`YouTube: RSS ${cached ? "cache" : "live"} (${items.length} items)`);
log(`YouTube: RSS ok (${items.length} items)`);
all.push(...items);
}
@@ -76,7 +95,11 @@ async function main() {
log("Podcast: skipped (missing PODCAST_RSS_URL)");
} else {
try {
const items = await fetchPodcastRss(cfg.podcastRssUrl, 50);
const cacheKey = `podcast:rss:${cfg.podcastRssUrl}:50`;
const { value: items, cached } = await cachedCompute(kv, cacheKey, () =>
fetchPodcastRss(cfg.podcastRssUrl!, 50),
);
log(`Podcast: RSS ${cached ? "cache" : "live"} (${items.length} items)`);
log(`Podcast: RSS ok (${items.length} items)`);
all.push(...items);
} catch (e) {
@@ -103,11 +126,17 @@ async function main() {
wordpress = existing?.wordpress || wordpress;
} else {
try {
const wp = await fetchWordpressContent({
baseUrl: cfg.wordpressBaseUrl,
username: cfg.wordpressUsername,
appPassword: cfg.wordpressAppPassword,
});
const cacheKey = `wp:content:${cfg.wordpressBaseUrl}`;
const { value: wp, cached } = await cachedCompute(kv, cacheKey, () =>
fetchWordpressContent({
baseUrl: cfg.wordpressBaseUrl!,
username: cfg.wordpressUsername,
appPassword: cfg.wordpressAppPassword,
}),
);
log(
`WordPress: wp-json ${cached ? "cache" : "live"} (${wp.posts.length} posts, ${wp.pages.length} pages, ${wp.categories.length} categories)`,
);
wordpress = wp;
log(
`WordPress: wp-json ok (${wp.posts.length} posts, ${wp.pages.length} pages, ${wp.categories.length} categories)`,
@@ -119,14 +148,16 @@ async function main() {
}
}
const cache: ContentCache = {
const contentCache: ContentCache = {
generatedAt,
items: dedupe(all),
wordpress,
};
await writeAtomic(outPath, JSON.stringify(cache, null, 2));
log(`Wrote cache: ${outPath} (${cache.items.length} total items)`);
await writeAtomic(outPath, JSON.stringify(contentCache, null, 2));
log(`Wrote cache: ${outPath} (${contentCache.items.length} total items)`);
await kv.close();
}
main().catch((e) => {

28
site/src/lib/cache/index.ts vendored Normal file
View File

@@ -0,0 +1,28 @@
import type { CacheLogFn, CacheStore } from "./redis-cache";
import {
createRedisCache,
resolveDefaultTtlSecondsFromEnv,
resolveRedisUrlFromEnv,
} from "./redis-cache";
import { createNoopCache } from "./noop-cache";
export async function createCacheFromEnv(
env: NodeJS.ProcessEnv,
opts?: { namespace?: string; log?: CacheLogFn },
): Promise<CacheStore> {
const url = resolveRedisUrlFromEnv(env);
if (!url) return createNoopCache(opts?.log);
try {
return await createRedisCache({
url,
defaultTtlSeconds: resolveDefaultTtlSecondsFromEnv(env),
namespace: opts?.namespace,
log: opts?.log,
});
} catch (e) {
opts?.log?.(`cache: disabled (redis connect failed: ${String(e)})`);
return createNoopCache(opts?.log);
}
}

16
site/src/lib/cache/memoize.ts vendored Normal file
View File

@@ -0,0 +1,16 @@
import type { CacheStore } from "./redis-cache";
export async function cachedCompute<T>(
cache: CacheStore,
key: string,
compute: () => Promise<T>,
ttlSeconds?: number,
): Promise<{ value: T; cached: boolean }> {
const hit = await cache.getJson<T>(key);
if (hit !== undefined) return { value: hit, cached: true };
const value = await compute();
await cache.setJson(key, value, ttlSeconds);
return { value, cached: false };
}

49
site/src/lib/cache/memory-cache.ts vendored Normal file
View File

@@ -0,0 +1,49 @@
import type { CacheStore } from "./redis-cache";
type Entry = { value: string; expiresAt: number };
export function createMemoryCache(defaultTtlSeconds: number): CacheStore {
const store = new Map<string, Entry>();
function nowMs() {
return Date.now();
}
function isExpired(e: Entry) {
return e.expiresAt !== 0 && nowMs() > e.expiresAt;
}
return {
async getJson<T>(key: string) {
const e = store.get(key);
if (!e) return undefined;
if (isExpired(e)) {
store.delete(key);
return undefined;
}
try {
return JSON.parse(e.value) as T;
} catch {
store.delete(key);
return undefined;
}
},
async setJson(key: string, value: unknown, ttlSeconds?: number) {
const ttl = Math.max(1, Math.floor(ttlSeconds ?? defaultTtlSeconds));
store.set(key, {
value: JSON.stringify(value),
expiresAt: nowMs() + ttl * 1000,
});
},
async flush() {
store.clear();
},
async close() {
// no-op
},
};
}

19
site/src/lib/cache/noop-cache.ts vendored Normal file
View File

@@ -0,0 +1,19 @@
import type { CacheLogFn, CacheStore } from "./redis-cache";
export function createNoopCache(log?: CacheLogFn): CacheStore {
return {
async getJson() {
return undefined;
},
async setJson() {
// no-op
},
async flush() {
log?.("cache: noop flush");
},
async close() {
// no-op
},
};
}

92
site/src/lib/cache/redis-cache.ts vendored Normal file
View File

@@ -0,0 +1,92 @@
import { createClient } from "redis";
export type CacheLogFn = (msg: string) => void;
export type CacheStore = {
getJson<T>(key: string): Promise<T | undefined>;
setJson(key: string, value: unknown, ttlSeconds?: number): Promise<void>;
flush(): Promise<void>;
close(): Promise<void>;
};
type RedisCacheOptions = {
url: string;
defaultTtlSeconds: number;
namespace?: string;
log?: CacheLogFn;
};
function nsKey(namespace: string | undefined, key: string) {
return namespace ? `${namespace}:${key}` : key;
}
export async function createRedisCache(opts: RedisCacheOptions): Promise<CacheStore> {
const log = opts.log;
const client = createClient({ url: opts.url });
client.on("error", (err) => {
log?.(`cache: redis error (${String(err)})`);
});
await client.connect();
return {
async getJson<T>(key: string) {
const k = nsKey(opts.namespace, key);
const raw = await client.get(k);
if (raw == null) {
log?.(`cache: miss ${k}`);
return undefined;
}
log?.(`cache: hit ${k}`);
try {
return JSON.parse(raw) as T;
} catch {
// Bad cache entry: treat as miss.
return undefined;
}
},
async setJson(key: string, value: unknown, ttlSeconds?: number) {
const k = nsKey(opts.namespace, key);
const ttl = Math.max(1, Math.floor(ttlSeconds ?? opts.defaultTtlSeconds));
const raw = JSON.stringify(value);
await client.set(k, raw, { EX: ttl });
},
async flush() {
await client.flushDb();
},
async close() {
try {
await client.quit();
} catch {
// ignore
}
},
};
}
export function resolveRedisUrlFromEnv(env: NodeJS.ProcessEnv): string | undefined {
const url = env.CACHE_REDIS_URL;
if (url) return url;
const host = env.CACHE_REDIS_HOST;
const port = env.CACHE_REDIS_PORT;
const db = env.CACHE_REDIS_DB;
if (!host) return undefined;
const p = port ? Number(port) : 6379;
const d = db ? Number(db) : 0;
if (!Number.isFinite(p) || !Number.isFinite(d)) return undefined;
return `redis://${host}:${p}/${d}`;
}
export function resolveDefaultTtlSecondsFromEnv(env: NodeJS.ProcessEnv): number {
const raw = env.CACHE_DEFAULT_TTL_SECONDS;
const n = raw ? Number(raw) : NaN;
if (Number.isFinite(n) && n > 0) return Math.floor(n);
return 3600;
}

View File

@@ -14,6 +14,11 @@ type IngestConfig = {
wordpressBaseUrl?: string;
wordpressUsername?: string;
wordpressAppPassword?: string;
cacheRedisUrl?: string;
cacheRedisHost?: string;
cacheRedisPort?: number;
cacheRedisDb?: number;
cacheDefaultTtlSeconds?: number;
};
export function getPublicConfig(): PublicConfig {
@@ -37,5 +42,12 @@ export function getIngestConfigFromEnv(env: NodeJS.ProcessEnv): IngestConfig {
wordpressBaseUrl: env.WORDPRESS_BASE_URL,
wordpressUsername: env.WORDPRESS_USERNAME,
wordpressAppPassword: env.WORDPRESS_APP_PASSWORD,
cacheRedisUrl: env.CACHE_REDIS_URL,
cacheRedisHost: env.CACHE_REDIS_HOST,
cacheRedisPort: env.CACHE_REDIS_PORT ? Number(env.CACHE_REDIS_PORT) : undefined,
cacheRedisDb: env.CACHE_REDIS_DB ? Number(env.CACHE_REDIS_DB) : undefined,
cacheDefaultTtlSeconds: env.CACHE_DEFAULT_TTL_SECONDS
? Number(env.CACHE_DEFAULT_TTL_SECONDS)
: undefined,
};
}

View File

@@ -0,0 +1,40 @@
import { describe, expect, it } from "vitest";
import { createMemoryCache } from "../src/lib/cache/memory-cache";
import { cachedCompute } from "../src/lib/cache/memoize";
function sleep(ms: number) {
return new Promise((r) => setTimeout(r, ms));
}
describe("cache wrapper", () => {
it("set/get JSON and expires by TTL", async () => {
const cache = createMemoryCache(1);
await cache.setJson("k", { a: 1 }, 1);
const v1 = await cache.getJson<{ a: number }>("k");
expect(v1).toEqual({ a: 1 });
await sleep(1100);
const v2 = await cache.getJson("k");
expect(v2).toBeUndefined();
});
it("cachedCompute hits on second call within TTL", async () => {
const cache = createMemoryCache(60);
let calls = 0;
const compute = async () => {
calls++;
return { ok: true, n: calls };
};
const r1 = await cachedCompute(cache, "x", compute, 60);
const r2 = await cachedCompute(cache, "x", compute, 60);
expect(r1.cached).toBe(false);
expect(r2.cached).toBe(true);
expect(calls).toBe(1);
expect(r2.value).toEqual(r1.value);
});
});