Initial commit - but way too late.

2026-02-10 00:22:18 -05:00
commit af112a713c
173 changed files with 27667 additions and 0 deletions
--- a/site/scripts/fetch-content.ts
+++ b/site/scripts/fetch-content.ts
@@ -0,0 +1,101 @@
+import "dotenv/config";
+
+import { promises as fs } from "node:fs";
+import path from "node:path";
+
+import { getIngestConfigFromEnv } from "../src/lib/config";
+import type { ContentCache, ContentItem } from "../src/lib/content/types";
+import { readInstagramEmbedPosts } from "../src/lib/ingest/instagram";
+import { fetchPodcastRss } from "../src/lib/ingest/podcast";
+import { fetchYoutubeViaApi, fetchYoutubeViaRss } from "../src/lib/ingest/youtube";
+
+function log(msg: string) {
+  // simple, cron-friendly logs
+  // eslint-disable-next-line no-console
+  console.log(`[fetch-content] ${msg}`);
+}
+
+async function writeAtomic(filePath: string, content: string) {
+  const tmpPath = `${filePath}.tmp`;
+  await fs.mkdir(path.dirname(filePath), { recursive: true });
+  await fs.writeFile(tmpPath, content, "utf8");
+  await fs.rename(tmpPath, filePath);
+}
+
+function dedupe(items: ContentItem[]): ContentItem[] {
+  const seen = new Set<string>();
+  const out: ContentItem[] = [];
+  for (const it of items) {
+    const k = `${it.source}:${it.id}`;
+    if (seen.has(k)) continue;
+    seen.add(k);
+    out.push(it);
+  }
+  return out;
+}
+
+async function main() {
+  const cfg = getIngestConfigFromEnv(process.env);
+  const generatedAt = new Date().toISOString();
+
+  const all: ContentItem[] = [];
+
+  // YouTube
+  if (!cfg.youtubeChannelId) {
+    log("YouTube: skipped (missing YOUTUBE_CHANNEL_ID)");
+  } else if (cfg.youtubeApiKey) {
+    try {
+      const items = await fetchYoutubeViaApi(cfg.youtubeChannelId, cfg.youtubeApiKey, 25);
+      log(`YouTube: API ok (${items.length} items)`);
+      all.push(...items);
+    } catch (e) {
+      log(`YouTube: API failed (${String(e)}), falling back to RSS`);
+      const items = await fetchYoutubeViaRss(cfg.youtubeChannelId, 25);
+      log(`YouTube: RSS ok (${items.length} items)`);
+      all.push(...items);
+    }
+  } else {
+    const items = await fetchYoutubeViaRss(cfg.youtubeChannelId, 25);
+    log(`YouTube: RSS ok (${items.length} items)`);
+    all.push(...items);
+  }
+
+  // Podcast
+  if (!cfg.podcastRssUrl) {
+    log("Podcast: skipped (missing PODCAST_RSS_URL)");
+  } else {
+    try {
+      const items = await fetchPodcastRss(cfg.podcastRssUrl, 50);
+      log(`Podcast: RSS ok (${items.length} items)`);
+      all.push(...items);
+    } catch (e) {
+      log(`Podcast: RSS failed (${String(e)})`);
+    }
+  }
+
+  // Instagram (embed-first list)
+  try {
+    const filePath = path.isAbsolute(cfg.instagramPostUrlsFile)
+      ? cfg.instagramPostUrlsFile
+      : path.join(process.cwd(), cfg.instagramPostUrlsFile);
+    const items = await readInstagramEmbedPosts(filePath);
+    log(`Instagram: embed list ok (${items.length} items)`);
+    all.push(...items);
+  } catch (e) {
+    log(`Instagram: embed list failed (${String(e)})`);
+  }
+
+  const cache: ContentCache = {
+    generatedAt,
+    items: dedupe(all),
+  };
+
+  const outPath = path.join(process.cwd(), "content", "cache", "content.json");
+  await writeAtomic(outPath, JSON.stringify(cache, null, 2));
+  log(`Wrote cache: ${outPath} (${cache.items.length} total items)`);
+}
+
+main().catch((e) => {
+  log(`fatal: ${String(e)}`);
+  process.exitCode = 1;
+});