import "dotenv/config"; import { promises as fs } from "node:fs"; import path from "node:path"; import { getIngestConfigFromEnv } from "../src/lib/config"; import type { ContentCache, ContentItem } from "../src/lib/content/types"; import { readInstagramEmbedPosts } from "../src/lib/ingest/instagram"; import { fetchPodcastRss } from "../src/lib/ingest/podcast"; import { fetchWordpressContent } from "../src/lib/ingest/wordpress"; import { fetchYoutubeViaApi, fetchYoutubeViaRss } from "../src/lib/ingest/youtube"; function log(msg: string) { // simple, cron-friendly logs // eslint-disable-next-line no-console console.log(`[fetch-content] ${msg}`); } async function writeAtomic(filePath: string, content: string) { const tmpPath = `${filePath}.tmp`; await fs.mkdir(path.dirname(filePath), { recursive: true }); await fs.writeFile(tmpPath, content, "utf8"); await fs.rename(tmpPath, filePath); } function dedupe(items: ContentItem[]): ContentItem[] { const seen = new Set(); const out: ContentItem[] = []; for (const it of items) { const k = `${it.source}:${it.id}`; if (seen.has(k)) continue; seen.add(k); out.push(it); } return out; } async function main() { const cfg = getIngestConfigFromEnv(process.env); const generatedAt = new Date().toISOString(); const all: ContentItem[] = []; const outPath = path.join(process.cwd(), "content", "cache", "content.json"); // Read the existing cache so we can keep last-known-good sections if a source fails. let existing: ContentCache | undefined; try { const raw = await fs.readFile(outPath, "utf8"); existing = JSON.parse(raw) as ContentCache; } catch { existing = undefined; } // YouTube if (!cfg.youtubeChannelId) { log("YouTube: skipped (missing YOUTUBE_CHANNEL_ID)"); } else if (cfg.youtubeApiKey) { try { const items = await fetchYoutubeViaApi(cfg.youtubeChannelId, cfg.youtubeApiKey, 25); log(`YouTube: API ok (${items.length} items)`); all.push(...items); } catch (e) { log(`YouTube: API failed (${String(e)}), falling back to RSS`); const items = await fetchYoutubeViaRss(cfg.youtubeChannelId, 25); log(`YouTube: RSS ok (${items.length} items)`); all.push(...items); } } else { const items = await fetchYoutubeViaRss(cfg.youtubeChannelId, 25); log(`YouTube: RSS ok (${items.length} items)`); all.push(...items); } // Podcast if (!cfg.podcastRssUrl) { log("Podcast: skipped (missing PODCAST_RSS_URL)"); } else { try { const items = await fetchPodcastRss(cfg.podcastRssUrl, 50); log(`Podcast: RSS ok (${items.length} items)`); all.push(...items); } catch (e) { log(`Podcast: RSS failed (${String(e)})`); } } // Instagram (embed-first list) try { const filePath = path.isAbsolute(cfg.instagramPostUrlsFile) ? cfg.instagramPostUrlsFile : path.join(process.cwd(), cfg.instagramPostUrlsFile); const items = await readInstagramEmbedPosts(filePath); log(`Instagram: embed list ok (${items.length} items)`); all.push(...items); } catch (e) { log(`Instagram: embed list failed (${String(e)})`); } // WordPress (optional; powers /blog) let wordpress: ContentCache["wordpress"] = { posts: [], pages: [], categories: [] }; if (!cfg.wordpressBaseUrl) { log("WordPress: skipped (missing WORDPRESS_BASE_URL)"); wordpress = existing?.wordpress || wordpress; } else { try { const wp = await fetchWordpressContent({ baseUrl: cfg.wordpressBaseUrl, username: cfg.wordpressUsername, appPassword: cfg.wordpressAppPassword, }); wordpress = wp; log( `WordPress: wp-json ok (${wp.posts.length} posts, ${wp.pages.length} pages, ${wp.categories.length} categories)`, ); } catch (e) { log(`WordPress: wp-json failed (${String(e)})`); // Keep last-known-good WP content if present, otherwise fall back to empty. wordpress = existing?.wordpress || wordpress; } } const cache: ContentCache = { generatedAt, items: dedupe(all), wordpress, }; await writeAtomic(outPath, JSON.stringify(cache, null, 2)); log(`Wrote cache: ${outPath} (${cache.items.length} total items)`); } main().catch((e) => { log(`fatal: ${String(e)}`); process.exitCode = 1; });