102 lines
3.1 KiB
TypeScript
102 lines
3.1 KiB
TypeScript
import "dotenv/config";
|
|
|
|
import { promises as fs } from "node:fs";
|
|
import path from "node:path";
|
|
|
|
import { getIngestConfigFromEnv } from "../src/lib/config";
|
|
import type { ContentCache, ContentItem } from "../src/lib/content/types";
|
|
import { readInstagramEmbedPosts } from "../src/lib/ingest/instagram";
|
|
import { fetchPodcastRss } from "../src/lib/ingest/podcast";
|
|
import { fetchYoutubeViaApi, fetchYoutubeViaRss } from "../src/lib/ingest/youtube";
|
|
|
|
function log(msg: string) {
|
|
// simple, cron-friendly logs
|
|
// eslint-disable-next-line no-console
|
|
console.log(`[fetch-content] ${msg}`);
|
|
}
|
|
|
|
async function writeAtomic(filePath: string, content: string) {
|
|
const tmpPath = `${filePath}.tmp`;
|
|
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
await fs.writeFile(tmpPath, content, "utf8");
|
|
await fs.rename(tmpPath, filePath);
|
|
}
|
|
|
|
function dedupe(items: ContentItem[]): ContentItem[] {
|
|
const seen = new Set<string>();
|
|
const out: ContentItem[] = [];
|
|
for (const it of items) {
|
|
const k = `${it.source}:${it.id}`;
|
|
if (seen.has(k)) continue;
|
|
seen.add(k);
|
|
out.push(it);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
async function main() {
|
|
const cfg = getIngestConfigFromEnv(process.env);
|
|
const generatedAt = new Date().toISOString();
|
|
|
|
const all: ContentItem[] = [];
|
|
|
|
// YouTube
|
|
if (!cfg.youtubeChannelId) {
|
|
log("YouTube: skipped (missing YOUTUBE_CHANNEL_ID)");
|
|
} else if (cfg.youtubeApiKey) {
|
|
try {
|
|
const items = await fetchYoutubeViaApi(cfg.youtubeChannelId, cfg.youtubeApiKey, 25);
|
|
log(`YouTube: API ok (${items.length} items)`);
|
|
all.push(...items);
|
|
} catch (e) {
|
|
log(`YouTube: API failed (${String(e)}), falling back to RSS`);
|
|
const items = await fetchYoutubeViaRss(cfg.youtubeChannelId, 25);
|
|
log(`YouTube: RSS ok (${items.length} items)`);
|
|
all.push(...items);
|
|
}
|
|
} else {
|
|
const items = await fetchYoutubeViaRss(cfg.youtubeChannelId, 25);
|
|
log(`YouTube: RSS ok (${items.length} items)`);
|
|
all.push(...items);
|
|
}
|
|
|
|
// Podcast
|
|
if (!cfg.podcastRssUrl) {
|
|
log("Podcast: skipped (missing PODCAST_RSS_URL)");
|
|
} else {
|
|
try {
|
|
const items = await fetchPodcastRss(cfg.podcastRssUrl, 50);
|
|
log(`Podcast: RSS ok (${items.length} items)`);
|
|
all.push(...items);
|
|
} catch (e) {
|
|
log(`Podcast: RSS failed (${String(e)})`);
|
|
}
|
|
}
|
|
|
|
// Instagram (embed-first list)
|
|
try {
|
|
const filePath = path.isAbsolute(cfg.instagramPostUrlsFile)
|
|
? cfg.instagramPostUrlsFile
|
|
: path.join(process.cwd(), cfg.instagramPostUrlsFile);
|
|
const items = await readInstagramEmbedPosts(filePath);
|
|
log(`Instagram: embed list ok (${items.length} items)`);
|
|
all.push(...items);
|
|
} catch (e) {
|
|
log(`Instagram: embed list failed (${String(e)})`);
|
|
}
|
|
|
|
const cache: ContentCache = {
|
|
generatedAt,
|
|
items: dedupe(all),
|
|
};
|
|
|
|
const outPath = path.join(process.cwd(), "content", "cache", "content.json");
|
|
await writeAtomic(outPath, JSON.stringify(cache, null, 2));
|
|
log(`Wrote cache: ${outPath} (${cache.items.length} total items)`);
|
|
}
|
|
|
|
main().catch((e) => {
|
|
log(`fatal: ${String(e)}`);
|
|
process.exitCode = 1;
|
|
});
|