santhoshj/clawfort
1
0
Fork 0
Code Issues 64 Pull Requests Actions Packages Projects Releases Wiki Activity

p08-seo-tweaks

Browse Source
This commit is contained in:
Santhosh Janardhanan
2026-02-13 00:49:22 -05:00
parent a1da041f14
commit 88a5540b7d
63 changed files with 2228 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

229
backend/news_service.py
View File

@@ -5,6 +5,7 @@ import logging
import os
import time
from io import BytesIO
from urllib.parse import quote_plus
import httpx
from PIL import Image
@@ -95,7 +96,12 @@ async def call_openrouter_api(query: str) -> dict | None:
async def call_perplexity_translation_api(
headline: str, summary: str, language: str
headline: str,
summary: str,
language: str,
tldr_points: list[str] | None = None,
summary_body: str | None = None,
source_citation: str | None = None,
) -> dict | None:
headers = {
"Authorization": f"Bearer {config.PERPLEXITY_API_KEY}",
@@ -119,6 +125,9 @@ async def call_perplexity_translation_api(
"target_language": language,
"headline": headline,
"summary": summary,
"tldr_points": tldr_points or [],
"summary_body": summary_body or "",
"source_citation": source_citation or "",
}
),
},
@@ -144,13 +153,31 @@ def parse_translation_response(response: dict) -> dict | None:
headline = str(parsed.get("headline", "")).strip()
summary = str(parsed.get("summary", "")).strip()
if headline and summary:
return {"headline": headline, "summary": summary}
tldr_points = parsed.get("tldr_points", [])
if not isinstance(tldr_points, list):
tldr_points = []
cleaned_points = [str(p).strip() for p in tldr_points if str(p).strip()]
summary_body = str(parsed.get("summary_body", "")).strip() or None
source_citation = str(parsed.get("source_citation", "")).strip() or None
return {
"headline": headline,
"summary": summary,
"tldr_points": cleaned_points,
"summary_body": summary_body,
"source_citation": source_citation,
}
except json.JSONDecodeError:
logger.error("Failed to parse translation response: %s", content[:200])
return None
async def generate_translations(headline: str, summary: str) -> dict[str, dict]:
async def generate_translations(
headline: str,
summary: str,
tldr_points: list[str] | None = None,
summary_body: str | None = None,
source_citation: str | None = None,
) -> dict[str, dict]:
translations: dict[str, dict] = {}
language_names = {"ta": "Tamil", "ml": "Malayalam"}
@@ -159,7 +186,14 @@ async def generate_translations(headline: str, summary: str) -> dict[str, dict]:
for language_code, language_name in language_names.items():
try:
response = await call_perplexity_translation_api(headline, summary, language_name)
response = await call_perplexity_translation_api(
headline=headline,
summary=summary,
language=language_name,
tldr_points=tldr_points,
summary_body=summary_body,
source_citation=source_citation,
)
if response:
parsed = parse_translation_response(response)
if parsed:
@@ -170,6 +204,146 @@ async def generate_translations(headline: str, summary: str) -> dict[str, dict]:
return translations
async def call_perplexity_summary_api(
headline: str, summary: str, source_url: str | None
) -> dict | None:
headers = {
"Authorization": f"Bearer {config.PERPLEXITY_API_KEY}",
"Content-Type": "application/json",
}
payload = {
"model": config.PERPLEXITY_MODEL,
"messages": [
{
"role": "system",
"content": (
"Generate concise structured JSON for UI modal. Return only JSON object with keys: "
"tldr_points (array of 3 short bullets), summary_body (detailed summary), "
"source_citation (concise source/citation text). "
"Always summarize from provided text only. No markdown."
),
},
{
"role": "user",
"content": json.dumps(
{
"headline": headline,
"summary": summary,
"source_citation": source_url or "Original source",
"summary_length_scale": config.SUMMARY_LENGTH_SCALE,
"summary_length_rule": (
"1=very short, 2=short, 3=medium, 4=long, 5=very long. "
"Use more detail as scale increases."
),
}
),
},
],
"temperature": 0.2,
}
async with httpx.AsyncClient(timeout=30.0) as client:
response = await client.post(config.PERPLEXITY_API_URL, headers=headers, json=payload)
response.raise_for_status()
return response.json()
def parse_summary_response(response: dict) -> dict | None:
content = response.get("choices", [{}])[0].get("message", {}).get("content", "")
content = content.strip()
if content.startswith("```"):
content = content.split("\n", 1)[-1].rsplit("```", 1)[0]
try:
parsed = json.loads(content)
except json.JSONDecodeError:
logger.error("Failed to parse summary response: %s", content[:200])
return None
if not isinstance(parsed, dict):
return None
tldr_points = parsed.get("tldr_points", [])
if not isinstance(tldr_points, list):
tldr_points = []
cleaned_points = [str(point).strip() for point in tldr_points if str(point).strip()]
summary_body = str(parsed.get("summary_body", "")).strip()
source_citation = str(parsed.get("source_citation", "")).strip()
if not cleaned_points and not summary_body:
return None
return {
"tldr_points": cleaned_points[:5],
"summary_body": summary_body or None,
"source_citation": source_citation or None,
}
def build_fallback_summary(summary: str, source_url: str | None) -> dict:
segments = [
s.strip() for s in summary.replace("!", ".").replace("?", ".").split(".") if s.strip()
]
points = segments[:3]
if not points and summary.strip():
points = [summary.strip()[:180]]
return {
"tldr_points": points,
"summary_body": summary,
"source_citation": source_url or "Original source",
}
async def fetch_royalty_free_image(query: str) -> tuple[str | None, str | None]:
if config.ROYALTY_IMAGE_MCP_ENDPOINT:
try:
async with httpx.AsyncClient(timeout=15.0) as client:
response = await client.post(
config.ROYALTY_IMAGE_MCP_ENDPOINT,
json={"query": query},
)
response.raise_for_status()
payload = response.json()
image_url = payload.get("image_url") or payload.get("url")
image_credit = payload.get("image_credit") or payload.get("credit")
if image_url:
return str(image_url), str(image_credit or "Royalty-free")
except Exception:
logger.exception("MCP image retrieval failed")
if config.ROYALTY_IMAGE_PROVIDER.lower() == "wikimedia":
try:
encoded_query = quote_plus(query[:120])
search_url = (
"https://commons.wikimedia.org/w/api.php"
"?action=query&format=json&generator=search&gsrnamespace=6&gsrlimit=1"
f"&gsrsearch={encoded_query}&prop=imageinfo&iiprop=url"
)
async with httpx.AsyncClient(
timeout=15.0,
headers={"User-Agent": "ClawFortBot/1.0 (news image enrichment)"},
) as client:
response = await client.get(search_url)
response.raise_for_status()
data = response.json()
pages = data.get("query", {}).get("pages", {})
if pages:
first_page = next(iter(pages.values()))
infos = first_page.get("imageinfo", [])
if infos:
url = infos[0].get("url")
if url:
return str(url), "Wikimedia Commons"
except Exception:
logger.exception("Wikimedia image retrieval failed")
if config.ROYALTY_IMAGE_PROVIDER.lower() == "picsum":
seed = hashlib.md5(query.encode("utf-8")).hexdigest()[:12]
return f"https://picsum.photos/seed/{seed}/1200/630", "Picsum Photos"
return None, None
def parse_news_response(response: dict) -> list[dict]:
content = response.get("choices", [{}])[0].get("message", {}).get("content", "")
content = content.strip()
@@ -261,6 +435,37 @@ async def process_and_store_news() -> int:
local_image = await download_and_optimize_image(item.get("image_url", ""))
image_url = local_image or PLACEHOLDER_IMAGE_PATH
summary_artifact: dict | None = None
if config.PERPLEXITY_API_KEY:
try:
summary_response = await call_perplexity_summary_api(
headline=headline,
summary=summary,
source_url=item.get("source_url"),
)
if summary_response:
summary_artifact = parse_summary_response(summary_response)
except Exception:
logger.exception("Summary generation failed for article: %s", headline[:80])
if summary_artifact is None:
summary_artifact = build_fallback_summary(summary, item.get("source_url"))
summary_image_url, summary_image_credit = await fetch_royalty_free_image(headline)
summary_local_image = None
if summary_image_url:
summary_local_image = await download_and_optimize_image(summary_image_url)
if summary_local_image:
summary_image_url = summary_local_image
if not summary_image_url:
summary_image_url = image_url
if not summary_image_credit:
summary_image_credit = item.get("image_credit")
tldr_points = summary_artifact.get("tldr_points") if summary_artifact else None
summary_body = summary_artifact.get("summary_body") if summary_artifact else None
source_citation = summary_artifact.get("source_citation") if summary_artifact else None
created_news_item = create_news(
db=db,
headline=headline,
@@ -268,9 +473,20 @@ async def process_and_store_news() -> int:
source_url=item.get("source_url"),
image_url=image_url,
image_credit=item.get("image_credit"),
tldr_points=tldr_points,
summary_body=summary_body,
source_citation=source_citation,
summary_image_url=summary_image_url,
summary_image_credit=summary_image_credit,
)
translations = await generate_translations(headline, summary)
translations = await generate_translations(
headline=headline,
summary=summary,
tldr_points=tldr_points,
summary_body=summary_body,
source_citation=source_citation,
)
for language_code, payload in translations.items():
if translation_exists(db, created_news_item.id, language_code):
continue
@@ -280,6 +496,9 @@ async def process_and_store_news() -> int:
language=language_code,
headline=payload["headline"],
summary=payload["summary"],
tldr_points=payload.get("tldr_points"),
summary_body=payload.get("summary_body"),
source_citation=payload.get("source_citation"),
)
stored += 1