139 lines
5.6 KiB
Python
139 lines
5.6 KiB
Python
import logging
|
|
from typing import List, Optional, Dict, Any
|
|
from db.models import Post, TargetChannel
|
|
from db.repository import Repository
|
|
from core.llm import LLMClient
|
|
from core.dedup import compute_content_hash
|
|
from core.metrics import DUPLICATES_DETECTED_TOTAL
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
TAG_EXTRACTION_SYSTEM_PROMPT = """
|
|
You are an AI news analyst and classifier.
|
|
Given a social media/channel post, extract:
|
|
1. "subject": A brief, specific headline/subject (3-8 words).
|
|
2. "tags": A JSON array of 3 to 6 lowercase keywords/topics/entities (e.g. ["ai", "nvidia", "gpus", "hardware"]).
|
|
Respond ONLY in JSON format:
|
|
{
|
|
"subject": "...",
|
|
"tags": ["tag1", "tag2", "tag3"]
|
|
}
|
|
"""
|
|
|
|
DUPLICATE_CHECK_SYSTEM_PROMPT = """
|
|
You are an expert news editor checking for duplicate news stories.
|
|
Given a NEW POST and a list of PREVIOUS POSTS, determine if the NEW POST is covering the same exact event, news item, or story as any of the previous posts.
|
|
|
|
Respond ONLY in JSON format:
|
|
{
|
|
"is_duplicate": true/false,
|
|
"duplicate_of_id": <id of matched previous post or null>,
|
|
"similarity_reason": "<short explanation of why it is or is not a duplicate>"
|
|
}
|
|
"""
|
|
|
|
POST_REWRITE_SYSTEM_PROMPT = """
|
|
You are an expert Telegram content creator and copywriter.
|
|
Rewrite the provided post to make it engaging, well-formatted, professional, and clear.
|
|
Use appropriate emojis, clear paragraphs, and markdown formatting.
|
|
Remove any original promotional links, author credits, or watermarks.
|
|
|
|
Respond ONLY in JSON format:
|
|
{
|
|
"ai_text": "...",
|
|
"suggested_target_id": <optional id of best matching target channel or null>
|
|
}
|
|
"""
|
|
|
|
class AIProcessor:
|
|
def __init__(self, repo: Repository, llm: Optional[LLMClient] = None):
|
|
self.repo = repo
|
|
self.llm = llm or LLMClient()
|
|
|
|
async def process_post(self, post_id: int) -> Optional[Post]:
|
|
post = await self.repo.get_post_by_id(post_id)
|
|
if not post or not post.raw_text:
|
|
return post
|
|
|
|
raw_text = post.raw_text
|
|
is_dup = False
|
|
dup_of_id = None
|
|
sim_reason = None
|
|
|
|
# 1. Exact hash duplicate check
|
|
content_hash = post.content_hash or compute_content_hash(raw_text)
|
|
if content_hash:
|
|
exact_dup = await self.repo.find_duplicate_post_by_hash(content_hash)
|
|
if exact_dup and exact_dup.id != post.id:
|
|
is_dup = True
|
|
dup_of_id = exact_dup.id
|
|
sim_reason = "Exact match on normalized text/media hash"
|
|
DUPLICATES_DETECTED_TOTAL.labels(method="hash").inc()
|
|
|
|
# 2. Extract Tags and Subject via AI
|
|
tags = []
|
|
subject = "General News"
|
|
try:
|
|
tag_res = await self.llm.generate_json(
|
|
prompt=f"Post content:\n\n{raw_text}",
|
|
system_prompt=TAG_EXTRACTION_SYSTEM_PROMPT,
|
|
action_name="extract_tags"
|
|
)
|
|
subject = tag_res.get("subject", subject)
|
|
tags = [t.lower().strip() for t in tag_res.get("tags", []) if isinstance(t, str)]
|
|
await self.repo.update_post_tags(post.id, tags, subject)
|
|
except Exception as e:
|
|
logger.error(f"Tag extraction failed for post {post.id}: {e}")
|
|
|
|
# 3. Candidate search & Semantic AI Deduplication check (if not already exact dup)
|
|
if not is_dup and tags:
|
|
candidates = await self.repo.find_candidate_posts_by_tags(tags, exclude_post_id=post.id, hours_lookback=72, limit=5)
|
|
if candidates:
|
|
cand_texts = "\n---\n".join([f"ID {c.id} (Subject: {c.subject}):\n{c.raw_text}" for c in candidates if c.raw_text])
|
|
prompt = f"NEW POST:\n{raw_text}\n\nPREVIOUS CANDIDATE POSTS:\n{cand_texts}"
|
|
try:
|
|
dup_res = await self.llm.generate_json(
|
|
prompt=prompt,
|
|
system_prompt=DUPLICATE_CHECK_SYSTEM_PROMPT,
|
|
action_name="check_duplicate"
|
|
)
|
|
if dup_res.get("is_duplicate"):
|
|
is_dup = True
|
|
dup_of_id = dup_res.get("duplicate_of_id")
|
|
sim_reason = dup_res.get("similarity_reason", "AI detected duplicate news topic")
|
|
DUPLICATES_DETECTED_TOTAL.labels(method="ai_semantic").inc()
|
|
except Exception as e:
|
|
logger.error(f"Semantic duplicate check failed for post {post.id}: {e}")
|
|
|
|
# 4. Rewrite post for our channels
|
|
ai_text = raw_text
|
|
suggested_target_id = None
|
|
targets = await self.repo.get_active_targets()
|
|
target_info = "\n".join([f"Target ID {t.id}: {t.title} (@{t.username or 'none'})" for t in targets])
|
|
rewrite_prompt = f"TARGET CHANNELS AVAILABLE:\n{target_info or 'None'}\n\nORIGINAL POST:\n{raw_text}"
|
|
|
|
try:
|
|
rewrite_res = await self.llm.generate_json(
|
|
prompt=rewrite_prompt,
|
|
system_prompt=POST_REWRITE_SYSTEM_PROMPT,
|
|
action_name="rewrite_post"
|
|
)
|
|
ai_text = rewrite_res.get("ai_text", raw_text)
|
|
suggested_target_id = rewrite_res.get("suggested_target_id")
|
|
except Exception as e:
|
|
logger.error(f"Post rewrite failed for post {post.id}: {e}")
|
|
|
|
# 5. Save AI results into database
|
|
await self.repo.update_ai_result(
|
|
post_id=post.id,
|
|
subject=subject,
|
|
ai_text=ai_text,
|
|
tags=tags,
|
|
suggested_target_id=suggested_target_id,
|
|
is_duplicate=is_dup,
|
|
duplicate_of_id=dup_of_id,
|
|
similarity_reason=sim_reason,
|
|
)
|
|
|
|
return await self.repo.get_post_by_id(post.id)
|