127 lines
4.6 KiB
Python
127 lines
4.6 KiB
Python
"""Covers the flow behind the '📥 استخراج ۲۰ پست' button, which used to abort silently."""
|
||
import asyncio
|
||
import sys
|
||
import types
|
||
|
||
sys.path.insert(0, "/app")
|
||
|
||
from db.database import init_db, close_db_pool
|
||
from db.repository import Repository
|
||
from services.collector import CollectorService
|
||
|
||
SOURCE_ID = -1009999000010
|
||
|
||
|
||
class FakeMessage:
|
||
def __init__(self, msg_id, text):
|
||
self.id = msg_id
|
||
self.raw_text = text
|
||
self.media = None
|
||
|
||
async def download_media(self, file=None):
|
||
return None
|
||
|
||
|
||
class FakeClient:
|
||
"""Minimal stand-in for the Telethon client used by the collector."""
|
||
|
||
def __init__(self, messages):
|
||
self._messages = messages
|
||
|
||
def is_connected(self):
|
||
return True
|
||
|
||
async def is_user_authorized(self):
|
||
return True
|
||
|
||
async def get_entity(self, ident):
|
||
return types.SimpleNamespace(id=abs(SOURCE_ID), title="Scrape Source")
|
||
|
||
def iter_messages(self, entity, limit=20):
|
||
async def gen():
|
||
for m in self._messages[:limit]:
|
||
yield m
|
||
return gen()
|
||
|
||
|
||
async def _cleanup(repo):
|
||
pool = await repo._get_pool()
|
||
async with pool.acquire() as conn:
|
||
await conn.execute("DELETE FROM posts WHERE source_channel_id = $1;", SOURCE_ID)
|
||
await conn.execute("DELETE FROM sources WHERE channel_id = $1;", SOURCE_ID)
|
||
# The failure-path assertion logs a real error row; don't leave it behind.
|
||
await conn.execute(
|
||
"DELETE FROM error_logs WHERE context->>'channel_id' = $1;", str(SOURCE_ID))
|
||
|
||
|
||
async def run_tests():
|
||
await init_db()
|
||
repo = Repository()
|
||
await _cleanup(repo)
|
||
await repo.add_source(SOURCE_ID, "Scrape Source", None)
|
||
|
||
reviewed = []
|
||
collector = CollectorService(repo=repo, on_post_received=lambda pid: _record(reviewed, pid))
|
||
# 20 distinct messages plus one exact repeat of the first, to exercise dedup.
|
||
messages = [FakeMessage(1000 + i, f"historical post number {i}") for i in range(20)]
|
||
messages.append(FakeMessage(1099, "historical post number 0"))
|
||
collector.client = FakeClient(messages)
|
||
|
||
progress = []
|
||
|
||
async def on_progress(text):
|
||
progress.append(text)
|
||
|
||
res = await collector.scrape_channel_history(SOURCE_ID, limit=21, progress_callback=on_progress)
|
||
|
||
assert res.collected == 21, f"expected 21 posts collected, got {res.collected}"
|
||
assert res.scanned == 21, f"expected 21 messages scanned, got {res.scanned}"
|
||
assert res.duplicates == 1, f"expected 1 duplicate, got {res.duplicates}"
|
||
assert res.already_stored == 0, f"expected 0 already-stored, got {res.already_stored}"
|
||
assert len(reviewed) == 21, f"expected 21 review cards dispatched, got {len(reviewed)}"
|
||
assert progress and progress[-1].startswith("✅"), f"admin was not told the result: {progress}"
|
||
|
||
# Re-running the same window adds nothing new, and the report must say WHY
|
||
# rather than looking like a dead button.
|
||
progress.clear()
|
||
again = await collector.scrape_channel_history(SOURCE_ID, limit=21, progress_callback=on_progress)
|
||
assert again.collected == 0, f"repeat scrape should add nothing, got {again.collected}"
|
||
assert again.already_stored == 21, f"expected 21 already-stored, got {again.already_stored}"
|
||
assert progress[-1].startswith("ℹ️"), f"repeat scrape must be explained: {progress[-1]}"
|
||
assert "قبلا ذخیره شده" in progress[-1]
|
||
|
||
stored = await repo.get_posts_by_status("pending_review", limit=100)
|
||
mine = [p for p in stored if p.source_channel_id == SOURCE_ID]
|
||
assert len(mine) == 21, f"expected 21 stored posts, got {len(mine)}"
|
||
|
||
# The repeated text must be flagged against the original rather than stored blind.
|
||
dupes = [p for p in mine if p.is_duplicate]
|
||
assert len(dupes) == 1, f"expected exactly 1 duplicate flagged, got {len(dupes)}"
|
||
assert dupes[0].duplicate_of_id is not None
|
||
|
||
# A channel that cannot be resolved reports the failure instead of vanishing.
|
||
class BrokenClient(FakeClient):
|
||
async def get_entity(self, ident):
|
||
raise RuntimeError("channel not reachable")
|
||
|
||
async def get_dialogs(self, limit=50):
|
||
return []
|
||
|
||
collector.client = BrokenClient([])
|
||
progress.clear()
|
||
result = await collector.scrape_channel_history(SOURCE_ID, limit=5, progress_callback=on_progress)
|
||
assert result.collected == 0 and result.error
|
||
assert progress and progress[-1].startswith("❌"), f"failure was not reported: {progress}"
|
||
|
||
await _cleanup(repo)
|
||
await close_db_pool()
|
||
print("All scrape-history tests passed successfully!")
|
||
|
||
|
||
async def _record(bucket, post_id):
|
||
bucket.append(post_id)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(run_tests())
|