import sys

sys.path.insert(0, "/app")

from core.dedup import normalize_text, compute_content_hash

def test_normalize_text():
    raw = "  Check out this link: https://t.me/example! @admin #tech news...  "
    norm = normalize_text(raw)
    assert "https" not in norm
    assert "admin" not in norm
    assert "tech" not in norm
    assert norm == "check out this link news"

def test_content_hash():
    text1 = "Breaking News: Bitcoin hits $100k! Check https://example.com"
    text2 = "Breaking News: Bitcoin hits $100k! Check https://other.com"
    hash1 = compute_content_hash(text1)
    hash2 = compute_content_hash(text2)
    # Both normalize to the same text after link removal
    assert hash1 == hash2

if __name__ == "__main__":
    test_normalize_text()
    test_content_hash()
    print("All deduplication unit tests passed!")