36 lines
1.3 KiB
Python
36 lines
1.3 KiB
Python
import hashlib
|
|
import re
|
|
from typing import Optional
|
|
|
|
def normalize_text(text: Optional[str]) -> str:
|
|
"""Normalize text by removing URLs, telegram handles, hashtags, excessive punctuation/whitespace, and lowercasing."""
|
|
if not text:
|
|
return ""
|
|
# Remove URLs
|
|
text = re.sub(r'https?://\S+|www\.\S+', '', text)
|
|
# Remove Telegram @mentions / hashtags
|
|
text = re.sub(r'[@#]\w+', '', text)
|
|
# Normalize punctuation and whitespace
|
|
text = re.sub(r'[^\w\s]', ' ', text)
|
|
text = re.sub(r'\s+', ' ', text)
|
|
return text.strip().lower()
|
|
|
|
def compute_content_hash(text: Optional[str], media_hash: Optional[str] = None) -> Optional[str]:
|
|
"""Generate SHA256 hash from normalized text and/or media hash."""
|
|
norm_text = normalize_text(text)
|
|
if not norm_text and not media_hash:
|
|
return None
|
|
raw_key = f"{norm_text}|{media_hash or ''}"
|
|
return hashlib.sha256(raw_key.encode('utf-8')).hexdigest()
|
|
|
|
def compute_file_hash(file_path: str) -> Optional[str]:
|
|
"""Generate SHA256 hash of a media file."""
|
|
try:
|
|
hasher = hashlib.sha256()
|
|
with open(file_path, 'rb') as f:
|
|
while chunk := f.read(65536):
|
|
hasher.update(chunk)
|
|
return hasher.hexdigest()
|
|
except Exception:
|
|
return None
|