#!/usr/bin/env python3 from __future__ import annotations import argparse, fcntl, json, os, sqlite3, traceback, tempfile, shutil, time from datetime import datetime, timedelta, timezone from pathlib import Path from urllib.parse import urljoin, urlparse import requests from bs4 import BeautifulSoup from PIL import Image, ImageChops, ImageOps from browser_fetch import HybridBrowser from dbutil import setting, setting_int, log_event from discover import fetch_page, web_search, UA from image_utils import cache_product_image, safe_image_get, IMAGE_DIR from matching import confidence, product_query, validate_product_match, color_matches, norm, tokens, COLOR_ALIASES DB=os.environ.get('FIYAT_RADAR_DB','/var/lib/fiyat-radar/fiyat-radar.sqlite') LOCK=os.environ.get('FIYAT_RADAR_IMAGE_LOCK','/var/lib/fiyat-radar/image-fetch.lock') TRIGGER=Path(os.environ.get('FIYAT_RADAR_IMAGE_TRIGGER','/var/lib/fiyat-radar/image.trigger')) # Görsel kimliği kapasiteden bağımsızdır: aynı model+renk farklı depolama # kapasitelerinde aynı kasayı/ürün fotoğrafını kullanır. Bu nedenle görsel # doğrulaması model varyantını ve rengi sert, kapasiteyi ise görmezden gelir. MODEL_SIBLING_MODIFIERS=( 'pro max','pro','plus','air','ultra','max','mini','fe','5g','4g','lte','se' ) VISUAL_STOP_TOKENS={'telefon','akilli','cep','urun','gb','tb','ram','hafiza','depolama'} def _strip_color_tokens(text): n=' '+norm(text)+' ' phrases=[] for key,aliases in COLOR_ALIASES.items(): phrases.append(norm(key)) phrases.extend(norm(x) for x in aliases) for phrase in sorted({x for x in phrases if x}, key=len, reverse=True): n=n.replace(' '+phrase+' ',' ') return ' '.join(n.split()) def _visual_expected_model(product): # model kirliyse (örn. "iphone 16 pro gold") ürün adını da aday olarak # değerlendir; renk sözcüklerini model kimliğinden temizle. vals=[] for key in ('model','name','product_name'): v=str(product.get(key) or '').strip() if v: clean=_strip_color_tokens(v) if clean and clean not in vals: vals.append(clean) return vals def _visual_text_match(product,text): """Görsel için model+renk doğrulaması; kapasite bilinçli olarak yok sayılır.""" n=' '+norm(text)+' ' brand_tokens=[t for t in tokens(product.get('brand','')) if t not in VISUAL_STOP_TOKENS] if brand_tokens and not any((' '+t+' ') in n for t in brand_tokens): raise ValueError('Görsel kanıtında marka eşleşmedi') models=_visual_expected_model(product) if not models: raise ValueError('Görsel doğrulaması için model bilgisi yok') matched_model=None for model in models: mt=[t for t in tokens(model) if t not in VISUAL_STOP_TOKENS] if mt and all((' '+t+' ') in n for t in mt): matched_model=model;break if not matched_model: raise ValueError('Görsel kanıtında tam model eşleşmedi') # Kardeş model karışmasını engelle: iPhone 17 görseli, iPhone 17 Pro/Plus/Air # sonucu ile; Galaxy A17 ise A17 5G sonucu ile doğrulanamaz. expected=' '+norm(matched_model)+' ' for mod in MODEL_SIBLING_MODIFIERS: phrase=' '+norm(mod)+' ' exp=phrase in expected got=phrase in n if exp != got: # "max" zaten "pro max" içinde olabilir; beklenen uzun değiştirici # eşleşiyorsa tek sözcük alt parçası yüzünden yeniden reddetme. if mod=='max' and ' pro max ' in expected and ' pro max ' in n: continue raise ValueError(f'Kardeş model ayrımı uyuşmadı: {mod}') color=str(product.get('color') or '').strip() if color and not color_matches(color,text): raise ValueError(f'Görsel kanıtında renk eşleşmedi: {color}') return True def _visual_confidence(product,text,base=.82): # Genel confidence kapasite puanı beklediği için görselde ayrı puan kullan. n=' '+norm(text)+' ' model_tokens=[] models=_visual_expected_model(product) if models:model_tokens=[t for t in tokens(models[0]) if t not in VISUAL_STOP_TOKENS] hit=sum(1 for t in model_tokens if (' '+t+' ') in n) model_score=(hit/max(1,len(model_tokens)))*0.70 brand_score=.10 if any((' '+t+' ') in n for t in tokens(product.get('brand',''))) else 0 color_score=.18 if (not product.get('color') or color_matches(str(product.get('color')),text)) else 0 return min(.995,max(base,model_score+brand_score+color_score)) def _image_query(product,with_capacity=False): brand=str(product.get('brand') or '').strip() model=_visual_expected_model(product)[0] if _visual_expected_model(product) else str(product.get('model') or product.get('name') or '').strip() color=str(product.get('color') or '').strip() parts=[x for x in (brand,model) if x] if color:parts.append(f'"{color}"') if with_capacity and product.get('capacity_gb'): parts.append(f'"{int(product["capacity_gb"])} GB"') return ' '.join(parts).strip() def _image_queries(product): queries=[_image_query(product,with_capacity=False)] color_n=norm(str(product.get('color') or '')) aliases=COLOR_ALIASES.get(color_n,[]) for alias in aliases: if norm(alias)==color_n:continue qproduct=dict(product);qproduct['color']=alias q=_image_query(qproduct,with_capacity=False) if q and q not in queries:queries.append(q) if len(queries)>=2:break return queries def _reuse_verified_sibling(c,product): family=str(product.get('family_key') or '').strip();color=norm(str(product.get('color') or '')) if not family or not color:return None rows=c.execute("SELECT * FROM products WHERE id<>? AND active=1 AND family_key=? AND condition_type=? AND image_path IS NOT NULL AND image_path<>'' AND image_locked=1 ORDER BY image_provider_rank DESC,image_match_confidence DESC,image_updated_at DESC LIMIT 20",(int(product['id']),family,product.get('condition_type') or 'new')).fetchall() for row in rows: r=dict(row) if norm(str(r.get('color') or ''))!=color:continue src=IMAGE_DIR/Path(str(r.get('image_path') or '')).name if not src.is_file():continue # Aynı family+renk fiziksel üründür; kapasite farklı olsa da aynı doğrulanmış dosyayı paylaş. c.execute("UPDATE products SET image_path=?,image_source_url=?,image_source_page_url=?,image_source_kind='consensus:sibling-shared',image_match_confidence=?,image_status='ready',image_locked=1,image_verified_at=COALESCE(?,CURRENT_TIMESTAMP),image_provider_rank=?,image_checked_at=CURRENT_TIMESTAMP,image_updated_at=CURRENT_TIMESTAMP WHERE id=?", (r.get('image_path'),r.get('image_source_url'),r.get('image_source_page_url'),float(r.get('image_match_confidence') or .99),r.get('image_verified_at'),int(r.get('image_provider_rank') or 90),int(product['id']))) c.execute("UPDATE image_fetch_requests SET status='done',finished_at=CURRENT_TIMESTAMP,next_attempt_at=NULL,image_url=?,source_page_url=?,source_kind='consensus:sibling-shared',confidence=?,last_error=NULL,note=? WHERE product_id=?", (r.get('image_source_url'),r.get('image_source_page_url'),float(r.get('image_match_confidence') or .99),f"Aynı model+renkte kilitli ürün #{int(r['id'])} görseli paylaşıldı; kapasite görsel kimliğini değiştirmez",int(product['id']))) b=c.execute("SELECT id,status,price_count,source_count FROM product_bootstrap_requests WHERE product_id=? ORDER BY id DESC LIMIT 1",(int(product['id']),)).fetchone() if b and int(b['price_count'] or 0)>0: c.execute("UPDATE product_bootstrap_requests SET status='done',image_ready=1,finished_at=CURRENT_TIMESTAMP,note=? WHERE id=?",(f"Hazır: {int(b['source_count'] or 0)} kaynak, {int(b['price_count'] or 0)} fiyat, doğrulanmış family+renk görseli paylaşıldı.",b['id'])) elif b:c.execute("UPDATE product_bootstrap_requests SET image_ready=1 WHERE id=?",(b['id'],)) return {'file':r.get('image_path'),'sibling_id':int(r['id']),'confidence':float(r.get('image_match_confidence') or .99),'shared':True} return None def _propagate_verified_image(c,product,filename,image_url,page_url,source_kind,confidence): family=str(product.get('family_key') or '').strip();color=norm(str(product.get('color') or '')) if not family or not color:return 0 host=_host(page_url or image_url);rank=100 if 'hepsiburada.com' in host else 90 rows=c.execute("SELECT id,color FROM products WHERE id<>? AND active=1 AND family_key=? AND condition_type=?",(int(product['id']),family,product.get('condition_type') or 'new')).fetchall();count=0 for r in rows: if norm(str(r['color'] or ''))!=color:continue c.execute("UPDATE products SET image_path=?,image_source_url=?,image_source_page_url=?,image_source_kind=?,image_match_confidence=?,image_status='ready',image_locked=1,image_verified_at=CURRENT_TIMESTAMP,image_provider_rank=?,image_checked_at=CURRENT_TIMESTAMP,image_updated_at=CURRENT_TIMESTAMP WHERE id=?", (filename,image_url,page_url,'consensus:family-color-shared',float(confidence),rank,int(r['id']))) c.execute("UPDATE image_fetch_requests SET status='done',finished_at=CURRENT_TIMESTAMP,next_attempt_at=NULL,image_url=?,source_page_url=?,source_kind='consensus:family-color-shared',confidence=?,last_error=NULL,note='Family+renk görseli tek doğrulamayla paylaşıldı' WHERE product_id=?", (image_url,page_url,float(confidence),int(r['id']))) count+=1 return count def _candidate_source_rank(cand): image_url,page_url,kind,score,text,probe_url=_candidate_parts(cand) host=_host(page_url or image_url) if 'hepsiburada.com' in host:return 120 # üretici/resmî kaynaklar ikinci öncelik; diğer doğrulanmış mağazalar sonra. if any(x in host for x in ('apple.com','samsung.com','mi.com','xiaomi.com','huawei.com','honor.com','oppo.com','realme.com','motorola.com','oneplus.com','google.com')):return 105 return 75 def _candidate_parts(cand): # v0.2.3 adayı: original_url,page_url,kind,score,text,probe_url if len(cand)>=6:return cand[0],cand[1],cand[2],cand[3],cand[4],cand[5] return cand[0],cand[1],cand[2],cand[3],cand[4],cand[0] BAD_IMAGE_HOST_PARTS=( "amazon.", "media-amazon.", "ssl-images-amazon.", "images-na.ssl-images-amazon.", "m.media-amazon.", "images-eu.ssl-images-amazon." ) def _host(url): try:return (urlparse(url or "").hostname or "").lower() except Exception:return "" def _evidence_domain(url): h=_host(url) if not h:return "" parts=h.split(".") if len(parts)>=3 and parts[-2] in {"com","net","org","gen"} and parts[-1] in {"tr","uk","au"}: return ".".join(parts[-3:]) return ".".join(parts[-2:]) if len(parts)>=2 else h def _amazonish(*urls): return any(any(part in _host(u) for part in BAD_IMAGE_HOST_PARTS) for u in urls if u) def _flat_pixels(im): # Pillow 14'te getdata() kaldırılacak. Yeni API varsa onu kullan. fn=getattr(im,'get_flattened_data',None) return list(fn()) if callable(fn) else list(im.getdata()) def _dhash_image(im,size=16): g=ImageOps.autocontrast(im.convert("L")).resize((size+1,size),Image.Resampling.LANCZOS) px=_flat_pixels(g);bits=0 for y in range(size): row=y*(size+1) for x in range(size):bits=(bits<<1) | (1 if px[row+x] > px[row+x+1] else 0) return bits def _ahash_image(im,size=16): g=ImageOps.autocontrast(im.convert("L")).resize((size,size),Image.Resampling.LANCZOS) px=_flat_pixels(g);avg=sum(px)/max(1,len(px));bits=0 for v in px:bits=(bits<<1) | (1 if v>=avg else 0) return bits def _rgb_on_white(im): if im.mode in ('RGBA','LA') or ('transparency' in im.info): rgba=im.convert('RGBA');bg=Image.new('RGBA',rgba.size,(255,255,255,255));bg.alpha_composite(rgba);return bg.convert('RGB') return im.convert('RGB') def _trim_background(im): """Beyaz/tek renk e-ticaret arka planını kırpar; kırpma başarısızsa orijinali döndürür.""" rgb=_rgb_on_white(im) w,h=rgb.size if w<20 or h<20:return rgb pts=[rgb.getpixel((1,1)),rgb.getpixel((w-2,1)),rgb.getpixel((1,h-2)),rgb.getpixel((w-2,h-2))] bgc=tuple(sorted(p[i] for p in pts)[len(pts)//2] for i in range(3)) bg=Image.new('RGB',rgb.size,bgc) diff=ImageChops.difference(rgb,bg).convert('L') # JPEG gürültüsünü/çok hafif gölgeyi arka plan sanma. mask=diff.point(lambda v:255 if v>18 else 0) bbox=mask.getbbox() if not bbox:return rgb l,t,r,b=bbox bw,bh=r-l,b-t # Aşırı küçük bir logo/ikon bbox'ını ürün gövdesi sanma. if bw*bh < w*h*0.10:return rgb pad=max(2,int(max(bw,bh)*0.04)) l=max(0,l-pad);t=max(0,t-pad);r=min(w,r+pad);b=min(h,b+pad) return rgb.crop((l,t,r,b)) def _fingerprints_from_file(path,size=16): with Image.open(path) as src: full=_rgb_on_white(src) trim=_trim_background(full) # İç kırpım, aynı katalog görselinin farklı web kırpmalarına karşı ikinci sinyal. tw,th=trim.size mx=max(0,int(tw*.035));my=max(0,int(th*.035)) inner=trim.crop((mx,my,max(mx+1,tw-mx),max(my+1,th-my))) if tw>40 and th>40 else trim return { 'd_full':_dhash_image(full,size), 'd_trim':_dhash_image(trim,size), 'd_inner':_dhash_image(inner,size), 'a_trim':_ahash_image(trim,size), } def _hash_distance(a,b): return int((int(a)^int(b)).bit_count()) def _fingerprint_distance(a,b): """Aynı ürün render'ının kırpma/arka plan varyasyonlarını toleranslı karşılaştır.""" ah=a.get('hashes') or {};bh=b.get('hashes') or {} ds={k:_hash_distance(ah[k],bh[k]) for k in ('d_full','d_trim','d_inner','a_trim') if k in ah and k in bh} return ds def _fingerprints_match(a,b,base_dist): ds=_fingerprint_distance(a,b) if not ds:return False,ds # 256 bit hashlerde yaklaşık %11-%15 fark; tam görselde daha sıkı, # arka planı kırpılmış parmak izinde daha toleranslı davran. full=max(18,int(base_dist)-8) trim=max(28,int(base_dist)) loose=max(trim+10,40) tight=(ds.get('d_full',999)<=full or ds.get('d_trim',999)<=trim or ds.get('d_inner',999)<=trim) corroborated=(ds.get('d_trim',999)<=loose and ds.get('a_trim',999)<=max(30,trim)) return bool(tight or corroborated),ds def _probe_image(session,image_url,page_url=None,min_width=300,min_height=300): r,final=safe_image_get(session,image_url,page_url);tmp=None try: fd,tmp=tempfile.mkstemp(prefix="fiyat-radar-img-",suffix=".bin");os.close(fd);total=0 with open(tmp,"wb") as f: for chunk in r.iter_content(65536): if not chunk:continue total+=len(chunk) if total>8*1024*1024:raise ValueError("görsel çok büyük") f.write(chunk) with Image.open(tmp) as im: w,h=im.size if w=2:continue if not allow_amazon and _amazonish(image_url,page_url): stats['amazon']+=1;_record_evidence(c,product['id'],cand,'rejected',None,'Amazon görseli sıkı modda kanıt sayılmaz');continue try:_visual_text_match(product,text) except Exception as e: stats['text_reject']+=1;_record_evidence(c,product['id'],cand,'rejected',None,str(e));continue domain_counts[domain]=domain_counts.get(domain,0)+1 prefilter.append(cand) # Kaynak çeşitliliği öncelikli; aynı domaini ardışık probe etme. prefilter.sort(key=lambda x:(-_candidate_source_rank(x),-float(_candidate_parts(x)[3]))) for cand in prefilter[:max_probe]: image_url,page_url,kind,score,text,probe_url=_candidate_parts(cand) domain=_evidence_domain(page_url or image_url) try: probe=_probe_image(session,probe_url or image_url,page_url,min_w,min_w) item={'candidate':cand,'domain':domain,'original_url':image_url,**probe};checked.append(item);stats['technical']+=1 _record_evidence(c,product['id'],cand,'checked',probe,'Tam model+renk ve görsel teknik doğrulaması geçti (kapasite görsel kimliğine dahil değildir)') except Exception as e: stats['probe_reject']+=1;_record_evidence(c,product['id'],cand,'rejected',None,f'Görsel teknik kontrolü: {e}');continue # Greedy yerine benzerlik grafiği: A↔B ve B↔C ise aynı render ailesinde tutulur. # Bu yaklaşım farklı CDN kırpımlarında sıra bağımlılığını ortadan kaldırır. graph={i:set() for i in range(len(checked))};pair_samples=[] for i in range(len(checked)): for j in range(i+1,len(checked)): ok,ds=_fingerprints_match(checked[i],checked[j],max_dist) if ok: graph[i].add(j);graph[j].add(i) if len(pair_samples)<20:pair_samples.append({'a':checked[i]['domain'],'b':checked[j]['domain'],'distance':ds}) clusters=[];visited=set() for i in range(len(checked)): if i in visited:continue stack=[i];visited.add(i);idx=[] while stack: n=stack.pop();idx.append(n) for nxt in graph[n]: if nxt not in visited:visited.add(nxt);stack.append(nxt) clusters.append([checked[k] for k in idx]) official_map={'apple':'apple.com','samsung':'samsung.com','xiaomi':'mi.com','huawei':'huawei.com','honor':'honor.com','oppo':'oppo.com','realme':'realme.com','motorola':'motorola.com','oneplus':'oneplus.com','google':'google.com'} official=official_map.get(norm(str(product.get('brand') or ''))) qualified=[] cluster_diag=[] for cl in clusters: domains={x['domain'] for x in cl} avg=sum(float(_candidate_parts(x['candidate'])[3]) for x in cl)/len(cl) official_hit=1 if official and any(d==official or d.endswith('.'+official) for d in domains) else 0 hepsi_hit=1 if any('hepsiburada.com' in d for d in domains) else 0 area=max((x['width']*x['height'] for x in cl),default=0) cluster_diag.append({'domains':sorted(domains),'evidence':len(cl),'avg_confidence':round(avg,4),'official':bool(official_hit),'hepsiburada':bool(hepsi_hit)}) if len(domains)>=min_sources: qualified.append((hepsi_hit,official_hit,len(domains),len(cl),avg,area,cl)) qualified.sort(key=lambda x:(x[0],x[1],x[2],x[3],x[4],x[5]),reverse=True) if not qualified: raise ValueError('Görsel mutabakatı oluşmadı: ' f"ham={stats['raw']}, teknik={stats['technical']}, kümeler={len(clusters)}, " f"metin-red={stats['text_reject']}, amazon-red={stats['amazon']}, " f"indirme-red={stats['probe_reject']}, gerekli bağımsız kaynak={min_sources}") winner=qualified[0];cl=winner[6] # Birden fazla bağımsız doğrulanmış küme genellikle aynı doğru ürünün farklı # ön/arka/yan render'larıdır. Hepsi exact model+renk metninden geçtiği ve her # biri >=2 bağımsız domain tarafından doğrulandığı için hata değildir; en # güçlü/otoriter kümeyi seç ve durumu teşhis verisine yaz. multiple_valid=len(qualified)>1 best=max(cl,key=lambda x:(1 if 'hepsiburada.com' in x['domain'] else 0,float(_candidate_parts(x['candidate'])[3]),x['width']*x['height'])) for x in cl:_record_evidence(c,product['id'],x['candidate'],'accepted',x,'Bağımsız kaynak görsel mutabakatı (çoklu parmak izi)') return best['candidate'],{'domains':sorted({x['domain'] for x in cl}),'evidence_count':len(cl),'independent_sources':winner[2], 'hash':str(best['hash']),'hashes':{k:str(v) for k,v in (best.get('hashes') or {}).items()}, 'stats':stats,'clusters':cluster_diag,'pair_samples':pair_samples,'multiple_valid_views':multiple_valid} def now(): return datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S') def connect(): c=sqlite3.connect(DB,timeout=30);c.row_factory=sqlite3.Row c.execute('PRAGMA foreign_keys=ON');c.execute('PRAGMA busy_timeout=30000');c.execute('PRAGMA synchronous=NORMAL') return c def queue_product(c,product_id,priority=100,note='Ürün görseli zorunlu olarak hazırlanacak'): # v0.3.10: eski sürümlerde status=running iken started_at NULL kalan kayıtlar # sonsuza kadar kilitlenebiliyordu. Yalnız gerçekten son 4 dakikada başlamış # çalışan işi koru; diğer running kayıtları pending'e döndür. c.execute('''INSERT INTO image_fetch_requests(product_id,status,priority,attempts,requested_at,next_attempt_at,note) VALUES(?,'pending',?,0,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,?) ON CONFLICT(product_id) DO UPDATE SET status=CASE WHEN image_fetch_requests.status='running' AND image_fetch_requests.started_at IS NOT NULL AND image_fetch_requests.started_at >= datetime('now','-4 minutes') THEN 'running' ELSE 'pending' END, started_at=CASE WHEN image_fetch_requests.status='running' AND image_fetch_requests.started_at IS NOT NULL AND image_fetch_requests.started_at >= datetime('now','-4 minutes') THEN image_fetch_requests.started_at ELSE NULL END, priority=MAX(image_fetch_requests.priority,excluded.priority), next_attempt_at=CASE WHEN image_fetch_requests.status='running' AND image_fetch_requests.started_at IS NOT NULL AND image_fetch_requests.started_at >= datetime('now','-4 minutes') THEN image_fetch_requests.next_attempt_at ELSE CURRENT_TIMESTAMP END, note=excluded.note''',(int(product_id),int(priority),note)) def queue_missing(limit=500): c=connect();rows=c.execute("SELECT id,image_path FROM products WHERE active=1 ORDER BY id DESC LIMIT ?",(limit,)).fetchall();refreshed=0 for r in rows: path=str(r['image_path'] or '').strip() physical=bool(path and (IMAGE_DIR/Path(path).name).is_file()) if physical: continue broken=bool(path);priority=450 if broken else 300 if broken: # DB referansı var ama dosya yoksa bunu hazır saymak en tehlikeli kapak hatasıdır. c.execute("UPDATE products SET image_path=NULL,image_locked=0,image_status='pending',image_checked_at=CURRENT_TIMESTAMP,image_updated_at=CURRENT_TIMESTAMP WHERE id=?",(int(r['id']),)) c.execute("""INSERT INTO image_fetch_requests(product_id,status,priority,attempts,requested_at,next_attempt_at,note) VALUES(?,'pending',?,0,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,?) ON CONFLICT(product_id) DO UPDATE SET status=CASE WHEN image_fetch_requests.status='running' THEN 'running' ELSE 'pending' END, priority=MAX(image_fetch_requests.priority,excluded.priority), next_attempt_at=CASE WHEN image_fetch_requests.status='running' THEN image_fetch_requests.next_attempt_at ELSE CURRENT_TIMESTAMP END, last_error=CASE WHEN image_fetch_requests.status='running' THEN image_fetch_requests.last_error ELSE NULL END, note=excluded.note""", (int(r['id']),priority,'Fiziksel kapak dosyası kayıp; öncelikli otomatik kurtarma')) else: # Normal görselsiz üründe mevcut retry/backoff'u bozma. Yalnız hiç kuyruk yoksa # ekle veya eski done kaydı yanlışlıkla görselsiz kalmışsa yeniden aç. c.execute("""INSERT INTO image_fetch_requests(product_id,status,priority,attempts,requested_at,next_attempt_at,note) VALUES(?,'pending',?,0,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,?) ON CONFLICT(product_id) DO UPDATE SET status=CASE WHEN image_fetch_requests.status='done' THEN 'pending' ELSE image_fetch_requests.status END, priority=MAX(image_fetch_requests.priority,excluded.priority), next_attempt_at=CASE WHEN image_fetch_requests.status='done' THEN CURRENT_TIMESTAMP ELSE image_fetch_requests.next_attempt_at END, note=CASE WHEN image_fetch_requests.status='done' THEN excluded.note ELSE image_fetch_requests.note END""", (int(r['id']),priority,'Fiziksel kapak dosyası yok; otomatik kurtarma kuyruğu')) refreshed+=1 c.commit();c.close() # Worker zaten timer/path tarafından çalıştırılmış durumda. Burada trigger'a yeniden # dokunmak eksik kapak kaldığı sürece sonsuz hızlı path döngüsü oluşturabilir. return refreshed def claim_next(): c=connect() try: row=c.execute('''UPDATE image_fetch_requests SET status='running',started_at=CURRENT_TIMESTAMP,attempts=attempts+1,last_error=NULL WHERE id=(SELECT id FROM image_fetch_requests WHERE status IN ('pending','retry') AND (next_attempt_at IS NULL OR next_attempt_at<=CURRENT_TIMESTAMP) ORDER BY priority DESC,requested_at ASC LIMIT 1) AND status IN ('pending','retry') RETURNING *''').fetchone() if row:c.execute("UPDATE products SET image_status='running',image_checked_at=CURRENT_TIMESTAMP WHERE id=?",(row['product_id'],)) c.commit();return dict(row) if row else None finally:c.close() def _json_images(v): out=[] if isinstance(v,str):out.append(v) elif isinstance(v,list): for x in v:out.extend(_json_images(x)) elif isinstance(v,dict): for k in ('url','contentUrl','thumbnailUrl'):out.extend(_json_images(v.get(k))) return out def _json_product_candidates(obj,base,product,out,context_text=''): if isinstance(obj,list): for x in obj:_json_product_candidates(x,base,product,out,context_text) return if not isinstance(obj,dict):return typ=obj.get('@type') types=[str(x).lower() for x in typ] if isinstance(typ,list) else [str(typ).lower()] if 'product' in types: bits=[] for k in ('name','description','color','size','sku','mpn','model'): v=obj.get(k) if isinstance(v,(str,int,float)):bits.append(str(v)) ap=obj.get('additionalProperty') if isinstance(ap,dict):ap=[ap] if isinstance(ap,list): for prop in ap: if isinstance(prop,dict):bits.extend(str(prop.get(k) or '') for k in ('name','value','description')) # Bazı mağazalar JSON-LD Product.name içinde rengi yazmaz ama sayfa başlığı, # canonical URL veya meta açıklamasında seçili renk açıkça bulunur. Doğrulama # için bu sayfa bağlamını Product nesnesine ekle; kapasite yine görsel kimliği değildir. text=' '.join(bits+[context_text]) try: _visual_text_match(product,text) score=max(.92,min(.995,_visual_confidence(product,text,.92))) for u in _json_images(obj.get('image')): if u:out.append((urljoin(base,u),base,'jsonld-product',score,text[:1800],urljoin(base,u))) except Exception:pass for v in obj.values():_json_product_candidates(v,base,product,out,context_text) def _page_context(soup,base): title=soup.title.get_text(' ',strip=True) if soup.title else '' desc='' md=soup.find('meta',attrs={'name':'description'}) or soup.find('meta',attrs={'property':'og:description'}) if md:desc=md.get('content') or '' canon=soup.find('link',attrs={'rel':'canonical'}) canonical=canon.get('href') if canon else '' selected=[] for tag in soup.find_all(attrs={'aria-selected':'true'},limit=12):selected.append(tag.get_text(' ',strip=True)) for tag in soup.select('[class*="selected"], [class*="active"]')[:20]:selected.append(tag.get_text(' ',strip=True)) return ' '.join(x for x in (title,desc,canonical,base,' '.join(selected)) if x).strip() def page_image_candidates(html,base,product): soup=BeautifulSoup(html,'lxml');out=[];page_text=_page_context(soup,base) # 1) JSON-LD Product.image: sayfa bağlamı ile model+renk doğrulanır. for script in soup.find_all('script',attrs={'type':'application/ld+json'}): raw=script.string or script.get_text('',strip=True) if not raw:continue try:_json_product_candidates(json.loads(raw),base,product,out,page_text) except Exception:continue # 2) OG/Twitter görseli: yalnız sayfa bağlamı exact model+renk doğrulamasından geçerse. try: _visual_text_match(product,page_text) score=max(.86,min(.98,_visual_confidence(product,page_text,.86))) for attrs in ({'property':'og:image'},{'property':'og:image:secure_url'},{'name':'twitter:image'},{'itemprop':'image'}): tag=soup.find('meta',attrs=attrs) u=tag.get('content') if tag else None if u:out.append((urljoin(base,u),base,'page-meta',score,page_text[:1800],urljoin(base,u))) except Exception:pass # 3) Ürün galerisindeki alt metni exact eşleşen resimler. for im in soup.find_all('img',limit=160): alt=' '.join(str(im.get(x) or '') for x in ('alt','title','data-alt')) if not alt.strip():continue evidence=(alt+' '+page_text).strip() try:_visual_text_match(product,evidence) except Exception:continue u=im.get('data-zoom-image') or im.get('data-large-image') or im.get('data-src') or im.get('data-original') or im.get('data-lazy-src') or im.get('src') if u and not str(u).startswith('data:'): out.append((urljoin(base,u),base,'img-gallery',max(.84,min(.97,_visual_confidence(product,evidence,.84))),evidence[:1800],urljoin(base,u))) seen=set();clean=[] for item in sorted(out,key=lambda x:x[3],reverse=True): if item[0] in seen:continue seen.add(item[0]);clean.append(item) return clean def official_variant_links(html,base,product,limit=4): """Resmî katalog aile sayfasından exact model+renk varyant sayfalarını çıkar.""" soup=BeautifulSoup(html,'lxml');host=(urlparse(base).hostname or '').lower();out=[];seen=set() for a in soup.find_all('a',href=True,limit=400): u=urljoin(base,a.get('href'));ph=(urlparse(u).hostname or '').lower() if not ph or (host and ph!=host and not ph.endswith('.'+host)):continue text=' '.join([a.get_text(' ',strip=True),a.get('title') or '',a.get('aria-label') or '',u]) try:_visual_text_match(product,text) except Exception:continue if u in seen:continue seen.add(u);out.append(u) if len(out)>=limit:break return out def page_urls(c,product_id): # v0.3.16: Görsel keşfi arama API'sinden değil, gerçek ürün/aday sayfalarından yapılır. rows=c.execute('''SELECT x.url,x.kind,x.merchant_id,m.domain FROM ( SELECT url,'source' kind,merchant_id FROM product_sources WHERE product_id=? AND active=1 AND condition_status='verified' UNION ALL SELECT url,'candidate' kind,merchant_id FROM discovery_candidates WHERE product_id=? AND status IN ('pending','approved','auto_approved') UNION ALL SELECT source_url,'admin-variant-evidence' kind,NULL merchant_id FROM variant_proposals WHERE created_product_id=? AND status='approved' AND source_url IS NOT NULL AND source_url<>'' ) x LEFT JOIN merchants m ON m.id=x.merchant_id ORDER BY CASE WHEN lower(COALESCE(m.domain,'')) LIKE '%hepsiburada.com%' THEN 0 ELSE 1 END, CASE x.kind WHEN 'source' THEN 0 WHEN 'admin-variant-evidence' THEN 1 ELSE 2 END''',(product_id,product_id,product_id)).fetchall() out=[];seen=set() for r in rows: u=(r['url'] or '').strip() if u and u not in seen:seen.add(u);out.append((u,r['kind'],r['merchant_id'])) # Resmî katalog sayfası yalnız ek kanıttır; ürün sayfası doğrulamasını geçmek zorunda. try: p=c.execute("SELECT family_key FROM products WHERE id=?",(product_id,)).fetchone() if p and p['family_key']: v=c.execute("SELECT source_url FROM variant_catalogs WHERE family_key=? AND source_url IS NOT NULL AND source_url<>'' LIMIT 1",(p['family_key'],)).fetchone() if v and v['source_url'] not in seen: out.append((v['source_url'],'official-catalog',None)) except Exception:pass return out[:24] def _strict_image_query(product): # Görsel aramasında depolama kapasitesi görseli değiştirmez; model+renk aranır. return _image_query(product,with_capacity=False) def _web_page_candidates(c,session,browser,product,run_id): # Yardımcı HTML araması sağlıklıysa birkaç ek ürün sayfası denenir; çalışmıyorsa sessizce atlanır. try:results,engine=web_search(session,product,limit=6,c=c,run_id=run_id) except Exception:return [] out=[] for result in results[:6]: u=result.get('url') if not u:continue try: html,final,_=fetch_page(session,u,browser,headers={'User-Agent':UA,'Accept-Language':'tr-TR,tr;q=0.9,en;q=0.7'},run_id=run_id,c=c,product_id=product['id'],context='image.web') out.extend(page_image_candidates(html,final,product)) if out:break except Exception:continue return out def fetch_product_image(product_id,*,force=False,run_id=None,browser_pages=None): c=connect();p=c.execute('SELECT * FROM products WHERE id=? AND active=1',(int(product_id),)).fetchone() if not p:c.close();raise ValueError('Ürün bulunamadı veya pasif') product=dict(p) # v0.3.10: image_path var diye doğrulamadan erken çıkmak image_fetch_requests # kaydını running bırakıyordu. Sadece daha önce kilitlenmiş/doğrulanmış görseli # hazır kabul et; unlocked görsel yeniden doğrulama akışına girsin. if product.get('image_path') and int(product.get('image_locked') or 0)==1 and not force: physical=IMAGE_DIR/Path(str(product.get('image_path') or '')).name if physical.is_file(): c.execute("UPDATE image_fetch_requests SET status='done',finished_at=CURRENT_TIMESTAMP,started_at=NULL,next_attempt_at=NULL,last_error=NULL,note='Mevcut kilitli/doğrulanmış görsel ve fiziksel dosya korundu' WHERE product_id=?",(int(product_id),)) c.commit();c.close();return {'ok':True,'file':product['image_path'],'source':'existing-locked','confidence':float(product.get('image_match_confidence') or 1)} c.execute("UPDATE products SET image_path=NULL,image_locked=0,image_status='pending',image_checked_at=CURRENT_TIMESTAMP,image_updated_at=CURRENT_TIMESTAMP WHERE id=?",(int(product_id),)) product['image_path']=None;product['image_locked']=0 log_event(c,'image.repair','warning','DB görsel referansı vardı fakat fiziksel kapak dosyası yoktu; yeniden üretime alındı',run_type='image',run_id=run_id,product_id=product_id,commit=True) sibling=None if force else _reuse_verified_sibling(c,product) if sibling: log_event(c,'image','info','Aynı model+renkte doğrulanmış kardeş varyant görseli yeniden kullanıldı',run_type='image',run_id=run_id,product_id=product_id,details=sibling,commit=True) c.commit();c.close();return {'ok':True,'file':sibling['file'],'source':'consensus:sibling-reuse','confidence':sibling['confidence']} max_pages=max(1,int(browser_pages or setting_int(c,'image_browser_max_pages',4))) browser=HybridBrowser(enabled=setting(c,'fetch_mode','hybrid')=='hybrid',max_pages=max_pages,timeout_ms=setting_int(c,'browser_timeout_ms',18000),user_agent=UA) session=requests.Session();session.headers.update({'User-Agent':UA,'Accept-Language':'tr-TR,tr;q=0.9,en;q=0.7'}) errors=[];candidates=[] log_event(c,'image','info','Zorunlu ürün görseli aranıyor',run_type='image',run_id=run_id,product_id=product_id,details={'query':_strict_image_query(product),'browser_pages':max_pages},commit=True) try: # v0.3.16: Brave yok. Önce fiyat/kaynak motorunun bulduğu gerçek ürün # sayfaları taranır. Exact JSON-LD Product.image, doğrulanmış source sayfasından # geldiyse tek kaynakla kilitlenebilir; diğer adaylar mutabakata gider. pages=page_urls(c,product_id) trusted_exact=[] ordered=sorted(pages,key=lambda x:(0 if 'hepsiburada.com' in _host(x[0]) else 1,0 if x[1]=='source' else 1)) for url,kind,merchant_id in ordered[:max(6,max_pages*3)]: if _amazonish(url): errors.append(f"{urlparse(url).hostname}: Amazon görsel kanıtı atlandı");continue try: html,final,mode=fetch_page(session,url,browser,headers={"User-Agent":UA,"Accept-Language":"tr-TR,tr;q=0.9,en;q=0.7"},run_id=run_id,c=c,product_id=product_id,merchant_id=merchant_id,context="image.verified-page") found=page_image_candidates(html,final,product);candidates.extend(found) if kind in ('source','admin-variant-evidence'): # Kaynak URL condition-verified veya yönetici onaylı varyant kanıtıdır. Sayfa bağlamı # model+renk doğrulamasından geçtiyse JSON-LD veya OG ana görseli tek kaynakla kilitleyebiliriz. trusted_exact.extend([x for x in found if _candidate_parts(x)[2] in ('jsonld-product','page-meta') and float(_candidate_parts(x)[3] or 0)>=.94]) if kind=='official-catalog': for variant_url in official_variant_links(html,final,product,limit=3): try: vhtml,vfinal,vmode=fetch_page(session,variant_url,browser,headers={"User-Agent":UA,"Accept-Language":"tr-TR,tr;q=0.9,en;q=0.7"},run_id=run_id,c=c,product_id=product_id,context="image.official-variant") vfound=page_image_candidates(vhtml,vfinal,product);candidates.extend(vfound) trusted_exact.extend([x for x in vfound if _candidate_parts(x)[2] in ('jsonld-product','page-meta') and float(_candidate_parts(x)[3] or 0)>=.95]) except Exception as ve:errors.append(f"official variant: {ve}") if found: log_event(c,'image.page','info',f'{urlparse(final).hostname}: {len(found)} model+renk görsel adayı',run_type='image',run_id=run_id,product_id=product_id,merchant_id=merchant_id,details={'kind':kind,'fetch_mode':mode},commit=True) except Exception as e:errors.append(f"{urlparse(url).hostname}: {e}") if trusted_exact: trusted_exact.sort(key=lambda x:(_candidate_source_rank(x),float(_candidate_parts(x)[3] or 0)),reverse=True) image_url,page_url,kind,score,text,probe_url=_candidate_parts(trusted_exact[0]) filename=cache_product_image(c,session,product,image_url,source_page_url=page_url or None,source_kind='trusted:verified-source-exact',confidence=min(.999,max(.96,float(score))),force=True) c.execute("UPDATE products SET image_locked=1,image_verified_at=CURRENT_TIMESTAMP,image_provider_rank=MAX(COALESCE(image_provider_rank,0),105) WHERE id=?",(product_id,)) shared=_propagate_verified_image(c,product,filename,image_url,page_url,'trusted:verified-source-exact',min(.999,max(.96,float(score)))) c.execute("UPDATE image_fetch_requests SET status='done',finished_at=CURRENT_TIMESTAMP,started_at=NULL,next_attempt_at=NULL,image_url=?,source_page_url=?,source_kind='trusted:verified-source-exact',confidence=?,last_error=NULL,note='Doğrulanmış ürün sayfasında exact JSON-LD Product.image' WHERE product_id=?",(image_url,page_url,min(.999,max(.96,float(score))),product_id)) log_event(c,'image','info','Brave olmadan doğrulanmış kaynak sayfasından görsel kilitlendi',run_type='image',run_id=run_id,product_id=product_id,details={'file':filename,'source_page_url':page_url,'confidence':score,'shared_siblings':shared},commit=True) c.commit() try:Path('/var/lib/fiyat-radar/family-sync.trigger').touch() except OSError:pass return {'ok':True,'file':filename,'source':'trusted:verified-source-exact','confidence':score} if not candidates:raise ValueError('Görsel adayı bulunamadı; '+(' | '.join(errors[-5:]) if errors else 'arama sonucu yok')) log_event(c,'image.search','info',f'Görsel mutabakatına {len(candidates)} ham aday gönderildi',run_type='image',run_id=run_id,product_id=product_id,details={'candidate_count':len(candidates),'errors':errors[-8:]},commit=True) winner,consensus=_strict_consensus(c,session,product,candidates) image_url,page_url,kind,score,text,probe_url=_candidate_parts(winner) final_kind="consensus:"+kind final_image_url=image_url final_conf=min(.999,max(.95,float(score))) try: filename=cache_product_image(c,session,product,image_url,source_page_url=page_url or None,source_kind=final_kind,confidence=final_conf,force=True) except Exception as original_error: # Mutabakat küçük önizleme üzerinde doğrulandıysa fakat mağazanın orijinal CDN'i # 403/hotlink engeli veriyorsa doğrulanmış proxy görselini kaybetme. if not probe_url or probe_url==image_url: raise final_kind="consensus:"+kind+":verified-thumbnail" final_image_url=probe_url final_conf=min(.985,max(.95,float(score))) filename=cache_product_image(c,session,product,probe_url,source_page_url=None,source_kind=final_kind,confidence=final_conf,force=True) errors.append(f'Orijinal görsel indirilemedi; doğrulanmış thumbnail kullanıldı: {original_error}') c.execute("UPDATE products SET image_locked=1,image_verified_at=CURRENT_TIMESTAMP,image_provider_rank=MAX(COALESCE(image_provider_rank,0),CASE WHEN lower(COALESCE(image_source_page_url,image_source_url,'')) LIKE '%hepsiburada.com%' THEN 100 ELSE 90 END) WHERE id=?",(product_id,)) shared=_propagate_verified_image(c,product,filename,final_image_url,page_url,final_kind,final_conf) c.execute("UPDATE image_fetch_requests SET status='done',finished_at=CURRENT_TIMESTAMP,next_attempt_at=NULL,image_url=?,source_page_url=?,source_kind=?,confidence=?,last_error=NULL,note=? WHERE product_id=?",(final_image_url,page_url,final_kind,final_conf,f"Görsel mutabakatı: {consensus['independent_sources']} bağımsız kaynak / {consensus['evidence_count']} kanıt",product_id)) b=c.execute("SELECT id,status,price_count,source_count FROM product_bootstrap_requests WHERE product_id=? ORDER BY id DESC LIMIT 1",(product_id,)).fetchone() if b and b["status"] in ("partial","pending") and int(b["price_count"] or 0)>0: c.execute("UPDATE product_bootstrap_requests SET status='done',image_ready=1,finished_at=CURRENT_TIMESTAMP,note=? WHERE id=?",(f"Hazır: {int(b['source_count'] or 0)} kaynak, {int(b['price_count'] or 0)} fiyat, mutabakatlı ürün görseli tamamlandı.",b["id"])) elif b:c.execute("UPDATE product_bootstrap_requests SET image_ready=1 WHERE id=?",(b["id"],)) log_event(c,"image","info","Mutabakatlı ürün görseli hazırlandı",run_type="image",run_id=run_id,product_id=product_id,details={"file":filename,"image_url":final_image_url,"source_page_url":page_url,"source_kind":final_kind,"confidence":score,"consensus":consensus,"candidate_text":text[:500],"fallback_notes":errors[-3:],"shared_siblings":shared});c.commit() try: Path('/var/lib/fiyat-radar/family-sync.trigger').touch() except OSError: pass return {"ok":True,"file":filename,"source":final_kind,"confidence":score,"consensus":consensus} finally: browser.close();c.close() def fail_request(req,error): c=connect();attempts=int(req.get('attempts') or 1) delays=[2,5,15,30,120,360,720] minutes=delays[min(max(0,attempts-1),len(delays)-1)] status='retry' next_at=(datetime.now(timezone.utc)+timedelta(minutes=minutes)).strftime('%Y-%m-%d %H:%M:%S') c.execute("UPDATE image_fetch_requests SET status=?,finished_at=CURRENT_TIMESTAMP,next_attempt_at=?,last_error=?,note=? WHERE product_id=?",(status,next_at,str(error)[:1800],f'Görsel bulunamadı; {minutes} dk sonra otomatik tekrar denenecek',req['product_id'])) c.execute("UPDATE products SET image_status='retry',image_checked_at=CURRENT_TIMESTAMP WHERE id=?",(req['product_id'],)) log_event(c,'image','warning',f'Görsel alınamadı: {error}',run_type='image',run_id=req.get('id'),product_id=req['product_id'],details={'attempts':attempts,'retry_minutes':minutes});c.commit();c.close() def process_queue(limit=6): # v0.3.10 watchdog: started_at NULL olan eski running kayıtları hiçbir zaman # readiness watchdog'una takılmıyordu. Worker kendi kendini de iyileştirir. c=connect() try: c.execute("""UPDATE image_fetch_requests SET status='retry',started_at=NULL,next_attempt_at=CURRENT_TIMESTAMP, note='v0.3.10 watchdog: sahipsiz running görsel işi kurtarıldı' WHERE status='running' AND (started_at IS NULL OR started_at < datetime('now','-4 minutes'))""") c.commit() finally:c.close() queue_missing(500) Path(LOCK).parent.mkdir(parents=True,exist_ok=True) with open(LOCK,'a+',encoding='utf-8') as fh: try:fcntl.flock(fh.fileno(),fcntl.LOCK_EX|fcntl.LOCK_NB) except BlockingIOError:return 0 done=0 while done