#!/usr/bin/env python3 from __future__ import annotations import argparse, ipaddress, os, re, socket, sqlite3, sys, time from urllib.parse import parse_qs, quote_plus, unquote, urljoin, urlparse import requests from pathlib import Path from bs4 import BeautifulSoup from extractors.generic import extract from matching import norm, product_query, product_family_query, product_family_key, tokens, validate_product_match, model_compatible, color_matches, capacity_values from browser_fetch import HybridBrowser from decision_engine import evaluate from dbutil import setting, setting_bool, setting_int, log_event from search_evidence import evaluate_search_result, SearchEvidence, prices_from_text from condition import assess_condition, detect_condition, normalize_condition from sitemap_discovery import discover_urls as sitemap_product_urls DB=os.environ.get('FIYAT_RADAR_DB','/var/lib/fiyat-radar/fiyat-radar.sqlite');UA=os.environ.get('FIYAT_RADAR_UA','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36');TIMEOUT=int(os.environ.get('FIYAT_RADAR_TIMEOUT','20'));SEARCH_CONNECT_TIMEOUT=float(os.environ.get('FIYAT_RADAR_SEARCH_CONNECT_TIMEOUT','3.5'));SEARCH_READ_TIMEOUT=float(os.environ.get('FIYAT_RADAR_SEARCH_READ_TIMEOUT','6'));MAX_HTML_BYTES=int(os.environ.get('FIYAT_RADAR_MAX_HTML_BYTES',str(5*1024*1024)));TARGET_SOURCES=int(os.environ.get('FIYAT_RADAR_TARGET_SOURCES','5'));LAST_ERRORS=[] EXTERNAL_SEARCH_DISABLED=os.environ.get('FIYAT_RADAR_EXTERNAL_SEARCH_DISABLED','1').strip().lower() not in ('0','false','no','off') DDG_DISABLED=EXTERNAL_SEARCH_DISABLED SEARCH_BLOCKED_HOSTS={'duckduckgo.com','google.com','bing.com','search.brave.com','youtube.com','facebook.com','instagram.com','x.com','twitter.com','tiktok.com','pinterest.com'} PRIORITY_DISCOVERY_DOMAINS=('hepsiburada.com','teknosa.com','mediamarkt.com.tr','vatanbilgisayar.com') def note_error(message): msg=str(message);LAST_ERRORS.append(msg);del LAST_ERRORS[:-8];print(msg,file=sys.stderr) def connect(): c=sqlite3.connect(DB,timeout=30);c.row_factory=sqlite3.Row;c.execute('PRAGMA foreign_keys=ON');c.execute('PRAGMA busy_timeout=30000');c.execute('PRAGMA synchronous=NORMAL');return c def _health_blocked(c, table, key_col, key): try: row=c.execute(f"SELECT blocked_until FROM {table} WHERE {key_col}=?",(key,)).fetchone() if not row or not row['blocked_until']: return False return bool(c.execute("SELECT datetime(?) > CURRENT_TIMESTAMP",(row['blocked_until'],)).fetchone()[0]) except sqlite3.Error: return False def _is_access_failure(error): text=str(error).lower() return any(x in text for x in ('403','429','timed out','timeout','connection','max retries','captcha','anti-bot','doğrulama','access denied','forbidden')) def _cooldown_minutes(error, *, provider=False): text=str(error).lower() # 402 Brave kota/abonelik hatasıdır; 10 dakikada bir tekrar vurmak hem boşuna # iş üretir hem de bootstrap kuyruğunu yavaşlatır. Bir gün boyunca provider'ı # kullanma; diğer anahtarsız keşif kanalları devam etsin. if '402' in text or 'payment required' in text or 'quota' in text or 'subscription' in text: return 1440 if provider else 240 if '403' in text or '429' in text or 'captcha' in text or 'anti-bot' in text or 'doğrulama' in text or 'access denied' in text: return 360 if provider else 60 if 'timed out' in text or 'timeout' in text or 'connection' in text or 'max retries' in text or 'çözümlenemedi' in text: return 30 if provider else 15 return 10 def provider_available(c, provider): return not _health_blocked(c,'search_provider_health','provider',provider) def provider_fail(c, provider, error, *, run_id=None, product_id=None): text=str(error).lower();mins=_cooldown_minutes(error,provider=True) quota=('402' in text or 'payment required' in text or 'quota' in text or 'subscription' in text) status='quota' if quota else 'cooldown' c.execute("""INSERT INTO search_provider_health(provider,status,consecutive_failures,blocked_until,last_error,updated_at) VALUES(?,?,1,datetime('now',?),?,CURRENT_TIMESTAMP) ON CONFLICT(provider) DO UPDATE SET status=excluded.status,consecutive_failures=search_provider_health.consecutive_failures+1,blocked_until=excluded.blocked_until,last_error=excluded.last_error,updated_at=CURRENT_TIMESTAMP""", (provider,status,f'+{mins} minutes',str(error)[:1200])) label='kota/abonelik nedeniyle' if quota else 'geçici olarak' log_event(c,'search.provider','warning',f'{provider} {label} beklemeye alındı ({mins} dk): {error}',run_type='discovery',run_id=run_id,product_id=product_id,details={'provider':provider,'status':status,'cooldown_minutes':mins}) c.commit() def provider_ok(c, provider): c.execute("""INSERT INTO search_provider_health(provider,status,consecutive_failures,blocked_until,last_error,last_ok_at,updated_at) VALUES(?,'ok',0,NULL,NULL,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP) ON CONFLICT(provider) DO UPDATE SET status='ok',consecutive_failures=0,blocked_until=NULL,last_error=NULL,last_ok_at=CURRENT_TIMESTAMP,updated_at=CURRENT_TIMESTAMP""",(provider,));c.commit() def merchant_available(c, merchant_id): return not _health_blocked(c,'merchant_access_health','merchant_id',int(merchant_id)) def merchant_fail(c, merchant_id, error, *, run_id=None, product_id=None): mins=_cooldown_minutes(error,provider=False) c.execute("""INSERT INTO merchant_access_health(merchant_id,status,consecutive_failures,blocked_until,last_error,updated_at) VALUES(?,'cooldown',1,datetime('now',?),?,CURRENT_TIMESTAMP) ON CONFLICT(merchant_id) DO UPDATE SET status='cooldown',consecutive_failures=merchant_access_health.consecutive_failures+1,blocked_until=datetime('now',?),last_error=excluded.last_error,updated_at=CURRENT_TIMESTAMP""",(int(merchant_id),f'+{mins} minutes',str(error)[:1200],f'+{mins} minutes')) log_event(c,'merchant.access','warning',f'Mağaza erişimi {mins} dk beklemeye alındı: {error}',run_type='discovery',run_id=run_id,product_id=product_id,merchant_id=merchant_id,details={'cooldown_minutes':mins}) c.commit() def merchant_ok(c, merchant_id): c.execute("""INSERT INTO merchant_access_health(merchant_id,status,consecutive_failures,blocked_until,last_error,last_ok_at,updated_at) VALUES(?,'ok',0,NULL,NULL,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP) ON CONFLICT(merchant_id) DO UPDATE SET status='ok',consecutive_failures=0,blocked_until=NULL,last_error=NULL,last_ok_at=CURRENT_TIMESTAMP,updated_at=CURRENT_TIMESTAMP""",(int(merchant_id),));c.commit() def _public_host(host): try:infos=socket.getaddrinfo(host,None,type=socket.SOCK_STREAM) except socket.gaierror:return False ips={x[4][0] for x in infos} if not ips:return False for raw in ips: try: if not ipaddress.ip_address(raw).is_global:return False except ValueError:return False return True def _safe_url(url): p=urlparse(url) if p.scheme not in ('http','https') or not p.hostname:raise ValueError('Geçersiz web URL') if not _public_host(p.hostname):raise ValueError('URL herkese açık bir adrese çözülmüyor') return url def safe_get(session,url,*,headers=None,method='GET',data=None,timeout=None): current=url for _ in range(5): _safe_url(current);request_timeout=timeout if timeout is not None else TIMEOUT;r=session.post(current,data=data,timeout=request_timeout,headers=headers or {},stream=True,allow_redirects=False) if method=='POST' else session.get(current,timeout=request_timeout,headers=headers or {},stream=True,allow_redirects=False) if r.status_code in (301,302,303,307,308): loc=r.headers.get('location');r.close() if not loc:raise ValueError('Yönlendirmede Location yok') current=urljoin(current,loc);method='GET';data=None;continue r.raise_for_status();ctype=(r.headers.get('content-type') or '').lower() if 'html' not in ctype and 'text/' not in ctype and 'json' not in ctype:r.close();raise ValueError('HTML/text yanıtı değil') chunks=[];total=0 for chunk in r.iter_content(65536): if not chunk:continue total+=len(chunk) if total>MAX_HTML_BYTES:r.close();raise ValueError('Sayfa boyut sınırını aşıyor') chunks.append(chunk) raw=b''.join(chunks);enc=r.encoding or 'utf-8';final=r.url or current;r.close() try:text=raw.decode(enc,errors='replace') except LookupError:text=raw.decode('utf-8',errors='replace') return text,final raise ValueError('Çok fazla yönlendirme') def fetch_page(session,url,browser,*,headers=None,run_id=None,c=None,product_id=None,merchant_id=None,context='discovery'): try:return (*safe_get(session,url,headers=headers),'http') except Exception as e: if c:log_event(c,context+'.fetch','warning',f'HTTP okuma başarısız: {e}',run_type='discovery',run_id=run_id,product_id=product_id,merchant_id=merchant_id,details={'url':url}) if not browser or not browser.available:raise html,final=browser.fetch(url) if c:log_event(c,context+'.browser','info','Chromium fallback ile sayfa açıldı',run_type='discovery',run_id=run_id,product_id=product_id,merchant_id=merchant_id,details={'url':final,'http_error':str(e)}) return html,final,'browser' def host_domain(url): host=(urlparse(url).hostname or '').lower().strip('.').removeprefix('www.');parts=host.split('.');multi={'com.tr','net.tr','org.tr','gen.tr','web.tr','co.uk','com.de','com.br'} if len(parts)>=3 and '.'.join(parts[-2:]) in multi:return '.'.join(parts[-3:]) return '.'.join(parts[-2:]) if len(parts)>=2 else host def same_domain(url,domain): host=(urlparse(url).hostname or '').lower().strip('.').removeprefix('www.');domain=(domain or '').lower().strip('.').removeprefix('www.');return bool(host and domain and (host==domain or host.endswith('.'+domain) or domain.endswith('.'+host))) def make_search_url(row, *, family=False): query=product_family_query(row) if family else product_query(row,include_price_word=False);tpl=row['search_url_template'] or '';return tpl.replace('{query}',quote_plus(query)).replace('{ean}',quote_plus(row['ean'] or query)) def candidate_links(html,base_url,domain,product): soup=BeautifulSoup(html,'lxml');wanted=tokens(product_query(product,include_price_word=False));scored=[];seen=set() for a in soup.find_all('a',href=True): u=urljoin(base_url,a['href']).split('#',1)[0] if u in seen or not same_domain(u,domain):continue seen.add(u);raw_text=a.get_text(' ',strip=True)+' '+u # Adayı daha sayfasını açmadan kardeş-model filtresinden geçir. # iPhone 16 aramasında iPhone 16 Plus/Pro/Pro Max linkleri listeye dahi giremez. if not model_compatible(product,raw_text): continue text=norm(raw_text);score=sum(1 for t in wanted if t in text) if score>0:scored.append((score,u)) scored.sort(key=lambda x:x[0],reverse=True);return [u for _,u in scored[:14]] def _clean_search_result_url(raw): if not raw:return None u=urljoin('https://html.duckduckgo.com/',raw);p=urlparse(u);qs=parse_qs(p.query) if 'uddg' in qs and qs['uddg']:u=unquote(qs['uddg'][0]) hp=host_domain(u) if not hp or hp in SEARCH_BLOCKED_HOSTS or urlparse(u).scheme not in ('http','https'):return None return u.split('#',1)[0] def _slug(text): return '-'.join(norm(text).split()) def _product_family_slug(product): brand=str(product.get('brand') or '').strip() model=str(product.get('model') or '').strip() if model and norm(brand) and norm(model).startswith(norm(brand)+' '): raw=model else: raw=' '.join(x for x in (brand,model) if x).strip() or str(product.get('name') or product.get('product_name') or '') return _slug(raw) def _capacity_url_tokens(value): try:v=int(value or 0) except Exception:return [] if not v:return [] if v>=1024 and v%1024==0: tb=v//1024 return [f'{tb}tb',f'{tb}-tb',f'{v}gb',f'{v}-gb'] return [f'{v}gb',f'{v}-gb'] def mutate_sibling_url(url,sibling,target): """Çalışmış kardeş varyant URL'sinden güvenli aday URL'ler üret. Yalnız URL'deki açık GB/TB ve renk slug'larını değiştirir. Tahmin edilen URL hiçbir zaman doğrudan kaynak sayılmaz; consider_page her adayı tekrar açıp model+durum+depolama+renk doğrulamasından geçirir. """ out=[] if not url:return out lower=url.lower() src_tokens=_capacity_url_tokens(sibling.get('capacity_gb')) dst_tokens=_capacity_url_tokens(target.get('capacity_gb')) for st in src_tokens: if st in lower: for dt in dst_tokens: cand=re.sub(re.escape(st),dt,url,flags=re.I) if cand!=url and cand not in out:out.append(cand) sc=_slug(str(sibling.get('color') or ''));tc=_slug(str(target.get('color') or '')) if sc and tc and sc!=tc and sc in lower: cand=re.sub(re.escape(sc),tc,url,flags=re.I) if cand!=url and cand not in out:out.append(cand) # kapasite + renk birlikte değişebilsin for base in list(out): if sc in base.lower(): both=re.sub(re.escape(sc),tc,base,flags=re.I) if both not in out:out.append(both) return out[:8] def bridge_seed_urls(product): """Arama API'siz, doğrulanabilir ürün indeks tohumları. Epey telefon ürün URL'leri model + kapasite için kararlı bir kanonik yapı kullanıyor. Bu sayfa fiyat kaynağı olarak doğrudan kabul edilmez; yalnız gerçek mağaza ürün sayfalarına giden aday linkleri çıkarmak için kullanılır. """ if not _phone_like(product):return [] slug=_product_family_slug(product) if not slug:return [] cap=product.get('capacity_gb') suffixes=[] try:v=int(cap or 0) except Exception:v=0 if v: if v>=1024 and v%1024==0: suffixes += [f'-{v//1024}tb',f'-{v}gb'] else: suffixes += [f'-{v}gb'] # Bazı modellerde temel kapasite yalın URL'dir (örn. Galaxy A16 / iPhone 16). suffixes += [''] out=[] for sf in suffixes: u=f'https://www.epey.com/akilli-telefonlar/{slug}{sf}.html' if u not in out:out.append(u) return out[:3] def _phone_like(product): b=norm(str(product.get('brand') or '')) txt=' '+norm(' '.join(str(product.get(k) or '') for k in ('name','model','category')))+' ' phone_brands={'apple','samsung','xiaomi','redmi','poco','huawei','honor','oppo','realme','oneplus','sony','lg','motorola','vivo','asus','nothing','google','zte','tecno','infinix','nokia'} return b in phone_brands or any(x in txt for x in (' iphone ',' galaxy ',' telefon ',' smartphone ')) def bridge_outbound_links(html,base_url,product,limit=16): """Karşılaştırma/indeks sayfasındaki gerçek mağaza çıkışlarını puanla.""" soup=BeautifulSoup(html,'lxml');base_domain=host_domain(base_url);wanted=tokens(product_query(product,include_price_word=False));scored=[];seen=set() expected_cap=None try:expected_cap=int(product.get('capacity_gb') or 0) or None except Exception:pass for a in soup.find_all('a',href=True): href=(a.get('href') or '').strip() if not href:continue u=urljoin(base_url,href).split('#',1)[0] if u in seen:continue seen.add(u) dom=host_domain(u) if not dom or dom in SEARCH_BLOCKED_HOSTS:continue # Epey içindeki ürün sayfasının kendisini tekrar aday yapma. Redirect/çıkış # linkleri ise aynı domainde olsa da path adından anlaşılırsa tutulabilir. if dom==base_domain and not re.search(r'(git|redirect|out|cikis|siteye|link)',urlparse(u).path,re.I): continue parent=a for _ in range(3): if parent.parent is None:break parent=parent.parent txt=parent.get_text(' ',strip=True) if len(txt)>=40:break context=(parent.get_text(' ',strip=True)+' '+a.get_text(' ',strip=True)+' '+u)[:1800] if not model_compatible(product,context):continue caps=capacity_values(context,product) if expected_cap and caps and expected_cap not in caps:continue score=sum(1 for t in wanted if t in norm(context)) if expected_cap and expected_cap in caps:score+=4 if product.get('color') and color_matches(product.get('color'),context):score+=4 # "Siteye Git" gibi çıkış linki tek başına bağlamsızsa ebeveyn metni model # kanıtı taşımıyorsa zaten score düşük kalır. if score>=3:scored.append((score,u,context[:500])) scored.sort(key=lambda x:(x[0],0 if host_domain(x[1])!=base_domain else -1),reverse=True) return scored[:limit] def epey_indirect_evidence(html,base_url,product): """Epey'deki tekil teklif bloklarından güvenli *dolaylı* fiyat kanıtı çıkar. Bu veri asla product_sources/direct price değildir. Ama eski sürümde çalışan "arama indeksi" davranışını geri getirir. Yeni ürün ailesinde yenilenmiş/açık kutu/2.el sinyali olan tekil bloklar kesin reddedilir. Standart Epey ürün sayfasındaki ve hiçbir non-new sinyali taşımayan tekil teklif, yalnız indeks kanıtı olarak `new` kabul edilir. """ if host_domain(base_url)!='epey.com':return None soup=BeautifulSoup(html,'lxml') price_re=re.compile(r'(?:\d{1,3}(?:[\.\s]\d{3})+|\d{4,7})(?:,\d{1,2})?\s*(?:TL|₺)|(?:TL|₺)\s*(?:\d{1,3}(?:[\.\s]\d{3})+|\d{4,7})(?:,\d{1,2})?',re.I) expected=normalize_condition(product.get('condition_type') or 'new') contexts=[];seen=set() # Fiyat içeren en küçük anlamlı DOM bloğunu seç. Tüm sayfayı condition metni # olarak kullanmak yenilenmiş öneri kartlarının sıfır teklifi zehirlemesine yol açar. for node in soup.find_all(string=price_re): parent=node.parent chosen=None for _ in range(6): if parent is None:break txt=' '.join(parent.get_text(' ',strip=True).split()) if 35<=len(txt)<=1400 and price_re.search(txt): # Model/kapasite bağlamı oluştuğu ilk küçük bloğu tercih et. try: family_product=dict(product);family_product['color']='' validate_product_match(family_product,txt,url=base_url,condition_text='',enforce_condition=False) chosen=txt break except Exception: pass parent=parent.parent if not chosen:continue key=norm(chosen)[:900] if key in seen:continue seen.add(key);contexts.append(chosen) accepted=[] for ctx in contexts: try: # İndeks satırı da model + gerçek storage + hedef renk bakımından kesin olmalı. validate_product_match(product,ctx,url=base_url,condition_text='',enforce_condition=False) except Exception: continue detected,cond_conf,cond_ev,_signals,_types=detect_condition(ctx,base_url,'') if expected=='new': if detected in ('refurbished','open_box','used'):continue if detected=='new': final_cond='new';final_conf=max(.93,cond_conf);evidence=cond_ev or 'epey-offer:new' else: # Epey standart ürün sayfasında kondisyonlu teklifler kendi kartında # "Yenilenmiş/Outlet/2.El" diye etiketlenir. Etiketsiz tekil kartı # doğrudan fiyat değil, kontrollü indeks sinyali olarak kabul ediyoruz. final_cond='new';final_conf=.90;evidence='epey-standard-offer: non-new etiketi yok; yalnız indeks kanıtı' else: if detected!=expected:continue final_cond=detected;final_conf=cond_conf;evidence=cond_ev vals=prices_from_text(ctx) if not vals:continue price=min(vals) # Çok kısa/çok düşük güvenli blokları reddet. Dolaylı fiyat güveni doğrudan # mağaza sayfasından bilinçli olarak daha düşük tutulur. base_conf=.80 if product.get('color') and color_matches(product.get('color'),ctx):base_conf+=.05 if product.get('capacity_gb') and int(product.get('capacity_gb')) in capacity_values(ctx,product):base_conf+=.04 conf=min(.89,base_conf) accepted.append((price,conf,ctx,final_cond,final_conf,evidence)) if not accepted:return None price,conf,ctx,ctype,cconf,cev=min(accepted,key=lambda x:(x[0],-x[1])) ev=SearchEvidence(price=price,currency='TRY',confidence=conf,text=ctx[:1800],reason='Epey tekil teklif indeksinde model+depolama+renk ve kondisyon güvenlik filtresi eşleşti; doğrudan mağaza doğrulaması bekliyor',detected_condition=ctype,condition_confidence=cconf,condition_evidence=cev) result={'url':base_url,'title':ctx[:300],'description':ctx[:1800],'engine':'epey-index'} return result,ev def _product_row(c,pid): row=c.execute("""SELECT p.id product_id,p.name product_name,p.name,p.brand,p.model,p.variant,p.category,p.condition_type,p.capacity_gb,p.color,p.ean,p.family_key FROM products p WHERE p.id=? AND p.active=1""",(int(pid),)).fetchone() return dict(row) if row else None def run_family_source_seeds(c,session,browser,run_id,product_id,max_pages=4): """Aynı ürün ailesindeki çalışan kaynakları yeni varyanta tohum yap.""" target=_product_row(c,product_id) if not target:return (0,0,0) family=target.get('family_key') or product_family_key(target) if not family:return (0,0,0) rows=c.execute("""SELECT s.url,s.last_status,s.last_price,s.last_checked_at,s.merchant_id, sp.id sibling_id,sp.capacity_gb sibling_capacity,sp.color sibling_color, m.name merchant,m.domain,m.trust_score FROM products sp JOIN product_sources s ON s.product_id=sp.id JOIN merchants m ON m.id=s.merchant_id WHERE sp.active=1 AND s.active=1 AND m.active=1 AND sp.id<>? AND COALESCE(sp.family_key,'')=? AND sp.condition_type=? ORDER BY CASE WHEN s.last_status='ok' THEN 0 ELSE 1 END, CASE WHEN s.last_price IS NOT NULL THEN 0 ELSE 1 END, m.trust_score DESC,COALESCE(s.last_checked_at,'') DESC LIMIT 20""",(int(product_id),family,target.get('condition_type') or 'new')).fetchall() tried=auto=queued=0;seen_urls=set();seen_merchants=set() for r in rows: if tried>=max_pages:break url=r['url'];mid=int(r['merchant_id']) if not url or url in seen_urls:continue # Arama sayfası cooldown'da olsa bile yakın zamanda çalışan kesin ürün URL'si # bir kez denenebilir; aynı turda mağazayı tekrar tekrar zorlama. if mid in seen_merchants:continue seen_merchants.add(mid);seen_urls.add(url);tried+=1 sibling={'capacity_gb':r['sibling_capacity'],'color':r['sibling_color']} merchant=c.execute('SELECT * FROM merchants WHERE id=?',(mid,)).fetchone() try: html,final,_=fetch_page(session,url,browser,headers={'User-Agent':UA,'Accept-Language':'tr-TR,tr;q=0.9,en;q=0.7'},run_id=run_id,c=c,product_id=product_id,merchant_id=mid,context='discovery.family-seed') links=[] for u in candidate_links(html,final,r['domain'],target)[:10]: if u not in links:links.append(u) for u in mutate_sibling_url(final,sibling,target): if u not in links:links.append(u) # Bir karşılaştırma/indeks sayfası kardeş kaynaksa gerçek mağaza çıkışlarını da kullan. for _score,u,_ctx in bridge_outbound_links(html,final,target,10): if u not in links:links.append(u) for u in links[:10]: try: st,cf,of,fu,me=consider_page(c,session,browser,target,u,'family_seed',run_id,None if host_domain(u)!=host_domain(final) else merchant) if st=='auto': auto+=1 log_event(c,'discovery.family-seed','info','Kardeş varyant kaynağından doğrulanmış hedef kaynak bulundu',run_type='discovery',run_id=run_id,product_id=product_id,merchant_id=me['id'],details={'sibling_product_id':r['sibling_id'],'seed_url':url,'target_url':fu,'score':cf}) return tried,auto,queued if st=='candidate':queued+=1 except Exception as e: log_event(c,'discovery.family-seed','debug',str(e),run_type='discovery',run_id=run_id,product_id=product_id,merchant_id=mid,details={'seed_url':url,'candidate_url':u}) except Exception as e: log_event(c,'discovery.family-seed','debug',f'Kardeş kaynak okunamadı: {e}',run_type='discovery',run_id=run_id,product_id=product_id,merchant_id=mid,details={'seed_url':url}) return tried,auto,queued def run_index_bridges(c,session,browser,run_id,product_id,max_pages=2): """Ücretsiz/anahtarsız ürün indekslerinden gerçek mağaza sayfası bul.""" product=_product_row(c,product_id) if not product:return (0,0,0) tried=auto=queued=0 for seed in bridge_seed_urls(product)[:max_pages]: tried+=1 try: html,final,_=fetch_page(session,seed,browser,headers={'User-Agent':UA,'Accept-Language':'tr-TR,tr;q=0.9,en;q=0.7'},run_id=run_id,c=c,product_id=product_id,context='discovery.bridge') # İndeks sayfasının kendisi hedef model+storage ailesine ait olmalı; renk # teklif satırlarında doğrulanacağı için burada renk boş bırakılır. family_variant=dict(product);family_variant['color']='' title=(BeautifulSoup(html,'lxml').title.get_text(' ',strip=True) if BeautifulSoup(html,'lxml').title else '') validate_product_match(family_variant,title,html='',url=final,condition_text='',enforce_condition=False) # Eski sürümde görünür olan indeks fiyatı fallback'ini geri getir. # Epey fiyatı hiçbir zaman direct/source sayılmaz; yalnız doğrulanmış # model+GB+renk+kondisyon eşleşmesinde indirect_offers'a girer. try: ie=epey_indirect_evidence(html,final,product) if ie: iresult,iev=ie upsert_indirect_offer(c,product,iresult,iev,'epey-index',run_id);queued+=1 log_event(c,'discovery.bridge-index','info','Epey indeks fiyat sinyali kaydedildi',run_type='discovery',run_id=run_id,product_id=product_id,details={'seed_url':final,'price':iev.price,'confidence':iev.confidence,'condition':iev.detected_condition}) except Exception as iex: log_event(c,'discovery.bridge-index','debug',f'Epey indeks fiyatı çıkarılamadı: {iex}',run_type='discovery',run_id=run_id,product_id=product_id,details={'seed_url':final}) links=bridge_outbound_links(html,final,product,20) log_event(c,'discovery.bridge','info','Anahtarsız indeks tohumu okundu',run_type='discovery',run_id=run_id,product_id=product_id,details={'seed_url':seed,'final_url':final,'outbound_candidates':len(links)}) for score,u,ctx in links[:10]: try: st,cf,of,fu,me=consider_page(c,session,browser,product,u,'index_bridge',run_id,None) if st=='auto': auto+=1 log_event(c,'discovery.bridge','info','İndeks köprüsünden doğrulanmış mağaza kaynağı bulundu',run_type='discovery',run_id=run_id,product_id=product_id,merchant_id=me['id'],details={'seed_url':seed,'target_url':fu,'bridge_score':score,'match_score':cf}) return tried,auto,queued if st=='candidate':queued+=1 except Exception as e: log_event(c,'discovery.bridge','debug',str(e),run_type='discovery',run_id=run_id,product_id=product_id,details={'seed_url':seed,'candidate_url':u,'context':ctx[:240]}) except Exception as e: log_event(c,'discovery.bridge','debug',f'İndeks tohumu kullanılamadı: {e}',run_type='discovery',run_id=run_id,product_id=product_id,details={'seed_url':seed}) return tried,auto,queued def web_search_ddg(session,query,limit=12,*,c=None,run_id=None,product_id=None): global DDG_DISABLED if DDG_DISABLED: return [] if c is not None and not provider_available(c,'duckduckgo'): DDG_DISABLED=True return [] headers={'User-Agent':UA,'Accept-Language':'tr-TR,tr;q=0.9,en;q=0.6'};last=None connect_timeout=max(2,min(10,setting_int(c,'search_connect_timeout_seconds',int(SEARCH_CONNECT_TIMEOUT)))) if c is not None else SEARCH_CONNECT_TIMEOUT read_timeout=max(3,min(15,setting_int(c,'search_read_timeout_seconds',int(SEARCH_READ_TIMEOUT)))) if c is not None else SEARCH_READ_TIMEOUT for endpoint in ('https://html.duckduckgo.com/html/','https://lite.duckduckgo.com/lite/'): try: html,_=safe_get(session,endpoint,headers=headers,method='POST',data={'q':query,'kl':'tr-tr','kp':'-2'},timeout=(connect_timeout,read_timeout));soup=BeautifulSoup(html,'lxml');out=[];seen=set() anchors=soup.select('a.result__a') or soup.select('a[data-testid="result-title-a"]') or soup.select('.result a[href]') or soup.select('a[href]') for a in anchors: u=_clean_search_result_url(a.get('href')) if not u or u in seen:continue seen.add(u);container=a.find_parent(class_=re.compile(r'result|web-result')) snippet='' if container: sn=container.select_one('.result__snippet,.result-snippet,.snippet') if sn:snippet=sn.get_text(' ',strip=True) out.append({'url':u,'title':a.get_text(' ',strip=True),'description':snippet,'extra_snippets':[],'engine':'duckduckgo'}) if len(out)>=limit:break if out: if c is not None: provider_ok(c,'duckduckgo') return out except (requests.Timeout, requests.ConnectionError) as e: last=e;DDG_DISABLED=True if c is not None:provider_fail(c,'duckduckgo',e,run_id=run_id,product_id=product_id) note_error(f'WARN DuckDuckGo bağlantısı bu çalışma için devre dışı: {e}') break except requests.HTTPError as e: last=e status=getattr(e.response,'status_code',None) if status in (403,429): DDG_DISABLED=True if c is not None:provider_fail(c,'duckduckgo',e,run_id=run_id,product_id=product_id) note_error(f'WARN DuckDuckGo bu çalışma için devre dışı: {e}') break except Exception as e: last=e if last and not DDG_DISABLED: note_error(f'WARN DuckDuckGo araması sonuçlanmadı: {last}') return [] def web_search(session,product,limit=12,*,c=None,run_id=None): """v0.3.17: Harici arama motoru kritik akıştan tamamen ayrıdır. Varsayılan kurulumda Brave ve DuckDuckGo dahil genel arama motorlarına ağ isteği gönderilmez. Kaynak keşfi; doğrulanmış aile kaynakları, mağaza aramaları, indeks/ürün sayfası köprüleri ve robots/sitemap kataloglarıyla sürer. """ if EXTERNAL_SEARCH_DISABLED: return [],'disabled' query=product_query(product,include_price_word=True) if c is not None: try: c.execute("INSERT INTO settings(key,value,updated_at) VALUES('brave_disabled','1',CURRENT_TIMESTAMP) ON CONFLICT(key) DO UPDATE SET value='1',updated_at=CURRENT_TIMESTAMP") except Exception:pass ddg=web_search_ddg(session,query,limit,c=c,run_id=run_id,product_id=product.get('product_id')) return ddg,'duckduckgo' def upsert_indirect_offer(c,product,result,evidence,method,run_id): url=result['url'];domain=host_domain(url) merchant=ensure_auto_merchant(c,url,result.get('title') or '',method+'_index') c.execute("""INSERT INTO indirect_offers(product_id,merchant_id,url,title,snippet,price,currency,confidence,source_engine,reason,detected_condition,condition_confidence,condition_evidence,condition_status,active,observed_at,expires_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,'verified',1,CURRENT_TIMESTAMP,datetime('now','+24 hours')) ON CONFLICT(product_id,url,source_engine) DO UPDATE SET merchant_id=excluded.merchant_id,title=excluded.title,snippet=excluded.snippet,price=excluded.price,currency=excluded.currency,confidence=excluded.confidence,reason=excluded.reason,detected_condition=excluded.detected_condition,condition_confidence=excluded.condition_confidence,condition_evidence=excluded.condition_evidence,condition_status='verified',active=1,observed_at=CURRENT_TIMESTAMP,expires_at=datetime('now','+24 hours')""", (product['product_id'],merchant['id'],url,result.get('title') or '',(result.get('description') or '')[:2000],evidence.price,evidence.currency,evidence.confidence,method,evidence.reason,evidence.detected_condition,evidence.condition_confidence,evidence.condition_evidence)) log_event(c,'discovery.indirect','warning','Engelli/okunamayan mağaza için arama indeksinden fiyat sinyali kaydedildi',run_type='discovery',run_id=run_id,product_id=product['product_id'],merchant_id=merchant['id'],details={'url':url,'domain':domain,'price':evidence.price,'confidence':evidence.confidence,'engine':method,'reason':evidence.reason}) c.commit();return merchant def merchant_name_from(domain,title=''): if title: parts=re.split(r'\s*[|\-–—]\s*',title) if len(parts)>1 and 1<=len(parts[-1])<=50 and parts[-1].strip():return parts[-1].strip() root=domain.split('.')[0] if domain else 'Web Mağazası';return root.replace('-',' ').replace('_',' ').title() def ensure_auto_merchant(c,url,title,method): domain=host_domain(url) if not domain:raise ValueError('Kaynak domaini belirlenemedi') row=c.execute('SELECT * FROM merchants WHERE domain=? OR domain=? LIMIT 1',(domain,'www.'+domain)).fetchone() if row:return row cur=c.execute('INSERT INTO merchants(name,domain,active,trust_score,search_url_template,discovery_active,auto_discovered,discovery_source) VALUES(?,?,1,68,NULL,0,1,?)',(merchant_name_from(domain,title),domain,method));c.commit();return c.execute('SELECT * FROM merchants WHERE id=?',(cur.lastrowid,)).fetchone() def upsert_candidate(c,product_id,merchant_id,url,offer,conf,status): condition_status='verified' if getattr(offer,'condition_type',None) else 'pending' c.execute("""INSERT INTO discovery_candidates(product_id,merchant_id,url,title,price,currency,confidence,detected_condition,condition_confidence,condition_evidence,condition_status,status,discovered_at,reviewed_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,CURRENT_TIMESTAMP,CASE WHEN ?='auto_approved' THEN CURRENT_TIMESTAMP ELSE NULL END) ON CONFLICT(product_id,merchant_id,url) DO UPDATE SET title=excluded.title,price=excluded.price,currency=excluded.currency,confidence=excluded.confidence,detected_condition=excluded.detected_condition,condition_confidence=excluded.condition_confidence,condition_evidence=excluded.condition_evidence,condition_status=excluded.condition_status,status=CASE WHEN discovery_candidates.status IN ('approved','rejected') THEN discovery_candidates.status ELSE excluded.status END,discovered_at=CURRENT_TIMESTAMP""",(product_id,merchant_id,url,offer.title,offer.price,offer.currency,conf,getattr(offer,'condition_type',None),getattr(offer,'condition_confidence',None),getattr(offer,'condition_evidence',None),condition_status,status,status)) def consider_page(c,session,browser,product,url,method,run_id,merchant=None): html,final_url,fetched_via=fetch_page(session,url,browser,headers={'User-Agent':UA,'Accept-Language':'tr-TR,tr;q=0.9,en;q=0.6'},run_id=run_id,c=c,product_id=product['product_id'],merchant_id=merchant['id'] if merchant else None) if len(html)<300:raise ValueError('Sayfa içeriği çok kısa') try: offer=extract(html,final_url);evidence_text=offer.match_text or offer.title validate_product_match(product,evidence_text,html=html,url=final_url,condition_text=offer.condition_text or '',enforce_condition=True) ca=assess_condition(product,evidence_text,final_url,offer.condition_text or '',require_known=True) except Exception as e: if fetched_via=='http' and browser and browser.available: log_event(c,'discovery.fetch','warning',f'HTTP içerik doğrulaması başarısız, Chromium deneniyor: {e}',run_type='discovery',run_id=run_id,product_id=product['product_id'],merchant_id=merchant['id'] if merchant else None) html,final_url=browser.fetch(url);fetched_via='browser';offer=extract(html,final_url);evidence_text=offer.match_text or offer.title validate_product_match(product,evidence_text,html=html,url=final_url,condition_text=offer.condition_text or '',enforce_condition=True) ca=assess_condition(product,evidence_text,final_url,offer.condition_text or '',require_known=True) else:raise if ca.status!='verified':raise ValueError(f'CONDITION_{ca.status.upper()}: {ca.evidence or "durum kanıtı yok"}') offer.condition_type=ca.detected or ca.expected;offer.condition_confidence=ca.confidence;offer.condition_evidence=ca.evidence if merchant is None:merchant=ensure_auto_merchant(c,final_url,offer.title,method+'_'+fetched_via) smart=setting_bool(c,'smart_engine',True) if smart:dec=evaluate(c,product,offer,html,trust_score=merchant['trust_score'],fetched_via=fetched_via) else: from matching import confidence cf=confidence(product,offer.title,html,final_url,offer.condition_text or '');dec=type('D',(),{'status':'auto' if cf>=.78 else ('candidate' if cf>=.56 else 'reject'),'score':cf,'summary':f'Klasik eşleşme: {cf:.2f}'})() log_event(c,'discovery.decision','info' if dec.status=='auto' else 'warning',dec.summary,run_type='discovery',run_id=run_id,product_id=product['product_id'],merchant_id=merchant['id'],details={'url':final_url,'title':offer.title,'price':offer.price,'decision':dec.status,'score':dec.score,'fetched_via':fetched_via,'condition':offer.condition_type,'condition_confidence':offer.condition_confidence}) if dec.status=='auto': c.execute("INSERT OR IGNORE INTO product_sources(product_id,merchant_id,url,extractor,active,check_interval_minutes,next_check_at,expected_condition,condition_status) VALUES(?,?,?,'generic',1,360,datetime('now','+360 minutes'),?,'pending')",(product['product_id'],merchant['id'],final_url,product.get('condition_type') or 'new')) src=c.execute('SELECT id FROM product_sources WHERE product_id=? AND url=? LIMIT 1',(product['product_id'],final_url)).fetchone() if src: c.execute("UPDATE product_sources SET expected_condition=?,detected_condition=?,condition_confidence=?,condition_evidence=?,condition_status='verified',last_price=?,last_currency=?,last_stock=?,last_title=?,last_confidence=?,last_status='ok',last_error=NULL,last_checked_at=CURRENT_TIMESTAMP,next_check_at=datetime('now','+360 minutes'),consecutive_failures=0,last_seller_name=?,last_shipping_price=?,last_shipping_known=?,last_total_price=?,stock_confidence=? WHERE id=?",(product.get('condition_type') or 'new',offer.condition_type,offer.condition_confidence,offer.condition_evidence,offer.price,offer.currency,None if offer.in_stock is None else int(offer.in_stock),offer.title,dec.score,offer.seller_name or merchant['name'],offer.shipping_price,int(offer.shipping_known),offer.total_price,offer.stock_confidence,src['id'])) c.execute('INSERT INTO price_observations(product_id,source_id,merchant_id,price,currency,in_stock,confidence,raw_title,seller_name,shipping_price,shipping_known,total_price,stock_confidence,condition_type,condition_confidence,condition_evidence,observed_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,CURRENT_TIMESTAMP)',(product['product_id'],src['id'],merchant['id'],offer.price,offer.currency,None if offer.in_stock is None else int(offer.in_stock),dec.score,offer.title,offer.seller_name or merchant['name'],offer.shipping_price,int(offer.shipping_known),offer.total_price,offer.stock_confidence,offer.condition_type,offer.condition_confidence,offer.condition_evidence)) upsert_candidate(c,product['product_id'],merchant['id'],final_url,offer,dec.score,'auto_approved');c.commit();return 'auto',dec.score,offer,final_url,merchant if dec.status=='candidate':upsert_candidate(c,product['product_id'],merchant['id'],final_url,offer,dec.score,'pending');c.commit();return 'candidate',dec.score,offer,final_url,merchant return 'low',dec.score,offer,final_url,merchant def run_sitemap_merchants(c,session,browser,run_id,product_id,merchant_limit=4,url_limit=3): """v0.3.15: arama API'si veya mağaza arama sayfası çalışmasa da public robots.txt/sitemap kataloglarından doğrudan ürün URL'si bul. Sitemap yalnız ADAY URL verir; ürün sayfası yine consider_page ile model, kapasite, renk ve ürün durumu bakımından tam doğrulanmadan fiyat yazılmaz. """ prow=c.execute("""SELECT p.id product_id,p.name product_name,p.name,p.brand,p.model,p.variant,p.category, p.condition_type,p.capacity_gb,p.color,p.ean,p.family_key FROM products p WHERE p.id=? AND p.active=1""",(int(product_id),)).fetchone() if not prow:return (0,0,0,0) product=dict(prow) rows=c.execute("""SELECT m.*,msc.status sitemap_status,msc.expires_at sitemap_expires,COALESCE(msc.url_count,0) sitemap_url_count, mah.blocked_until access_blocked FROM merchants m LEFT JOIN merchant_sitemap_cache msc ON msc.merchant_id=m.id LEFT JOIN merchant_access_health mah ON mah.merchant_id=m.id WHERE m.active=1 AND m.discovery_active=1 AND NOT EXISTS(SELECT 1 FROM product_sources ps WHERE ps.product_id=? AND ps.merchant_id=m.id AND ps.active=1) ORDER BY CASE WHEN msc.status='ok' AND msc.expires_at>CURRENT_TIMESTAMP THEN 0 ELSE 1 END, CASE WHEN mah.blocked_until IS NULL OR mah.blocked_until<=CURRENT_TIMESTAMP THEN 0 ELSE 1 END, m.trust_score DESC,m.id LIMIT 24""",(int(product_id),)).fetchall() attempted=auto=queued=failed=0;catalog_checked=0 for merchant in rows: if attempted>=max(1,int(merchant_limit)) or catalog_checked>=max(2,int(merchant_limit)+1):break catalog_checked+=1 try: urls=sitemap_product_urls(c,session,merchant,product,run_id=run_id,limit=max(1,int(url_limit))) except Exception as e: log_event(c,'discovery.sitemap','debug',f'Sitemap aday üretilemedi: {e}',run_type='discovery',run_id=run_id,product_id=product_id,merchant_id=merchant['id']) continue if not urls:continue attempted+=1;best=None;access_failures=0 for u in urls[:max(1,int(url_limit))]: try: result=consider_page(c,session,browser,product,u,'merchant_sitemap',run_id,merchant) rank={'auto':3,'candidate':2,'low':1}.get(result[0],0) if best is None or (rank,result[1])>(best[0],best[1]):best=(rank,result[1],result) if result[0]=='auto':break except Exception as e: failed+=1 if _is_access_failure(e):access_failures+=1 log_event(c,'discovery.sitemap-skip','debug',str(e),run_type='discovery',run_id=run_id,product_id=product_id,merchant_id=merchant['id'],details={'url':u}) if access_failures>=2:break if best: status=best[2][0] if status=='auto': auto+=1;break if status=='candidate':queued+=1 log_event(c,'discovery.sitemap','info','Sitemap ürün keşfi tamamlandı',run_type='discovery',run_id=run_id,product_id=product_id,details={'catalog_checked':catalog_checked,'merchant_attempted':attempted,'auto':auto,'candidate':queued,'failed_pages':failed}) return attempted,auto,queued,failed def run_known_merchants_exact(c,session,browser,run_id,limit=6,product_id=None,force_access_probe=False): """v0.3.14: v0.1.x'in çalışan *tam varyant* mağaza keşfini geri getir. Kritik farklar: - Arama sorgusu aile adı değil tam ürün varyantıdır (model + GB + renk + kondisyon). - Cooldown'daki mağaza `limit` bütçesini tüketmez. Yalnız gerçek HTTP/Chromium arama isteği `attempted` sayacını artırır. - Bulunan sayfa yine güncel katı model/GB/renk/kondisyon doğrulamasından geçer; eski sürümün gevşek kabul davranışı geri getirilmez. """ sql="""SELECT p.id product_id,p.name product_name,p.name,p.brand,p.model,p.variant,p.category, p.condition_type,p.capacity_gb,p.color,p.ean,p.family_key, m.id merchant_id,m.name merchant,m.domain,m.trust_score,m.search_url_template, mah.status access_status,mah.blocked_until,COALESCE(mah.consecutive_failures,0) access_failures FROM products p CROSS JOIN merchants m LEFT JOIN merchant_access_health mah ON mah.merchant_id=m.id WHERE p.active=1 AND m.active=1 AND m.discovery_active=1 AND m.search_url_template IS NOT NULL AND trim(m.search_url_template)<>'' AND NOT EXISTS (SELECT 1 FROM product_sources s WHERE s.product_id=p.id AND s.merchant_id=m.id AND s.active=1)""" args=[] if product_id is not None: sql+=' AND p.id=?';args.append(int(product_id)) # Sağlıklı mağazalar önce. Eski davranışın aksine LIMIT SQL'de küçük tutulmaz; # cooldown satırlarını atladıktan sonra gerçek deneme bütçesini Python uygular. sql+=""" ORDER BY CASE WHEN mah.blocked_until IS NOT NULL AND mah.blocked_until>CURRENT_TIMESTAMP THEN 1 ELSE 0 END, CASE WHEN lower(m.domain)='hepsiburada.com' THEN 0 WHEN lower(m.domain)='mediamarkt.com.tr' THEN 1 WHEN lower(m.domain)='vatanbilgisayar.com' THEN 2 WHEN lower(m.domain)='teknosa.com' THEN 3 ELSE 8 END, m.trust_score DESC,p.updated_at DESC,p.id DESC,m.id LIMIT 250""" rows=c.execute(sql,args).fetchall() attempted=auto=queued=failed=0;skipped_cooldown=0 for row in rows: if attempted>=max(1,int(limit)):break merchant_id=int(row['merchant_id']) available=merchant_available(c,merchant_id) if not available and not force_access_probe: skipped_cooldown+=1 log_event(c,'merchant.access','debug','Tam-varyant keşifte cooldown mağaza bütçe tüketmeden atlandı',run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id,details={'query':product_query(dict(row),include_price_word=False),'blocked_until':row['blocked_until']}) continue if not available and force_access_probe: log_event(c,'merchant.access','info','Tam-varyant sıkı kurtarma: cooldown mağazaya tek kontrollü deneme',run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id) attempted+=1 product=dict(row);merchant=c.execute('SELECT * FROM merchants WHERE id=?',(merchant_id,)).fetchone();best=None try: search_url=make_search_url(row,family=False) if not same_domain(search_url,row['domain']):raise ValueError('Arama URL şablonu mağaza domainiyle eşleşmiyor') log_event(c,'discovery.exact','info','Tam varyant mağaza araması',run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id,details={'query':product_query(product,include_price_word=False),'search_url':search_url,'attempt':attempted,'budget':limit}) search_html,search_final,_=fetch_page(session,search_url,browser,headers={'User-Agent':UA,'Accept-Language':'tr-TR,tr;q=0.9,en;q=0.7'},run_id=run_id,c=c,product_id=row['product_id'],merchant_id=merchant_id,context='discovery.exact-search') merchant_ok(c,merchant_id) links=candidate_links(search_html,search_final,row['domain'],product)[:6] if not links:raise ValueError('Tam varyant aramasında uygun ürün bağlantısı bulunamadı') access_failures=0 for u in links: try: result=consider_page(c,session,browser,product,u,'merchant_exact_search',run_id,merchant) rank={'auto':3,'candidate':2,'low':1}.get(result[0],0) if best is None or (rank,result[1])>(best[0],best[1]):best=(rank,result[1],result) if result[0]=='auto':break except Exception as e: if _is_access_failure(e): access_failures+=1 if access_failures>=2: merchant_fail(c,merchant_id,e,run_id=run_id,product_id=row['product_id']) break log_event(c,'discovery.exact-skip','debug',str(e),run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id,details={'url':u}) if not best:raise ValueError('Tam varyant aramasında doğrulanabilir ürün sayfası bulunamadı') status,conf,offer,u,m=best[2] if status=='auto': auto+=1 # Bootstrap'ın hedefi bir doğrulanmış fiyat ise mağaza bütçesini hemen bırak. if product_id is not None:break elif status=='candidate':queued+=1 else:failed+=1 except Exception as e: failed+=1 if _is_access_failure(e):merchant_fail(c,merchant_id,e,run_id=run_id,product_id=row['product_id']) note_error(f"ERR-MAGAZA-EXACT {row['merchant']} | {product_query(product,include_price_word=False)} | {e}") log_event(c,'discovery.exact','error',str(e),run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id,details={'query':product_query(product,include_price_word=False)}) time.sleep(.03 if product_id is not None else .08) log_event(c,'discovery.exact','info','Tam varyant mağaza taraması tamamlandı',run_type='discovery',run_id=run_id,product_id=product_id,details={'attempted':attempted,'auto':auto,'candidate':queued,'failed':failed,'cooldown_skipped_without_budget':skipped_cooldown,'budget':limit}) return attempted,auto,queued,failed def run_known_merchants(c,session,browser,run_id,limit=30,product_id=None,force_access_probe=False): if product_id is not None: return run_known_merchants_exact(c,session,browser,run_id,limit=limit,product_id=product_id,force_access_probe=force_access_probe) sql="""SELECT p.id product_id,p.name product_name,p.name,p.brand,p.model,p.variant,p.condition_type,p.capacity_gb,p.color,p.ean,m.id merchant_id,m.name merchant,m.domain,m.trust_score,m.search_url_template FROM products p CROSS JOIN merchants m WHERE p.active=1 AND m.active=1 AND m.discovery_active=1 AND m.search_url_template IS NOT NULL AND NOT EXISTS (SELECT 1 FROM product_sources s WHERE s.product_id=p.id AND s.merchant_id=m.id AND s.active=1)""" args=[] if product_id is not None:sql+=' AND p.id=?';args.append(int(product_id)) # Aile bazlı önbellekleme için biraz daha fazla satır al; mağaza isteği sayısı yine grup sayısıyla sınırlıdır. sql+=" ORDER BY CASE WHEN lower(m.domain)='hepsiburada.com' THEN 0 WHEN lower(m.domain)='mediamarkt.com.tr' THEN 1 WHEN lower(m.domain)='vatanbilgisayar.com' THEN 2 WHEN lower(m.domain)='teknosa.com' THEN 3 ELSE 9 END,p.updated_at DESC,p.id DESC,m.id LIMIT ?";args.append(max(limit,limit*4)) rows=c.execute(sql,args).fetchall();auto=queued=failed=0 groups={} for row in rows: key=(int(row['merchant_id']),product_family_key(dict(row))) groups.setdefault(key,[]).append(row) if sum(len(v) for v in groups.values())>=max(limit,limit*4):break processed=0 for (merchant_id,family_key),family_rows in groups.items(): if processed>=limit:break processed+=1 merchant=c.execute('SELECT * FROM merchants WHERE id=?',(merchant_id,)).fetchone() if not merchant:continue if not merchant_available(c,merchant_id) and not force_access_probe: log_event(c,'merchant.access','debug','Mağaza erişim cooldown nedeniyle bu çalışmada atlandı',run_type='discovery',run_id=run_id,merchant_id=merchant_id,details={'family':family_key,'variants':len(family_rows)}) failed+=len(family_rows);continue if not merchant_available(c,merchant_id) and force_access_probe: log_event(c,'merchant.access','info','Sıkı kurtarma: cooldown içindeki mağazaya tek kontrollü yeniden erişim deneniyor',run_type='discovery',run_id=run_id,merchant_id=merchant_id,details={'family':family_key,'variants':len(family_rows)}) base=family_rows[0] family_product=dict(base);family_product['capacity_gb']=None;family_product['color']='';family_product['ean']='' try: search_url=make_search_url(base,family=True) if not same_domain(search_url,base['domain']):raise ValueError('Arama URL şablonu mağaza domainiyle eşleşmiyor') search_html,search_final,_=fetch_page(session,search_url,browser,headers={'User-Agent':UA,'Accept-Language':'tr-TR,tr;q=0.9,en;q=0.7'},run_id=run_id,c=c,product_id=base['product_id'],merchant_id=merchant_id,context='discovery.merchant-search') merchant_ok(c,merchant_id) family_links=candidate_links(search_html,search_final,base['domain'],family_product)[:24] if not family_links:raise ValueError('Mağaza aramasında ürün ailesi bağlantısı bulunamadı') stop_group=False;group_access_failures=0 for row_index,row in enumerate(family_rows): product=dict(row);best=None # Aynı arama HTML'ini tüm GB/renk varyantlarında tekrar kullan. Böylece 10 varyant = 1 mağaza araması. variant_links=candidate_links(search_html,search_final,row['domain'],product)[:10] links=[] for u in variant_links+family_links: if u not in links:links.append(u) for u in links[:10]: try: result=consider_page(c,session,browser,product,u,'merchant_search',run_id,merchant);rank={'auto':3,'candidate':2,'low':1}.get(result[0],0) if best is None or (rank,result[1])>(best[0],best[1]):best=(rank,result[1],result) if result[0]=='auto':break except Exception as e: if _is_access_failure(e): group_access_failures+=1 if group_access_failures>=2: merchant_fail(c,merchant_id,e,run_id=run_id,product_id=row['product_id']);stop_group=True log_event(c,'discovery.skip','debug',str(e),run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id,details={'url':u,'family':family_key,'access_failures':group_access_failures}) if stop_group:break if stop_group: failed+=len(family_rows)-row_index log_event(c,'merchant.access','warning','Ürün sayfaları erişim engeli verdi; aynı ailenin kalan varyantlarında mağaza zorlanmadı',run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id,details={'family':family_key,'remaining_variants':len(family_rows)-row_index}) break if not best: failed+=1 log_event(c,'discovery','warning','Ürün ailesi bulundu ancak bu GB/renk varyantı doğrulanamadı',run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id,details={'family':family_key}) continue status,conf,offer,u,m=best[2] if status=='auto':auto+=1 elif status=='candidate':queued+=1 else:failed+=1 except Exception as e: failed+=len(family_rows) if _is_access_failure(e):merchant_fail(c,merchant_id,e,run_id=run_id,product_id=base['product_id']) note_error(f"ERR-MAGAZA {base['merchant']} | {product_family_query(base)} | {e}") log_event(c,'discovery','error',str(e),run_type='discovery',run_id=run_id,product_id=base['product_id'],merchant_id=merchant_id,details={'family':family_key,'variant_count':len(family_rows)}) time.sleep(.03 if product_id is not None else .08) return sum(len(v) for v in list(groups.values())[:processed]),auto,queued,failed def search_blocked_domain_evidence(c,session,product,method_preference,run_id,max_domains=5): # v0.3.17: Harici arama motorları varsayılan olarak tamamen kapalıdır. if EXTERNAL_SEARCH_DISABLED or DDG_DISABLED or not provider_available(c,'duckduckgo'): return 0 rows=c.execute('''SELECT DISTINCT m.domain FROM product_sources s JOIN merchants m ON m.id=s.merchant_id WHERE s.product_id=? AND s.active=1 AND (s.last_status='error' OR s.consecutive_failures>0) ORDER BY s.consecutive_failures DESC LIMIT ?''',(product['product_id'],max_domains)).fetchall() saved=0 for br in rows: domain=(br['domain'] or '').strip() if not domain:continue query=f"site:{domain} {product_query(product,include_price_word=True)}" try: results=web_search_ddg(session,query,8,c=c,run_id=run_id,product_id=product['product_id']) for result in results: if host_domain(result.get('url') or '')!=host_domain('https://'+domain):continue ev=evaluate_search_result(product,result) if not ev:continue upsert_indirect_offer(c,product,result,ev,'duckduckgo',run_id);saved+=1;break except Exception as e: log_event(c,'discovery.indirect','debug',f'{domain} indeks fiyat araması sonuç vermedi: {e}',run_type='discovery',run_id=run_id,product_id=product['product_id'],details={'domain':domain}) return saved def run_web(c,session,browser,run_id,limit=20,product_id=None,target_sources=None): target=max(1,int(target_sources or TARGET_SOURCES)) sql="""SELECT p.id product_id,p.name product_name,p.name,p.brand,p.model,p.variant,p.condition_type,p.capacity_gb,p.color,p.ean, (SELECT COUNT(*) FROM product_sources s WHERE s.product_id=p.id AND s.active=1) source_count FROM products p WHERE p.active=1 AND (SELECT COUNT(*) FROM product_sources s WHERE s.product_id=p.id AND s.active=1) < ?""" args=[target] if product_id is not None:sql+=' AND p.id=?';args.append(int(product_id)) sql+=' ORDER BY source_count ASC,p.updated_at DESC,p.id DESC LIMIT ?';args.append(limit) rows=c.execute(sql,args).fetchall();auto=queued=failed=0 for row in rows: product=dict(row);need=max(1,target-int(row['source_count'] or 0)) try: # Önce daha önce engel veren kaynakların domaininde indeks fiyatı ara; ucuz mağazayı 403 yüzünden tamamen kaybetme. indirect_saved=search_blocked_domain_evidence(c,session,product,'duckduckgo',run_id) indirect_any=indirect_saved if indirect_saved:queued+=indirect_saved added=0 # v0.3.9: Yeni/eksik ürün hızlı hattı. Genel web sonucunu beklemeden # güvenilir perakende domainlerinde site: araması yap. Hepsiburada ilk. if product_id is not None and need>0: for pdomain in PRIORITY_DISCOVERY_DOMAINS: if added>=need:break try: q=f"site:{pdomain} {product_query(product,include_price_word=False)}" pmethod='duckduckgo' pres=web_search_ddg(session,q,6,c=c,run_id=run_id,product_id=row['product_id']) for pr in pres: if host_domain(pr.get('url') or '')!=pdomain:continue try: st,cf,of,fu,me=consider_page(c,session,browser,product,pr['url'],'priority_'+pmethod,run_id,None) if st=='auto':auto+=1;added+=1;break elif st=='candidate':queued+=1 except Exception as pe: log_event(c,'discovery.priority','debug',str(pe),run_type='discovery',run_id=run_id,product_id=row['product_id'],details={'domain':pdomain,'url':pr.get('url')}) except Exception as pe: log_event(c,'discovery.priority','debug',f'{pdomain} site araması başarısız: {pe}',run_type='discovery',run_id=run_id,product_id=row['product_id']) results,method=web_search(session,product,limit=max(8,min(16,max(1,need-added)*4)),c=c,run_id=run_id) if not results: log_event(c,'discovery.search','warning','Genel web araması kullanılamadı; bilinen mağaza aramasıyla devam edildi',run_type='discovery',run_id=run_id,product_id=row['product_id'],details={'brave_disabled':True,'duckduckgo_disabled':DDG_DISABLED}) if not c.execute("SELECT 1 FROM discovery_candidates WHERE product_id=? AND status='pending' LIMIT 1",(row['product_id'],)).fetchone():failed+=1 continue log_event(c,'discovery.search','info','Web arama sonuçları alındı',run_type='discovery',run_id=run_id,product_id=row['product_id'],details={'engine':method,'count':len(results),'query':product_query(product,True),'priority_product':product_id is not None}) for result in results: u=result['url'];domain=host_domain(u) if not domain or domain in SEARCH_BLOCKED_HOSTS:continue existing=c.execute('SELECT s.last_status,s.consecutive_failures FROM product_sources s WHERE s.product_id=? AND s.url=? AND s.active=1 LIMIT 1',(row['product_id'],u)).fetchone() if existing and existing['last_status']=='ok' and int(existing['consecutive_failures'] or 0)==0:continue try: status,conf,offer,final_url,merchant=consider_page(c,session,browser,product,u,'web_'+method,run_id,None) if status=='auto':auto+=1;added+=1 elif status=='candidate':queued+=1 if added>=need:break except Exception as e: indirect=None if setting_bool(c,'indirect_search_prices',True): try:indirect=evaluate_search_result(product,result) except Exception:indirect=None if indirect: try: upsert_indirect_offer(c,product,result,indirect,method,run_id);queued+=1;indirect_any+=1 note_error(f"INDEX-WEB {row['product_name']} | {domain} | {indirect.price} TRY | doğrudan erişim: {e}") except Exception as ie:note_error(f"SKIP-WEB {row['product_name']} | {domain} | {e} | indeks kaydı: {ie}") else: note_error(f"SKIP-WEB {row['product_name']} | {domain} | {e}") log_event(c,'discovery.skip','warning',str(e),run_type='discovery',run_id=run_id,product_id=row['product_id'],details={'domain':domain,'url':u,'indirect_saved':bool(indirect)}) time.sleep(.03 if product_id is not None else .10) if added==0 and indirect_any==0 and not c.execute("SELECT 1 FROM discovery_candidates WHERE product_id=? AND status='pending' LIMIT 1",(row['product_id'],)).fetchone():failed+=1 except Exception as e:failed+=1;note_error(f"ERR-WEB {row['product_name']} | {e}");log_event(c,'discovery','error',str(e),run_type='discovery',run_id=run_id,product_id=row['product_id']) time.sleep(.05 if product_id is not None else .18) return len(rows),auto,queued,failed def run(limit=30,web_limit=20,product_id=None,target_sources=None,browser_max_pages=None,force_access_probe=False,browser_timeout_ms=None,web_first=False,sitemap_first=False): c=connect();c.execute("UPDATE indirect_offers SET active=0 WHERE expires_at IS NOT NULL AND expires_at < CURRENT_TIMESTAMP");run_id=c.execute("INSERT INTO discovery_runs(status) VALUES('running')").lastrowid;c.commit();session=requests.Session();session.headers.update({'User-Agent':UA,'Accept-Language':'tr-TR,tr;q=0.9,en;q=0.7','Accept':'text/html,application/xhtml+xml'}) max_pages=int(browser_max_pages if browser_max_pages is not None else setting_int(c,'browser_max_pages',4)) timeout_ms=max(5000,int(browser_timeout_ms if browser_timeout_ms is not None else setting_int(c,'browser_timeout_ms',18000))) browser=HybridBrowser(enabled=setting(c,'fetch_mode','hybrid')=='hybrid' and max_pages>0,max_pages=max_pages,timeout_ms=timeout_ms,user_agent=UA) log_event(c,'discovery','info','Öncelikli ürün keşfi başladı' if product_id is not None else 'Kaynak keşfi başladı',run_type='discovery',run_id=run_id,product_id=product_id,details={'fetch_mode':setting(c,'fetch_mode','hybrid'),'browser_max_pages':browser.max_pages,'priority_product':product_id,'target_sources':target_sources or TARGET_SOURCES},commit=True) try: family_seed=(0,0,0);bridge_seed=(0,0,0);sitemap_seed=(0,0,0,0) if web_first and product_id is not None: # v0.3.14: Eski çalışan sıra geri getirildi. Önce tam varyant sorgusuyla # sağlıklı mağazalar aranır. Kaynak bulma geniştir; kabul filtresi ise # consider_page içinde model+GB+renk+kondisyon bakımından katıdır. if sitemap_first and setting_bool(c,'sitemap_discovery_enabled',True): sitemap_seed=run_sitemap_merchants(c,session,browser,run_id,product_id,merchant_limit=min(3,max(2,int(limit)//3)),url_limit=2) verified_price_sources=int(c.execute("SELECT COUNT(*) FROM product_sources ps JOIN products p ON p.id=ps.product_id WHERE ps.product_id=? AND ps.active=1 AND ps.condition_status='verified' AND ps.detected_condition=p.condition_type AND ps.last_price IS NOT NULL",(int(product_id),)).fetchone()[0]) if verified_price_sources