### v0.1.9 run_known_merchants def run_known_merchants(c,session,browser,run_id,limit=30,product_id=None): sql="""SELECT p.id product_id,p.name product_name,p.name,p.brand,p.model,p.variant,p.capacity_gb,p.color,p.ean,m.id merchant_id,m.name merchant,m.domain,m.trust_score,m.search_url_template FROM products p CROSS JOIN merchants m WHERE p.active=1 AND m.active=1 AND m.discovery_active=1 AND m.search_url_template IS NOT NULL AND NOT EXISTS (SELECT 1 FROM product_sources s WHERE s.product_id=p.id AND s.merchant_id=m.id AND s.active=1)""" args=[] if product_id is not None:sql+=' AND p.id=?';args.append(int(product_id)) sql+=' ORDER BY p.updated_at DESC,p.id DESC,m.id LIMIT ?';args.append(limit) rows=c.execute(sql,args).fetchall();auto=queued=failed=0 for row in rows: try: search_url=make_search_url(row) if not same_domain(search_url,row['domain']):raise ValueError('Arama URL şablonu mağaza domainiyle eşleşmiyor') search_html,search_final,_=fetch_page(session,search_url,browser,headers={'User-Agent':UA,'Accept-Language':'tr-TR,tr;q=0.9,en;q=0.7'},run_id=run_id,c=c,product_id=row['product_id'],merchant_id=row['merchant_id']) product=dict(row);merchant=c.execute('SELECT * FROM merchants WHERE id=?',(row['merchant_id'],)).fetchone();best=None for u in candidate_links(search_html,search_final,row['domain'],product)[:8]: try: result=consider_page(c,session,browser,product,u,'merchant_search',run_id,merchant);rank={'auto':3,'candidate':2,'low':1}.get(result[0],0) if best is None or (rank,result[1])>(best[0],best[1]):best=(rank,result[1],result) if result[0]=='auto':break except Exception as e:log_event(c,'discovery.skip','debug',str(e),run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=row['merchant_id'],details={'url':u}) if not best:raise ValueError('Uygun ürün sayfası bulunamadı') status,conf,offer,u,m=best[2] if status=='auto':auto+=1 elif status=='candidate':queued+=1 else:failed+=1 except Exception as e:failed+=1;note_error(f"ERR-MAGAZA {row['merchant']} | {row['product_name']} | {e}");log_event(c,'discovery','error',str(e),run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=row['merchant_id']) time.sleep(.05 if product_id is not None else .12) return len(rows),auto,queued,failed ### v0.3.13 run_known_merchants def run_known_merchants(c,session,browser,run_id,limit=30,product_id=None,force_access_probe=False): sql="""SELECT p.id product_id,p.name product_name,p.name,p.brand,p.model,p.variant,p.condition_type,p.capacity_gb,p.color,p.ean,m.id merchant_id,m.name merchant,m.domain,m.trust_score,m.search_url_template FROM products p CROSS JOIN merchants m WHERE p.active=1 AND m.active=1 AND m.discovery_active=1 AND m.search_url_template IS NOT NULL AND NOT EXISTS (SELECT 1 FROM product_sources s WHERE s.product_id=p.id AND s.merchant_id=m.id AND s.active=1)""" args=[] if product_id is not None:sql+=' AND p.id=?';args.append(int(product_id)) # Aile bazlı önbellekleme için biraz daha fazla satır al; mağaza isteği sayısı yine grup sayısıyla sınırlıdır. sql+=" ORDER BY CASE WHEN lower(m.domain)='hepsiburada.com' THEN 0 WHEN lower(m.domain)='mediamarkt.com.tr' THEN 1 WHEN lower(m.domain)='vatanbilgisayar.com' THEN 2 WHEN lower(m.domain)='teknosa.com' THEN 3 ELSE 9 END,p.updated_at DESC,p.id DESC,m.id LIMIT ?";args.append(max(limit,limit*4)) rows=c.execute(sql,args).fetchall();auto=queued=failed=0 groups={} for row in rows: key=(int(row['merchant_id']),product_family_key(dict(row))) groups.setdefault(key,[]).append(row) if sum(len(v) for v in groups.values())>=max(limit,limit*4):break processed=0 for (merchant_id,family_key),family_rows in groups.items(): if processed>=limit:break processed+=1 merchant=c.execute('SELECT * FROM merchants WHERE id=?',(merchant_id,)).fetchone() if not merchant:continue if not merchant_available(c,merchant_id) and not force_access_probe: log_event(c,'merchant.access','debug','Mağaza erişim cooldown nedeniyle bu çalışmada atlandı',run_type='discovery',run_id=run_id,merchant_id=merchant_id,details={'family':family_key,'variants':len(family_rows)}) failed+=len(family_rows);continue if not merchant_available(c,merchant_id) and force_access_probe: log_event(c,'merchant.access','info','Sıkı kurtarma: cooldown içindeki mağazaya tek kontrollü yeniden erişim deneniyor',run_type='discovery',run_id=run_id,merchant_id=merchant_id,details={'family':family_key,'variants':len(family_rows)}) base=family_rows[0] family_product=dict(base);family_product['capacity_gb']=None;family_product['color']='';family_product['ean']='' try: search_url=make_search_url(base,family=True) if not same_domain(search_url,base['domain']):raise ValueError('Arama URL şablonu mağaza domainiyle eşleşmiyor') search_html,search_final,_=fetch_page(session,search_url,browser,headers={'User-Agent':UA,'Accept-Language':'tr-TR,tr;q=0.9,en;q=0.7'},run_id=run_id,c=c,product_id=base['product_id'],merchant_id=merchant_id,context='discovery.merchant-search') merchant_ok(c,merchant_id) family_links=candidate_links(search_html,search_final,base['domain'],family_product)[:24] if not family_links:raise ValueError('Mağaza aramasında ürün ailesi bağlantısı bulunamadı') stop_group=False;group_access_failures=0 for row_index,row in enumerate(family_rows): product=dict(row);best=None # Aynı arama HTML'ini tüm GB/renk varyantlarında tekrar kullan. Böylece 10 varyant = 1 mağaza araması. variant_links=candidate_links(search_html,search_final,row['domain'],product)[:10] links=[] for u in variant_links+family_links: if u not in links:links.append(u) for u in links[:10]: try: result=consider_page(c,session,browser,product,u,'merchant_search',run_id,merchant);rank={'auto':3,'candidate':2,'low':1}.get(result[0],0) if best is None or (rank,result[1])>(best[0],best[1]):best=(rank,result[1],result) if result[0]=='auto':break except Exception as e: if _is_access_failure(e): group_access_failures+=1 if group_access_failures>=2: merchant_fail(c,merchant_id,e,run_id=run_id,product_id=row['product_id']);stop_group=True log_event(c,'discovery.skip','debug',str(e),run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id,details={'url':u,'family':family_key,'access_failures':group_access_failures}) if stop_group:break if stop_group: failed+=len(family_rows)-row_index log_event(c,'merchant.access','warning','Ürün sayfaları erişim engeli verdi; aynı ailenin kalan varyantlarında mağaza zorlanmadı',run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id,details={'family':family_key,'remaining_variants':len(family_rows)-row_index}) break if not best: failed+=1 log_event(c,'discovery','warning','Ürün ailesi bulundu ancak bu GB/renk varyantı doğrulanamadı',run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id,details={'family':family_key}) continue status,conf,offer,u,m=best[2] if status=='auto':auto+=1 elif status=='candidate':queued+=1 else:failed+=1 except Exception as e: failed+=len(family_rows) if _is_access_failure(e):merchant_fail(c,merchant_id,e,run_id=run_id,product_id=base['product_id']) note_error(f"ERR-MAGAZA {base['merchant']} | {product_family_query(base)} | {e}") log_event(c,'discovery','error',str(e),run_type='discovery',run_id=run_id,product_id=base['product_id'],merchant_id=merchant_id,details={'family':family_key,'variant_count':len(family_rows)}) time.sleep(.03 if product_id is not None else .08) return sum(len(v) for v in list(groups.values())[:processed]),auto,queued,failed ### v0.3.14 exact path upsert_candidate(c,product['product_id'],merchant['id'],final_url,offer,dec.score,'auto_approved');c.commit();return 'auto',dec.score,offer,final_url,merchant if dec.status=='candidate':upsert_candidate(c,product['product_id'],merchant['id'],final_url,offer,dec.score,'pending');c.commit();return 'candidate',dec.score,offer,final_url,merchant return 'low',dec.score,offer,final_url,merchant def run_known_merchants_exact(c,session,browser,run_id,limit=6,product_id=None,force_access_probe=False): """v0.3.14: v0.1.x'in çalışan *tam varyant* mağaza keşfini geri getir. Kritik farklar: - Arama sorgusu aile adı değil tam ürün varyantıdır (model + GB + renk + kondisyon). - Cooldown'daki mağaza `limit` bütçesini tüketmez. Yalnız gerçek HTTP/Chromium arama isteği `attempted` sayacını artırır. - Bulunan sayfa yine güncel katı model/GB/renk/kondisyon doğrulamasından geçer; eski sürümün gevşek kabul davranışı geri getirilmez. """ sql="""SELECT p.id product_id,p.name product_name,p.name,p.brand,p.model,p.variant,p.category, p.condition_type,p.capacity_gb,p.color,p.ean,p.family_key, m.id merchant_id,m.name merchant,m.domain,m.trust_score,m.search_url_template, mah.status access_status,mah.blocked_until,COALESCE(mah.consecutive_failures,0) access_failures FROM products p CROSS JOIN merchants m LEFT JOIN merchant_access_health mah ON mah.merchant_id=m.id WHERE p.active=1 AND m.active=1 AND m.discovery_active=1 AND m.search_url_template IS NOT NULL AND trim(m.search_url_template)<>'' AND NOT EXISTS (SELECT 1 FROM product_sources s WHERE s.product_id=p.id AND s.merchant_id=m.id AND s.active=1)""" args=[] if product_id is not None: sql+=' AND p.id=?';args.append(int(product_id)) # Sağlıklı mağazalar önce. Eski davranışın aksine LIMIT SQL'de küçük tutulmaz; # cooldown satırlarını atladıktan sonra gerçek deneme bütçesini Python uygular. sql+=""" ORDER BY CASE WHEN mah.blocked_until IS NOT NULL AND mah.blocked_until>CURRENT_TIMESTAMP THEN 1 ELSE 0 END, CASE WHEN lower(m.domain)='hepsiburada.com' THEN 0 WHEN lower(m.domain)='mediamarkt.com.tr' THEN 1 WHEN lower(m.domain)='vatanbilgisayar.com' THEN 2 WHEN lower(m.domain)='teknosa.com' THEN 3 ELSE 8 END, m.trust_score DESC,p.updated_at DESC,p.id DESC,m.id LIMIT 250""" rows=c.execute(sql,args).fetchall() attempted=auto=queued=failed=0;skipped_cooldown=0 for row in rows: if attempted>=max(1,int(limit)):break merchant_id=int(row['merchant_id']) available=merchant_available(c,merchant_id) if not available and not force_access_probe: skipped_cooldown+=1 log_event(c,'merchant.access','debug','Tam-varyant keşifte cooldown mağaza bütçe tüketmeden atlandı',run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id,details={'query':product_query(dict(row),include_price_word=False),'blocked_until':row['blocked_until']}) continue if not available and force_access_probe: log_event(c,'merchant.access','info','Tam-varyant sıkı kurtarma: cooldown mağazaya tek kontrollü deneme',run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id) attempted+=1 product=dict(row);merchant=c.execute('SELECT * FROM merchants WHERE id=?',(merchant_id,)).fetchone();best=None try: search_url=make_search_url(row,family=False) if not same_domain(search_url,row['domain']):raise ValueError('Arama URL şablonu mağaza domainiyle eşleşmiyor') log_event(c,'discovery.exact','info','Tam varyant mağaza araması',run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id,details={'query':product_query(product,include_price_word=False),'search_url':search_url,'attempt':attempted,'budget':limit}) search_html,search_final,_=fetch_page(session,search_url,browser,headers={'User-Agent':UA,'Accept-Language':'tr-TR,tr;q=0.9,en;q=0.7'},run_id=run_id,c=c,product_id=row['product_id'],merchant_id=merchant_id,context='discovery.exact-search') merchant_ok(c,merchant_id) links=candidate_links(search_html,search_final,row['domain'],product)[:6] if not links:raise ValueError('Tam varyant aramasında uygun ürün bağlantısı bulunamadı') access_failures=0 for u in links: try: result=consider_page(c,session,browser,product,u,'merchant_exact_search',run_id,merchant) rank={'auto':3,'candidate':2,'low':1}.get(result[0],0) if best is None or (rank,result[1])>(best[0],best[1]):best=(rank,result[1],result) if result[0]=='auto':break except Exception as e: if _is_access_failure(e): access_failures+=1 if access_failures>=2: merchant_fail(c,merchant_id,e,run_id=run_id,product_id=row['product_id']) break log_event(c,'discovery.exact-skip','debug',str(e),run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id,details={'url':u}) if not best:raise ValueError('Tam varyant aramasında doğrulanabilir ürün sayfası bulunamadı') status,conf,offer,u,m=best[2] if status=='auto': auto+=1 # Bootstrap'ın hedefi bir doğrulanmış fiyat ise mağaza bütçesini hemen bırak. if product_id is not None:break elif status=='candidate':queued+=1 else:failed+=1 except Exception as e: failed+=1 if _is_access_failure(e):merchant_fail(c,merchant_id,e,run_id=run_id,product_id=row['product_id']) note_error(f"ERR-MAGAZA-EXACT {row['merchant']} | {product_query(product,include_price_word=False)} | {e}") log_event(c,'discovery.exact','error',str(e),run_type='discovery',run_id=run_id,product_id=row['product_id'],merchant_id=merchant_id,details={'query':product_query(product,include_price_word=False)}) time.sleep(.03 if product_id is not None else .08) log_event(c,'discovery.exact','info','Tam varyant mağaza taraması tamamlandı',run_type='discovery',run_id=run_id,product_id=product_id,details={'attempted':attempted,'auto':auto,'candidate':queued,'failed':failed,'cooldown_skipped_without_budget':skipped_cooldown,'budget':limit}) return attempted,auto,queued,failed