#!/usr/bin/env python3
from __future__ import annotations
import gzip, hashlib, os, re, time
from pathlib import Path
from urllib.parse import urlparse
import xml.etree.ElementTree as ET

from matching import norm, tokens, expected_model_text, model_compatible, color_matches
from dbutil import log_event

CACHE_DIR=Path(os.environ.get('FIYAT_RADAR_SITEMAP_CACHE','/var/lib/fiyat-radar/sitemap-cache'))
UA=os.environ.get('FIYAT_RADAR_UA','Mozilla/5.0 (compatible; FiyatRadar/0.3; +local-price-monitor)')
MAX_XML_BYTES=int(os.environ.get('FIYAT_RADAR_SITEMAP_MAX_BYTES','6291456'))
CACHE_SECONDS=int(os.environ.get('FIYAT_RADAR_SITEMAP_CACHE_SECONDS','43200'))


def _domain(v):
    try:return str(v['domain'] or '').strip().lower().removeprefix('www.')
    except Exception:return str(getattr(v,'domain','') or '').strip().lower().removeprefix('www.')

def _mid(v):
    try:return int(v['id'])
    except Exception:return int(getattr(v,'id',0) or 0)

def _read_limited(resp, limit=MAX_XML_BYTES):
    chunks=[];total=0
    for chunk in resp.iter_content(65536):
        if not chunk:continue
        total+=len(chunk)
        if total>limit:raise ValueError(f'sitemap içerik limiti aşıldı ({limit} byte)')
        chunks.append(chunk)
    raw=b''.join(chunks)
    if (resp.url or '').lower().endswith('.gz') or raw[:2]==b'\x1f\x8b':
        raw=gzip.decompress(raw)
        if len(raw)>limit:raise ValueError('açılmış sitemap içerik limiti aşıldı')
    return raw

def _get(session,url,timeout=(2.0,4.0),max_bytes=MAX_XML_BYTES):
    r=session.get(url,headers={'User-Agent':UA,'Accept':'application/xml,text/xml,text/plain,*/*;q=0.5'},timeout=timeout,allow_redirects=True,stream=True)
    r.raise_for_status()
    return _read_limited(r,max_bytes),r.url

def _robots_sitemaps(session,domain):
    out=[]
    try:
        raw,_=_get(session,f'https://{domain}/robots.txt',max_bytes=512000)
        text=raw.decode('utf-8','replace')
        for m in re.finditer(r'^\s*Sitemap\s*:\s*(https?://\S+)',text,re.I|re.M):
            u=m.group(1).strip()
            if u not in out:out.append(u)
    except Exception:
        pass
    for path in ('/sitemap.xml','/sitemap_index.xml','/sitemap-index.xml'):
        u=f'https://{domain}{path}'
        if u not in out:out.append(u)
    return out[:8]

def _parse_xml_locs(raw):
    try:root=ET.fromstring(raw)
    except Exception:return None,[]
    tag=root.tag.rsplit('}',1)[-1].lower()
    locs=[]
    for el in root.iter():
        if el.tag.rsplit('}',1)[-1].lower()=='loc' and el.text:
            u=el.text.strip()
            if u.startswith(('http://','https://')):locs.append(u)
    return tag,locs

def _child_score(url):
    n=norm(url)
    score=0
    for t,w in (('product',8),('products',8),('urun',8),('urundetay',8),('catalog',3),('phone',2),('telefon',2)):
        if t in n:score+=w
    if any(x in n for x in ('image','video','news','blog','category','kategori')):score-=4
    return score

def _fetch_url_catalog(session,domain):
    roots=_robots_sitemaps(session,domain)
    urls=[];seen=set();children=[]
    for root_url in roots[:3]:
        if root_url in seen:continue
        seen.add(root_url)
        try:
            raw,final=_get(session,root_url)
            kind,locs=_parse_xml_locs(raw)
            if not locs:continue
            if kind=='sitemapindex':
                children.extend(locs)
                break
            else:
                urls.extend(locs)
                if urls: break
        except Exception:
            continue
        if len(urls)>=25000:break
    # Ürün sitemap'leri önce; index adları anlamsızsa ilk birkaç sitemap de denenir.
    ranked=sorted(dict.fromkeys(children),key=lambda u:(_child_score(u),u),reverse=True)
    chosen=ranked[:3]
    if ranked and all(_child_score(u)<=0 for u in chosen): chosen=ranked[:2]
    for child in chosen:
        if child in seen:continue
        seen.add(child)
        try:
            raw,_=_get(session,child)
            kind,locs=_parse_xml_locs(raw)
            if kind!='sitemapindex':urls.extend(locs)
        except Exception:
            continue
        if len(urls)>=30000:break
    # Aynı domain ve http(s) URL'leri tut.
    clean=[];seen_u=set()
    for u in urls:
        if u in seen_u:continue
        try:
            h=(urlparse(u).hostname or '').lower().removeprefix('www.')
            if not (h==domain or h.endswith('.'+domain)):continue
        except Exception:continue
        seen_u.add(u);clean.append(u)
        if len(clean)>=30000:break
    return clean,roots,chosen

def _cache_file(mid,domain):
    CACHE_DIR.mkdir(parents=True,exist_ok=True)
    digest=hashlib.sha1(domain.encode()).hexdigest()[:10]
    return CACHE_DIR/f'{mid}-{digest}.urls'

def _load_cached(path):
    try:
        if not path.exists() or time.time()-path.stat().st_mtime>CACHE_SECONDS:return None
        return [x.strip() for x in path.read_text('utf-8',errors='ignore').splitlines() if x.strip()]
    except Exception:return None

def _save_cache(path,urls):
    tmp=path.with_suffix(path.suffix+'.tmp')
    tmp.write_text('\n'.join(urls)+'\n',encoding='utf-8')
    tmp.replace(path)

def refresh_catalog(c,session,merchant,run_id=None,product_id=None,force=False):
    mid=_mid(merchant);domain=_domain(merchant);path=_cache_file(mid,domain)
    cached=None if force else _load_cached(path)
    if cached is not None:
        return cached,False
    status='error';err=None;urls=[];roots=[];chosen=[]
    try:
        urls,roots,chosen=_fetch_url_catalog(session,domain)
        if urls:
            _save_cache(path,urls);status='ok'
        else:err='Sitemap/robots içinde ürün URL kataloğu bulunamadı'
    except Exception as e:err=str(e)
    try:
        c.execute("""INSERT INTO merchant_sitemap_cache(merchant_id,domain,cache_path,status,url_count,fetched_at,expires_at,last_error,updated_at)
                     VALUES(?,?,?,?,?,CURRENT_TIMESTAMP,datetime('now','+12 hours'),?,CURRENT_TIMESTAMP)
                     ON CONFLICT(merchant_id) DO UPDATE SET domain=excluded.domain,cache_path=excluded.cache_path,status=excluded.status,url_count=excluded.url_count,fetched_at=CURRENT_TIMESTAMP,expires_at=datetime('now','+12 hours'),last_error=excluded.last_error,updated_at=CURRENT_TIMESTAMP""",
                  (mid,domain,str(path),status,len(urls),err))
        log_event(c,'discovery.sitemap','info' if status=='ok' else 'debug',f'Sitemap kataloğu {status}: {domain} ({len(urls)} URL)',run_type='discovery',run_id=run_id,product_id=product_id,merchant_id=mid,details={'roots':roots[:4],'children':chosen[:6],'cache':str(path),'error':err})
        c.commit()
    except Exception:pass
    return urls,True

def _capacity_tokens(cap):
    try:v=int(cap or 0)
    except Exception:return []
    if not v:return []
    if v>=1024 and v%1024==0:
        return [f'{v//1024}tb',f'{v}gb',str(v)]
    return [f'{v}gb',str(v)]

def rank_urls(product,urls,limit=8):
    model=expected_model_text(product)
    mt=[t for t in tokens(model) if len(t)>=2]
    brand=norm(str(product.get('brand') or '')) if hasattr(product,'get') else ''
    cap_tokens=_capacity_tokens(product.get('capacity_gb') if hasattr(product,'get') else None)
    color=str(product.get('color') or '') if hasattr(product,'get') else ''
    scored=[]
    for u in urls:
        if not model_compatible(product,u):continue
        n=norm(u);compact=n.replace(' ','')
        # Marka/model çekirdeği URL'de bulunmuyorsa aday olmasın.
        hits=sum(1 for t in mt if (' '+t+' ') in (' '+n+' ') or t in compact)
        min_hits=1 if len(mt)<=1 else max(2,min(len(mt),3))
        if hits<min_hits:continue
        score=hits*6
        if brand and brand in n:score+=2
        if cap_tokens:
            if any(tok in compact for tok in cap_tokens):score+=9
            else:score-=2
        if color:
            if color_matches(color,u):score+=5
        if any(x in n for x in ('aksesuar','kilif','case','ekran koruyucu','sarj','kablo')):score-=12
        if score>0:scored.append((score,-len(u),u))
    scored.sort(reverse=True)
    return [u for _,__,u in scored[:max(1,int(limit))]]

def discover_urls(c,session,merchant,product,run_id=None,limit=8,force_refresh=False):
    urls,_=refresh_catalog(c,session,merchant,run_id=run_id,product_id=product.get('product_id') if hasattr(product,'get') else None,force=force_refresh)
    out=rank_urls(product,urls,limit=limit)
    if out:
        try:log_event(c,'discovery.sitemap','info',f'Sitemap exact aday: {len(out)}',run_type='discovery',run_id=run_id,product_id=product.get('product_id'),merchant_id=_mid(merchant),details={'urls':out[:5]})
        except Exception:pass
    return out