<?php
/**
 * Universal News Scraper (v2 - clean output)
 * যেকোনো নিউজ সাইট থেকে title, content, image, meta data নিয়ে আসে
 * v2: nav/menu/breadcrumb/related/byline সব strip করে clean output দেয়
 */

class NewsScraper
{
    private $url;
    private $html;
    private $dom;
    private $xpath;
    private $baseHost;

    public function __construct($url)
    {
        $this->url = trim($url);
        $parts = parse_url($this->url);
        $this->baseHost = ($parts['scheme'] ?? 'https') . '://' . ($parts['host'] ?? '');
    }

    private function fetchHtml()
    {
        $ch = curl_init();
        curl_setopt_array($ch, [
            CURLOPT_URL            => $this->url,
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_MAXREDIRS      => 5,
            CURLOPT_TIMEOUT        => 25,
            CURLOPT_CONNECTTIMEOUT => 10,
            CURLOPT_SSL_VERIFYPEER => false,
            CURLOPT_SSL_VERIFYHOST => false,
            CURLOPT_ENCODING       => '',
            CURLOPT_USERAGENT      => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
            CURLOPT_HTTPHEADER     => [
                'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                'Accept-Language: bn,en-US;q=0.9,en;q=0.8',
            ],
        ]);
        $html = curl_exec($ch);
        $err  = curl_error($ch);
        $code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
        curl_close($ch);

        if ($html === false || $code >= 400) {
            throw new Exception("URL fetch failed (HTTP $code) " . $err);
        }
        return $html;
    }

    private function loadDom()
    {
        if ($this->dom) return;
        $this->html = $this->fetchHtml();

        if (!preg_match('/<meta[^>]+charset/i', $this->html)) {
            $this->html = '<meta http-equiv="Content-Type" content="text/html; charset=utf-8">' . $this->html;
        }

        libxml_use_internal_errors(true);
        $this->dom = new DOMDocument();
        $this->dom->loadHTML('<?xml encoding="utf-8" ?>' . $this->html);
        libxml_clear_errors();
        $this->xpath = new DOMXPath($this->dom);

        // Global cleanup: remove site-wide junk before article extraction
        $this->stripGlobalJunk();
    }

    /** Remove header, nav, footer, sidebar, scripts globally from DOM */
    private function stripGlobalJunk()
    {
        $globalKill = [
            "//script", "//style", "//noscript", "//iframe", "//form",
            "//header", "//footer", "//nav", "//aside",
            "//*[@role='navigation']", "//*[@role='banner']", "//*[@role='contentinfo']",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'menu')]",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'navbar')]",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'breadcrumb')]",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'sidebar')]",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'related')]",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'share')]",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'social')]",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'tag')][self::ul or self::div]",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'comment')]",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'advert')]",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'newsletter')]",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'subscribe')]",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'byline')]",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'author')]",
            "//*[contains(translate(@class,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'meta')][self::div or self::span or self::p]",
            "//*[contains(translate(@id,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'menu')]",
            "//*[contains(translate(@id,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'nav')]",
            "//*[contains(translate(@id,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'sidebar')]",
            "//*[contains(translate(@id,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'footer')]",
            "//*[contains(translate(@id,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'header')]",
        ];
        foreach ($globalKill as $q) {
            $nodes = $this->xpath->query($q);
            if (!$nodes) continue;
            foreach (iterator_to_array($nodes) as $n) {
                if ($n->parentNode) $n->parentNode->removeChild($n);
            }
        }
    }

    private function metaContent($selectors)
    {
        foreach ($selectors as $q) {
            $nodes = $this->xpath->query($q);
            if ($nodes && $nodes->length) {
                $v = trim($nodes->item(0)->getAttribute('content'));
                if ($v !== '') return $v;
            }
        }
        return '';
    }

    private function nodeText($selectors)
    {
        foreach ($selectors as $q) {
            $nodes = $this->xpath->query($q);
            if ($nodes && $nodes->length) {
                $v = trim($nodes->item(0)->textContent);
                if ($v !== '') return $v;
            }
        }
        return '';
    }

    private function findArticleNode()
    {
        $candidates = [
            "//article",
            "//*[@itemprop='articleBody']",
            "//div[contains(@class,'article-body')]",
            "//div[contains(@class,'article-content')]",
            "//div[contains(@class,'post-content')]",
            "//div[contains(@class,'entry-content')]",
            "//div[contains(@class,'news-content')]",
            "//div[contains(@class,'story-body')]",
            "//div[contains(@class,'story-content')]",
            "//div[contains(@class,'detail-content')]",
            "//div[contains(@class,'content-body')]",
            "//main",
        ];

        $best = null;
        $bestLen = 0;
        foreach ($candidates as $q) {
            $nodes = $this->xpath->query($q);
            if (!$nodes) continue;
            foreach ($nodes as $n) {
                $len = strlen(trim($n->textContent));
                if ($len > $bestLen) { $bestLen = $len; $best = $n; }
            }
        }

        if (!$best || $bestLen < 200) {
            $divs = $this->xpath->query("//div");
            foreach ($divs as $d) {
                $ps = 0; $tx = 0;
                foreach ($d->getElementsByTagName('p') as $p) {
                    $ps++; $tx += strlen(trim($p->textContent));
                }
                if ($ps >= 3 && $tx > $bestLen) { $bestLen = $tx; $best = $d; }
            }
        }
        return $best;
    }

    /**
     * Walk article node and collect ONLY clean paragraphs + headings.
     * Drops anything containing <ul>/<li>, "Login", "EngLogin", "আরও পড়ুন",
     * date-only lines, byline-only lines, image-caption-only lines.
     */
    private function extractCleanParagraphs($node, $titleText = '', $imageCaption = '')
    {
        if (!$node) return '';

        // Walk only <p>, <h2>, <h3>, <h4>, <blockquote> elements
        $blocks = [];
        $allowedTags = ['p', 'h2', 'h3', 'h4', 'blockquote'];

        foreach ($allowedTags as $tag) {
            foreach ($node->getElementsByTagName($tag) as $el) {
                $blocks[] = $el;
            }
        }

        // Sort by document order
        usort($blocks, function ($a, $b) {
            return strcmp($a->getNodePath(), $b->getNodePath());
        });

        $out = [];
        $seen = [];
        $junkRegex = '/(^|\s)(Login|EngLogin|Eng\s*Login|আরও পড়ুন|গুগল নিউজ|ফলো করুন|শেয়ার|Share|Tweet|Facebook|WhatsApp|বিজ্ঞাপন|ই-পেপার|খুঁজুন|প্রিন্ট|Subscribe|সাবস্ক্রাইব|কপি করুন|Copy link)($|\s|:)/iu';
        $menuWords = ['সর্বশেষ','বাংলাদেশ','রাজনীতি','বিশ্ব','বাণিজ্য','মতামত','খেলা','বিনোদন','চাকরি','জীবনযাপন','ভিডিও','হোম','লাইফস্টাইল','প্রযুক্তি','শিক্ষা','ধর্ম','স্বাস্থ্য','সম্পাদকীয়','আন্তর্জাতিক'];

        foreach ($blocks as $el) {
            // Skip if any ancestor is also one of the collected blocks (avoid duplicates)
            $parent = $el->parentNode;
            $skip = false;
            while ($parent && $parent->nodeType === XML_ELEMENT_NODE) {
                if (in_array(strtolower($parent->nodeName), $allowedTags, true)) { $skip = true; break; }
                $parent = $parent->parentNode;
            }
            if ($skip) continue;

            // Skip blocks containing <ul>/<ol>/<nav>/<form>
            $bad = false;
            foreach (['ul', 'ol', 'nav', 'form', 'figure'] as $b) {
                if ($el->getElementsByTagName($b)->length > 0) { $bad = true; break; }
            }
            if ($bad) continue;

            $text = trim(preg_replace('/\s+/u', ' ', $el->textContent));
            if ($text === '') continue;

            // Skip pure menu / nav text
            $menuHits = 0;
            foreach ($menuWords as $mw) {
                if (mb_stripos($text, $mw) !== false) $menuHits++;
            }
            if ($menuHits >= 3) continue;

            // Skip junk patterns
            if (preg_match($junkRegex, $text)) continue;

            // Skip very short non-heading lines
            $tag = strtolower($el->nodeName);
            if ($tag === 'p' && mb_strlen($text) < 30) continue;

            // Skip duplicate title
            if ($titleText && mb_stripos($text, $titleText) !== false && mb_strlen($text) < mb_strlen($titleText) + 20) continue;

            // Skip image caption duplicate
            if ($imageCaption && $text === trim($imageCaption)) continue;

            // Skip "byline" patterns: "ক্রীড়া প্রতিবেদক", "নিজস্ব প্রতিবেদক", "ডেস্ক রিপোর্ট"
            if ($tag === 'p' && preg_match('/^(নিজস্ব প্রতিবেদক|ক্রীড়া প্রতিবেদক|ডেস্ক রিপোর্ট|স্টাফ রিপোর্টার|বিশেষ প্রতিনিধি|প্রথম আলো|বিডিনিউজ|বাংলা ট্রিবিউন)[।:,\s]/u', $text)) continue;

            // Skip date-only lines like "১৬ মে ২০২৬" or "1 ঘণ্টা আগে"
            if ($tag === 'p' && mb_strlen($text) < 50 && preg_match('/^[\d০-৯\s\-\/:\.]+ ?(মে|জানুয়ারি|ফেব্রুয়ারি|মার্চ|এপ্রিল|জুন|জুলাই|আগস্ট|সেপ্টেম্বর|অক্টোবর|নভেম্বর|ডিসেম্বর|January|February|March|April|May|June|July|August|September|October|November|December|ঘণ্টা|মিনিট|সেকেন্ড)/u', $text)) continue;

            // De-duplicate
            $key = mb_substr($text, 0, 80);
            if (isset($seen[$key])) continue;
            $seen[$key] = true;

            // Build clean HTML
            $inner = $this->cleanInline($el);
            if (trim(strip_tags($inner)) === '') continue;
            $out[] = '<' . $tag . '>' . $inner . '</' . $tag . '>';
        }

        return implode("\n", $out);
    }

    /** Return inner HTML of $el keeping only <a>, <strong>, <em>, <br> */
    private function cleanInline($el)
    {
        $html = '';
        foreach ($el->childNodes as $c) {
            $html .= $this->dom->saveHTML($c);
        }
        $html = strip_tags($html, '<a><strong><b><em><i><br>');
        $html = preg_replace_callback('/<a([^>]*)>/i', function ($m) {
            if (preg_match('/href\s*=\s*"([^"]+)"/i', $m[1], $h)) {
                $href = htmlspecialchars($h[1], ENT_QUOTES);
                return '<a href="' . $href . '" target="_blank" rel="noopener">';
            }
            return '';
        }, $html);
        $html = preg_replace('/\s+/', ' ', $html);
        return trim($html);
    }

    private function absoluteUrl($url)
    {
        if (!$url) return '';
        if (preg_match('#^https?://#i', $url)) return $url;
        if (strpos($url, '//') === 0) return 'https:' . $url;
        if (strpos($url, '/') === 0) return $this->baseHost . $url;
        return $this->baseHost . '/' . ltrim($url, '/');
    }

    public function scrape()
    {
        $this->loadDom();

        $title = $this->metaContent([
            "//meta[@property='og:title']",
            "//meta[@name='twitter:title']",
        ]);
        if (!$title) $title = $this->nodeText(["//h1", "//title"]);
        $title = trim(preg_replace('/\s*\|.*$/u', '', $title)); // strip "| Site Name"

        $description = $this->metaContent([
            "//meta[@property='og:description']",
            "//meta[@name='description']",
            "//meta[@name='twitter:description']",
        ]);

        $image = $this->metaContent([
            "//meta[@property='og:image']",
            "//meta[@property='og:image:url']",
            "//meta[@name='twitter:image']",
        ]);
        $image = $this->absoluteUrl($image);

        $node = $this->findArticleNode();

        // Try to find image caption from <figcaption>
        $caption = '';
        if ($node) {
            $caps = $node->getElementsByTagName('figcaption');
            if ($caps->length) $caption = trim($caps->item(0)->textContent);
        }

        $content = $this->extractCleanParagraphs($node, $title, $caption);

        // FALLBACK 1: if content too short, walk ALL <p> in body
        if (mb_strlen(strip_tags($content)) < 200) {
            $body = $this->xpath->query("//body")->item(0);
            if ($body) {
                $content = $this->extractCleanParagraphs($body, $title, $caption);
            }
        }

        // FALLBACK 2: still empty → use og:description as single paragraph
        if (mb_strlen(strip_tags($content)) < 100 && $description) {
            $content = '<p>' . htmlspecialchars($description, ENT_QUOTES, 'UTF-8') . '</p>';
        }


        $author = $this->metaContent([
            "//meta[@name='author']",
            "//meta[@property='article:author']",
        ]);

        $published = $this->metaContent([
            "//meta[@property='article:published_time']",
            "//meta[@itemprop='datePublished']",
        ]);

        $siteName = $this->metaContent([
            "//meta[@property='og:site_name']",
        ]);

        return [
            'title'       => html_entity_decode($title, ENT_QUOTES, 'UTF-8'),
            'subtitle'    => mb_substr(html_entity_decode($description, ENT_QUOTES, 'UTF-8'), 0, 250),
            'content'     => $content,
            'image_url'   => $image,
            'image_caption' => $caption,
            'author'      => $author,
            'published'   => $published,
            'source_url'  => $this->url,
            'source_name' => $siteName ?: parse_url($this->url, PHP_URL_HOST),
        ];
    }

    public static function downloadImage($imageUrl, $targetDir, $prefix = 'AutoNews_')
    {
        if (!$imageUrl) return null;
        if (!is_dir($targetDir)) mkdir($targetDir, 0755, true);

        $ch = curl_init($imageUrl);
        curl_setopt_array($ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_TIMEOUT        => 30,
            CURLOPT_SSL_VERIFYPEER => false,
            CURLOPT_USERAGENT      => 'Mozilla/5.0',
        ]);
        $data = curl_exec($ch);
        $type = curl_getinfo($ch, CURLINFO_CONTENT_TYPE);
        curl_close($ch);
        if (!$data) return null;

        $ext = 'jpg';
        if (stripos($type, 'png') !== false) $ext = 'png';
        elseif (stripos($type, 'webp') !== false) $ext = 'webp';
        elseif (stripos($type, 'gif') !== false) $ext = 'gif';

        $filename = $prefix . time() . '_' . mt_rand(1000, 9999) . '.' . $ext;
        $path = rtrim($targetDir, '/') . '/' . $filename;
        if (file_put_contents($path, $data) === false) return null;
        return $filename;
    }
}
