<?php
/**
 * DBA_TIS_Crawler — Topic Intelligence Scanner: Content Discovery & Extraction
 *
 * Two discovery modes:
 *   1. LOCAL (default): Queries WP database for all published pages + posts.
 *      Fast, accurate, no HTTP overhead. Uses post_content directly.
 *   2. SITEMAP: Fetches /sitemap.xml → discovers URLs → fetches each page.
 *      Useful for validating what Google sees vs. what's in the DB.
 *
 * Content extraction:
 *   - Strips HTML, shortcodes, scripts, styles
 *   - Extracts: title, URL, word count, headings (H1-H3), meta description
 *   - Chunks content into ~2000 word segments for AI analysis
 *
 * @since 2.17.0
 */
if ( ! defined( 'ABSPATH' ) ) exit;

class DBA_TIS_Crawler {

    /** Max pages to crawl per scan (safety cap) */
    const MAX_PAGES = 5000;

    /** Max chars of content per page to send to AI */
    const MAX_CONTENT_CHARS = 8000;

    /** Sitemap URLs to try */
    const SITEMAP_URLS = array(
        '/sitemap_index.xml',
        '/sitemap.xml',
        '/wp-sitemap.xml',
    );

    /* ══════════════════════════════════════════════════════════════
     *  LOCAL DISCOVERY — Query WP database directly
     * ══════════════════════════════════════════════════════════════ */

    /**
     * Discover all published pages and posts from the WP database.
     *
     * @param  array $args  Optional filters: post_type, category, language.
     * @return array        Array of [ id, url, title, type, word_count, content_excerpt, headings, categories ].
     */
    public static function discover_local( $args = array() ) {
        global $wpdb;

        $defaults = array(
            'post_types' => array( 'post', 'page', 'investor', 'startup', 'collection' ),
            'limit'      => self::MAX_PAGES,
            'language'   => '',
        );
        $args = wp_parse_args( $args, $defaults );

        // Default to English when WPML is active
        $wpml_active = defined( 'ICL_SITEPRESS_VERSION' ) || function_exists( 'icl_get_languages' );
        if ( empty( $args['language'] ) && $wpml_active ) {
            $args['language'] = apply_filters( 'wpml_default_language', 'en' );
        }

        // Build post type placeholders for SQL
        $pt_placeholders = implode( ',', array_fill( 0, count( $args['post_types'] ), '%s' ) );

        // If WPML active, get post IDs for the selected language via direct SQL
        // This is the only reliable way — WPML's WP_Query filters are unreliable.
        if ( $wpml_active && ! empty( $args['language'] ) ) {
            $icl_table = $wpdb->prefix . 'icl_translations';

            // Check table exists
            $table_exists = $wpdb->get_var( "SHOW TABLES LIKE '{$icl_table}'" ) === $icl_table;

            if ( $table_exists ) {
                // Build element_type conditions: post_post, post_page, post_investor, etc.
                $et_conditions = array();
                $et_values     = array();
                foreach ( $args['post_types'] as $pt ) {
                    $et_conditions[] = 't.element_type = %s';
                    $et_values[]     = 'post_' . $pt;
                }
                $et_sql = implode( ' OR ', $et_conditions );

                $sql_params = array_merge(
                    $et_values,
                    array( $args['language'], (int) $args['limit'] )
                );

                $post_ids = $wpdb->get_col( $wpdb->prepare(
                    "SELECT p.ID FROM {$wpdb->posts} p
                     INNER JOIN {$icl_table} t ON t.element_id = p.ID
                     WHERE p.post_status = 'publish'
                       AND ({$et_sql})
                       AND t.language_code = %s
                     ORDER BY p.post_date DESC
                     LIMIT %d",
                    ...$sql_params
                ) );

                error_log( 'DBA TIS Crawler: SQL found ' . count( $post_ids ) . ' IDs for lang=' . $args['language'] );

                if ( empty( $post_ids ) ) {
                    error_log( 'DBA TIS Crawler: No posts found for language ' . $args['language'] );
                    return array();
                }

                // Now use WP_Query with explicit post IDs — no WPML interference
                $query_args = array(
                    'post__in'         => $post_ids,
                    'post_type'        => $args['post_types'],
                    'post_status'      => 'publish',
                    'posts_per_page'   => count( $post_ids ),
                    'orderby'          => 'date',
                    'order'            => 'DESC',
                    'suppress_filters' => true,  // Bypass WPML — we already filtered
                );
            } else {
                // No WPML table — fall back to normal query
                $query_args = array(
                    'post_type'        => $args['post_types'],
                    'post_status'      => 'publish',
                    'posts_per_page'   => $args['limit'],
                    'orderby'          => 'date',
                    'order'            => 'DESC',
                    'suppress_filters' => true,
                );
            }
        } else {
            // No WPML — normal query
            $query_args = array(
                'post_type'        => $args['post_types'],
                'post_status'      => 'publish',
                'posts_per_page'   => $args['limit'],
                'orderby'          => 'date',
                'order'            => 'DESC',
                'suppress_filters' => true,
            );
        }

        $query = new WP_Query( $query_args );
        $pages = array();

        if ( $query->have_posts() ) {
            while ( $query->have_posts() ) {
                $query->the_post();
                $post = get_post();

                $raw_content = $post->post_content;
                $clean       = self::clean_content( $raw_content );
                $headings    = self::extract_headings( $raw_content );
                $categories  = wp_get_post_categories( $post->ID, array( 'fields' => 'names' ) );
                $word_count  = str_word_count( $clean );

                // Get Yoast/RankMath meta description
                $meta_desc = get_post_meta( $post->ID, '_yoast_wpseo_metadesc', true );
                if ( empty( $meta_desc ) ) {
                    $meta_desc = get_post_meta( $post->ID, 'rank_math_description', true );
                }
                if ( empty( $meta_desc ) ) {
                    $meta_desc = wp_trim_words( $clean, 30, '...' );
                }

                // Get focus keyword from Yoast/RankMath
                $focus_kw = get_post_meta( $post->ID, '_yoast_wpseo_focuskw', true );
                if ( empty( $focus_kw ) ) {
                    $focus_kw = get_post_meta( $post->ID, 'rank_math_focus_keyword', true );
                }

                // Detect which bot created it
                $bot_source = self::detect_bot_source( $post->ID );

                $pages[] = array(
                    'id'            => $post->ID,
                    'url'           => get_permalink( $post->ID ),
                    'title'         => get_the_title( $post->ID ),
                    'type'          => $post->post_type,
                    'date'          => $post->post_date,
                    'word_count'    => $word_count,
                    'content'       => mb_substr( $clean, 0, self::MAX_CONTENT_CHARS ),
                    'headings'      => $headings,
                    'categories'    => $categories,
                    'meta_desc'     => $meta_desc,
                    'focus_keyword' => $focus_kw,
                    'bot_source'    => $bot_source,
                );
            }
        }
        wp_reset_postdata();

        error_log( 'DBA TIS Crawler: Discovered ' . count( $pages ) . ' pages/posts from local DB (lang=' . ( $args['language'] ?: 'all' ) . ').' );
        return $pages;
    }

    /* ══════════════════════════════════════════════════════════════
     *  SITEMAP DISCOVERY — Fetch from sitemap XML
     * ══════════════════════════════════════════════════════════════ */

    /**
     * Discover all URLs from the site's sitemap.xml.
     *
     * @return array  Array of URLs found in sitemap.
     */
    public static function discover_sitemap() {
        $site_url = get_site_url();
        $all_urls = array();

        foreach ( self::SITEMAP_URLS as $path ) {
            $url  = $site_url . $path;
            $resp = wp_remote_get( $url, array( 'timeout' => 15, 'sslverify' => false ) );

            if ( is_wp_error( $resp ) || wp_remote_retrieve_response_code( $resp ) !== 200 ) {
                continue;
            }

            $body = wp_remote_retrieve_body( $resp );
            if ( empty( $body ) ) continue;

            // Suppress XML errors
            libxml_use_internal_errors( true );
            $xml = simplexml_load_string( $body );
            libxml_clear_errors();

            if ( ! $xml ) continue;

            // Check if this is a sitemap index (contains <sitemap> elements)
            if ( isset( $xml->sitemap ) ) {
                foreach ( $xml->sitemap as $child ) {
                    $child_url  = (string) $child->loc;
                    $child_urls = self::parse_sitemap_urls( $child_url );
                    $all_urls   = array_merge( $all_urls, $child_urls );
                }
            }

            // Check if this is a regular sitemap (contains <url> elements)
            if ( isset( $xml->url ) ) {
                foreach ( $xml->url as $entry ) {
                    $all_urls[] = (string) $entry->loc;
                }
            }

            if ( ! empty( $all_urls ) ) break; // Found a working sitemap
        }

        $all_urls = array_unique( $all_urls );
        $all_urls = array_slice( $all_urls, 0, self::MAX_PAGES );

        error_log( 'DBA TIS Crawler: Discovered ' . count( $all_urls ) . ' URLs from sitemap.' );
        return $all_urls;
    }

    /**
     * Parse a child sitemap URL and return its URLs.
     */
    private static function parse_sitemap_urls( $sitemap_url ) {
        $resp = wp_remote_get( $sitemap_url, array( 'timeout' => 15, 'sslverify' => false ) );
        if ( is_wp_error( $resp ) || wp_remote_retrieve_response_code( $resp ) !== 200 ) {
            return array();
        }

        $body = wp_remote_retrieve_body( $resp );
        libxml_use_internal_errors( true );
        $xml = simplexml_load_string( $body );
        libxml_clear_errors();

        if ( ! $xml || ! isset( $xml->url ) ) return array();

        $urls = array();
        foreach ( $xml->url as $entry ) {
            $urls[] = (string) $entry->loc;
        }
        return $urls;
    }

    /* ══════════════════════════════════════════════════════════════
     *  CONTENT HELPERS
     * ══════════════════════════════════════════════════════════════ */

    /**
     * Strip HTML, shortcodes, scripts, styles → clean plain text.
     */
    public static function clean_content( $html ) {
        // Remove scripts and styles
        $html = preg_replace( '/<script[^>]*>.*?<\/script>/is', '', $html );
        $html = preg_replace( '/<style[^>]*>.*?<\/style>/is', '', $html );

        // Remove shortcodes
        $html = strip_shortcodes( $html );

        // Remove Divi/Elementor builder noise
        $html = preg_replace( '/\[et_pb_[^\]]*\]/', '', $html );
        $html = preg_replace( '/\[\/et_pb_[^\]]*\]/', '', $html );

        // Strip remaining HTML
        $text = wp_strip_all_tags( $html );

        // Normalize whitespace
        $text = preg_replace( '/\s+/', ' ', $text );
        $text = trim( $text );

        return $text;
    }

    /**
     * Extract H1-H3 headings from HTML content.
     */
    public static function extract_headings( $html ) {
        $headings = array();

        if ( preg_match_all( '/<(h[1-3])[^>]*>(.*?)<\/\1>/is', $html, $matches, PREG_SET_ORDER ) ) {
            foreach ( $matches as $m ) {
                $headings[] = array(
                    'level' => strtoupper( $m[1] ),
                    'text'  => wp_strip_all_tags( $m[2] ),
                );
            }
        }

        return $headings;
    }

    /**
     * Detect which bot/module created a post.
     */
    public static function detect_bot_source( $post_id ) {
        $meta_map = array(
            '_dba_auto_created'            => 'Core / GeoBot',
            '_dba_pe_auto_created'         => 'PE NewsBot',
            '_dba_luxstrat_auto_created'   => 'LuxStrat EN',
            '_dba_luxstrat_fr_auto_created' => 'LuxStrat FR',
            '_dba_luxstrat_de_auto_created' => 'LuxStrat DE',
            '_dba_luxstrat_es_auto_created' => 'LuxStrat ES',
            '_dba_luxstrat_pt_auto_created' => 'LuxStrat PT',
            '_dba_luxstrat_it_auto_created' => 'LuxStrat IT',
            '_dba_latam_auto_created'      => 'LATAM',
            '_dba_eeurope_auto_created'    => 'Eastern Europe',
            '_dba_startup_hub_auto_created' => 'Startup Hub',
            '_dba_sniper_auto_created'     => 'SEO Sniper',
        );

        foreach ( $meta_map as $key => $label ) {
            if ( get_post_meta( $post_id, $key, true ) ) {
                return $label;
            }
        }

        return 'Manual / Unknown';
    }

    /* ══════════════════════════════════════════════════════════════
     *  CONTENT STATS — Quick summary without AI
     * ══════════════════════════════════════════════════════════════ */

    /**
     * Get quick content statistics from the DB.
     */
    public static function get_content_stats() {
        global $wpdb;

        $stats = array();

        // Total posts by type
        $stats['by_type'] = array(
            'posts' => (int) wp_count_posts( 'post' )->publish,
            'pages' => (int) wp_count_posts( 'page' )->publish,
        );

        // Total by bot source
        $meta_keys = array(
            '_dba_auto_created'            => 'Core / GeoBot',
            '_dba_pe_auto_created'         => 'PE NewsBot',
            '_dba_luxstrat_auto_created'   => 'LuxStrat EN',
            '_dba_latam_auto_created'      => 'LATAM',
            '_dba_eeurope_auto_created'    => 'Eastern Europe',
            '_dba_startup_hub_auto_created' => 'Startup Hub',
            '_dba_sniper_auto_created'     => 'SEO Sniper',
        );

        $stats['by_bot'] = array();
        foreach ( $meta_keys as $key => $label ) {
            $count = (int) $wpdb->get_var( $wpdb->prepare(
                "SELECT COUNT(DISTINCT p.ID) FROM {$wpdb->posts} p
                 INNER JOIN {$wpdb->postmeta} pm ON p.ID = pm.post_id
                 WHERE p.post_status = 'publish' AND pm.meta_key = %s",
                $key
            ) );
            if ( $count > 0 ) {
                $stats['by_bot'][ $label ] = $count;
            }
        }

        // Manual posts (not auto-created)
        $auto_total = array_sum( $stats['by_bot'] );
        $total_published = $stats['by_type']['posts'] + $stats['by_type']['pages'];
        $stats['by_bot']['Manual / Editorial'] = max( 0, $total_published - $auto_total );

        // By category
        $categories = get_categories( array( 'hide_empty' => true ) );
        $stats['by_category'] = array();
        foreach ( $categories as $cat ) {
            $stats['by_category'][ $cat->name ] = $cat->count;
        }
        arsort( $stats['by_category'] );
        $stats['by_category'] = array_slice( $stats['by_category'], 0, 20, true );

        // Posts per month (last 12 months)
        $stats['monthly_output'] = array();
        for ( $i = 11; $i >= 0; $i-- ) {
            $month = date( 'Y-m', strtotime( "-{$i} months" ) );
            $count = (int) $wpdb->get_var( $wpdb->prepare(
                "SELECT COUNT(*) FROM {$wpdb->posts}
                 WHERE post_status = 'publish' AND post_type IN ('post','page')
                 AND DATE_FORMAT(post_date, '%%Y-%%m') = %s",
                $month
            ) );
            $stats['monthly_output'][ $month ] = $count;
        }

        $stats['total_published'] = $total_published;

        return $stats;
    }
}
