<?php
/**
 * DBA Sitemap & Page Scanner v5.2 — WPML-Safe, Zero Memory Bloat
 *
 * ALL scanning uses raw SQL — never calls get_permalink(), get_the_category(),
 * get_the_tags(), or any function that triggers WPML translation machinery.
 *
 * Storage: per-language registries (dba_full_registry_en, _it, _fr, etc.)
 * Results saved per-language after each batch — no giant transient accumulator.
 */
class DBA_Sitemap {

    private static $instance = null;
    private $registry_option       = 'dba_link_registry';
    private $idf_option_prefix     = 'dba_idf_weights';
    private $registry_prefix       = 'dba_full_registry';
    private $scan_meta_option      = 'dba_scan_meta';

    const SCAN_BATCH_SIZE = 25;

    public static function get_instance() {
        if ( null === self::$instance ) self::$instance = new self();
        return self::$instance;
    }

    private function __construct() {
        add_action( 'dba_daily_sitemap_refresh', array( $this, 'refresh_registry' ) );
    }

    /* ================================================================
     *  PUBLIC GETTERS
     * ================================================================ */

    public function get_full_registry( $language = null ) {
        if ( $language ) {
            return get_option( $this->registry_prefix . '_' . $language, array() );
        }
        $meta = $this->get_scan_meta();
        $langs = $meta['languages'] ?? array( 'en' );
        $all = array();
        foreach ( $langs as $lang ) {
            $lp = get_option( $this->registry_prefix . '_' . $lang, array() );
            if ( is_array( $lp ) ) $all = array_merge( $all, $lp );
        }
        return $all;
    }

    public function get_registry() {
        return get_option( $this->registry_option, array() );
    }

    public function get_scan_meta() {
        return get_option( $this->scan_meta_option, array(
            'last_scan' => null, 'total_pages' => 0, 'languages' => array(), 'lang_counts' => array(),
        ) );
    }

    /* ================================================================
     *  RAW SQL HELPERS — No WPML, No WP cache bloat
     * ================================================================ */

    /**
     * Bulk-load WPML languages for an array of post IDs.
     * Single SQL query. Returns [ post_id => 'en', ... ]
     */
    private function bulk_get_languages( $post_ids ) {
        global $wpdb;
        $map = array();
        if ( empty( $post_ids ) ) return $map;

        $table = $wpdb->prefix . 'icl_translations';
        $has_wpml = ( $wpdb->get_var( "SHOW TABLES LIKE '{$table}'" ) === $table );

        if ( ! $has_wpml ) {
            foreach ( $post_ids as $id ) $map[ $id ] = 'en';
            return $map;
        }

        $ids_str = implode( ',', array_map( 'intval', $post_ids ) );
        $rows = $wpdb->get_results(
            "SELECT element_id, language_code FROM {$table}
             WHERE element_id IN ({$ids_str}) AND element_type LIKE 'post_%'"
        );
        foreach ( $rows as $r ) $map[ intval( $r->element_id ) ] = $r->language_code;
        foreach ( $post_ids as $id ) { if ( ! isset( $map[ $id ] ) ) $map[ $id ] = 'en'; }
        return $map;
    }

    /**
     * Bulk-load post meta for specific keys. Single SQL query.
     * Returns [ post_id => [ 'meta_key' => 'value', ... ], ... ]
     */
    private function bulk_get_meta( $post_ids, $keys ) {
        global $wpdb;
        $map = array();
        if ( empty( $post_ids ) || empty( $keys ) ) return $map;

        $ids_str  = implode( ',', array_map( 'intval', $post_ids ) );
        $key_in   = implode( ',', array_map( function( $k ) { global $wpdb; return $wpdb->prepare( '%s', $k ); }, $keys ) );

        $rows = $wpdb->get_results(
            "SELECT post_id, meta_key, meta_value FROM {$wpdb->postmeta}
             WHERE post_id IN ({$ids_str}) AND meta_key IN ({$key_in})"
        );

        foreach ( $rows as $r ) {
            $pid = intval( $r->post_id );
            if ( ! isset( $map[ $pid ] ) ) $map[ $pid ] = array();
            $map[ $pid ][ $r->meta_key ] = $r->meta_value;
        }
        return $map;
    }

    /**
     * Bulk-load taxonomy terms (categories + tags) for post IDs. Two SQL queries.
     * Returns [ post_id => [ 'categories' => [...], 'tags' => [...] ] ]
     */
    private function bulk_get_terms( $post_ids ) {
        global $wpdb;
        $map = array();
        if ( empty( $post_ids ) ) return $map;

        $ids_str = implode( ',', array_map( 'intval', $post_ids ) );

        $rows = $wpdb->get_results(
            "SELECT tr.object_id AS post_id, t.name, tt.taxonomy
             FROM {$wpdb->term_relationships} tr
             JOIN {$wpdb->term_taxonomy} tt ON tr.term_taxonomy_id = tt.term_taxonomy_id
             JOIN {$wpdb->terms} t ON tt.term_id = t.term_id
             WHERE tr.object_id IN ({$ids_str})
             AND tt.taxonomy IN ('category', 'post_tag')"
        );

        foreach ( $rows as $r ) {
            $pid = intval( $r->post_id );
            if ( ! isset( $map[ $pid ] ) ) $map[ $pid ] = array( 'categories' => array(), 'tags' => array() );
            if ( $r->taxonomy === 'category' ) {
                $map[ $pid ]['categories'][] = $r->name;
            } else {
                $map[ $pid ]['tags'][] = $r->name;
            }
        }
        return $map;
    }

    /**
     * Build permalink from raw post data WITHOUT calling get_permalink().
     * Uses site permalink structure. Safe — no WPML hooks triggered.
     */
    private function build_raw_permalink( $post ) {
        $site_url = rtrim( get_option( 'siteurl' ), '/' );

        if ( $post->post_type === 'page' ) {
            // Pages: just /slug/ (or /parent/slug/ for nested)
            return $site_url . '/' . $post->post_name . '/';
        }

        // Posts: use permalink structure from settings
        $struct = get_option( 'permalink_structure', '/%postname%/' );

        if ( empty( $struct ) || $struct === '' ) {
            // Plain permalinks: ?p=ID
            return $site_url . '/?p=' . $post->ID;
        }

        // Build from structure — handle common patterns
        $url = $struct;
        $url = str_replace( '%postname%', $post->post_name, $url );
        $url = str_replace( '%post_id%', $post->ID, $url );

        // Date-based
        $date = strtotime( $post->post_date );
        if ( $date ) {
            $url = str_replace( '%year%', date( 'Y', $date ), $url );
            $url = str_replace( '%monthnum%', date( 'm', $date ), $url );
            $url = str_replace( '%day%', date( 'd', $date ), $url );
        }

        // Category — if structure needs it, get first category
        if ( strpos( $url, '%category%' ) !== false ) {
            // We'll fill this from bulk_get_terms data later; placeholder for now
            $url = str_replace( '%category%', 'uncategorized', $url );
        }

        return $site_url . $url;
    }

    /* ================================================================
     *  MODE 1: FULL SCAN — AJAX batched, zero memory bloat
     * ================================================================ */

    public function count_scannable_posts() {
        global $wpdb;
        return intval( $wpdb->get_var(
            "SELECT COUNT(*) FROM {$wpdb->posts}
             WHERE post_status = 'publish' AND post_type IN ('post', 'page') AND post_title != ''"
        ) );
    }

    public function scan_start() {
        global $wpdb;

        $total = $this->count_scannable_posts();

        // Clean up any leftover chunks from a previous failed scan
        $wpdb->query( "DELETE FROM {$wpdb->options} WHERE option_name LIKE 'dba_scan_chunk_%'" );

        // Clear old per-language registries
        $old_meta = $this->get_scan_meta();
        $old_langs = $old_meta['languages'] ?? array();
        foreach ( $old_langs as $lang ) {
            delete_option( $this->registry_prefix . '_' . $lang );
            delete_option( $this->idf_option_prefix . '_' . $lang );
        }

        // Store scan state (tiny)
        update_option( 'dba_scan_state', array(
            'total'  => $total,
            'offset' => 0,
            'chunk'  => 0,
            'start'  => microtime( true ),
        ), false );

        error_log( 'DBA Scanner: Full scan started — ' . $total . ' posts/pages' );
        return $total;
    }

    public function scan_batch() {
        global $wpdb;

        $state  = get_option( 'dba_scan_state', array() );

        // If we're in finalize phase, run finalize batch
        if ( ( $state['phase'] ?? 'scanning' ) === 'finalizing' ) {
            return $this->finalize_batch();
        }

        $offset = intval( $state['offset'] ?? 0 );
        $total  = intval( $state['total'] ?? 0 );
        $chunk  = intval( $state['chunk'] ?? 0 );

        if ( $total <= 0 ) {
            return array( 'done' => true, 'processed' => 0, 'offset' => 0, 'total' => 0 );
        }

        // ── Fetch batch of raw post rows ──
        $posts = $wpdb->get_results( $wpdb->prepare(
            "SELECT ID, post_title, post_name, post_type, post_excerpt, post_date, post_modified
             FROM {$wpdb->posts}
             WHERE post_status = 'publish' AND post_type IN ('post', 'page') AND post_title != ''
             ORDER BY ID ASC
             LIMIT %d OFFSET %d",
            self::SCAN_BATCH_SIZE, $offset
        ) );

        if ( ! empty( $posts ) ) {
            $batch_ids = array_map( function( $p ) { return $p->ID; }, $posts );

            $lang_map = $this->bulk_get_languages( $batch_ids );
            $meta_map = $this->bulk_get_meta( $batch_ids, array(
                '_yoast_wpseo_focuskw', '_yoast_wpseo_metadesc',
                'rank_math_focus_keyword',
                '_dba_subject', '_dba_theme',
            ) );
            $term_map = $this->bulk_get_terms( $batch_ids );

            $batch_entries = array();
            foreach ( $posts as $post ) {
                $pid  = $post->ID;
                $lang = $lang_map[ $pid ] ?? 'en';
                $meta = $meta_map[ $pid ] ?? array();
                $terms = $term_map[ $pid ] ?? array( 'categories' => array(), 'tags' => array() );

                $entry = $this->build_entry_raw( $post, $lang, $meta, $terms );
                if ( $entry ) $batch_entries[] = $entry;
            }

            // Save as numbered chunk (fast: tiny INSERT, no merging)
            if ( ! empty( $batch_entries ) ) {
                update_option( 'dba_scan_chunk_' . $chunk, $batch_entries, false );
            }
        }

        $new_offset = $offset + self::SCAN_BATCH_SIZE;
        $processed  = min( $new_offset, $total );
        $scan_done  = ( $new_offset >= $total || empty( $posts ) );

        $state['offset'] = $new_offset;
        $state['chunk']  = $chunk + 1;

        if ( $scan_done ) {
            // Switch to finalize phase — JS keeps calling scan_batch, we route to finalize_batch
            $state['phase']       = 'finalizing';
            $state['fin_chunk']   = 0;
            $state['total_chunks'] = $chunk + 1;
        }

        update_option( 'dba_scan_state', $state, false );

        return array(
            'done'      => false, // Not done until finalize completes
            'processed' => $processed,
            'offset'    => $new_offset,
            'total'     => $total,
            'phase'     => $scan_done ? 'finalizing' : 'scanning',
        );
    }

    /**
     * Build page entry from raw data — ZERO WordPress/WPML function calls
     */
    private function build_entry_raw( $post, $language, $meta, $terms ) {
        $slug = $post->post_name;
        if ( empty( $slug ) || strlen( $slug ) < 2 ) return null;

        $url = $this->build_raw_permalink( $post );
        $url_lower = strtolower( $url );

        // Skip junk
        $skip = array( '/tag/', '/author/', '/wp-content/', '/feed/', 'replytocom=' );
        foreach ( $skip as $sp ) { if ( strpos( $url_lower, $sp ) !== false ) return null; }

        $title      = html_entity_decode( $post->post_title, ENT_QUOTES, 'UTF-8' );
        $categories = $terms['categories'] ?? array();
        $tags       = $terms['tags'] ?? array();
        $yoast_kp   = $meta['_yoast_wpseo_focuskw'] ?? ( $meta['rank_math_focus_keyword'] ?? '' );
        $yoast_meta = $meta['_yoast_wpseo_metadesc'] ?? '';
        $dba_subj   = $meta['_dba_subject'] ?? '';
        $dba_theme  = $meta['_dba_theme'] ?? '';
        $excerpt    = $post->post_excerpt ?: '';

        // Topic words from all real data
        $topic_text = implode( ' ', array_filter( array(
            $title, $slug, implode( ' ', $categories ), implode( ' ', $tags ),
            $yoast_kp, $dba_subj, $dba_theme,
        ) ) );
        $topic_words = $this->extract_topic_words( strtolower( str_replace( ' ', '-', $topic_text ) ) );

        // Keyphrases
        $keyphrases = array( $title );
        if ( $yoast_kp ) $keyphrases[] = $yoast_kp;
        foreach ( $categories as $c ) { if ( strlen( $c ) >= 4 && strtolower( $c ) !== 'uncategorized' ) $keyphrases[] = $c; }
        foreach ( $tags as $t ) { if ( strlen( $t ) >= 4 ) $keyphrases[] = $t; }
        $keyphrases = array_unique( array_merge( $keyphrases, $this->build_smart_keyphrases( $slug, $title ) ) );

        $is_blog = ( strpos( $url, '/blog/' ) !== false || $post->post_type === 'post' );

        return array(
            'url'             => $url,
            'slug'            => $slug,
            'title'           => $title,
            'post_type'       => $post->post_type,
            'language'        => $language,
            'categories'      => $categories,
            'tags'            => $tags,
            'yoast_keyphrase' => $yoast_kp,
            'yoast_meta'      => $yoast_meta,
            'excerpt'         => $excerpt,
            'dba_subject'     => $dba_subj,
            'dba_theme'       => $dba_theme,
            'keyphrases'      => $keyphrases,
            'topic_words'     => $topic_words,
            'is_blog'         => $is_blog,
            'last_modified'   => $post->post_modified,
        );
    }

    /* ================================================================
     *  FINALIZE — Compute IDF per language, save metadata
     * ================================================================ */

    /**
     * Finalize in batches — processes 100 chunks per AJAX call.
     * Appends to per-language registries incrementally.
     */
    private function finalize_batch() {
        global $wpdb;

        $state        = get_option( 'dba_scan_state', array() );
        $fin_chunk    = intval( $state['fin_chunk'] ?? 0 );
        $total_chunks = intval( $state['total_chunks'] ?? 0 );
        $total_pages  = intval( $state['total'] ?? 0 );
        $chunks_per_call = 100;

        $end_chunk = min( $fin_chunk + $chunks_per_call, $total_chunks );

        // Process this batch of chunks
        $by_lang = array();
        for ( $i = $fin_chunk; $i < $end_chunk; $i++ ) {
            $entries = get_option( 'dba_scan_chunk_' . $i, array() );
            if ( is_array( $entries ) ) {
                foreach ( $entries as $entry ) {
                    $lang = $entry['language'] ?? 'en';
                    if ( ! isset( $by_lang[ $lang ] ) ) $by_lang[ $lang ] = array();
                    $by_lang[ $lang ][] = $entry;
                }
            }
            delete_option( 'dba_scan_chunk_' . $i );
        }

        // Append to per-language registries
        foreach ( $by_lang as $lang => $entries ) {
            $existing = get_option( $this->registry_prefix . '_' . $lang, array() );
            if ( ! is_array( $existing ) ) $existing = array();
            $existing = array_merge( $existing, $entries );
            update_option( $this->registry_prefix . '_' . $lang, $existing, false );
        }

        $state['fin_chunk'] = $end_chunk;
        $fin_done = ( $end_chunk >= $total_chunks );

        if ( $fin_done ) {
            // All chunks merged — compute IDF and save metadata
            $this->finalize_complete( $state['start'] ?? microtime( true ) );
        } else {
            update_option( 'dba_scan_state', $state, false );
        }

        $processed = min( $total_pages, $total_pages ); // scanning was already 100%

        return array(
            'done'      => $fin_done,
            'processed' => $total_pages,
            'offset'    => $total_pages,
            'total'     => $total_pages,
            'phase'     => $fin_done ? 'complete' : 'finalizing',
        );
    }

    /**
     * Final step — compute IDF per language, save metadata. Called once.
     */
    private function finalize_complete( $start_time ) {
        global $wpdb;
        $duration = round( microtime( true ) - floatval( $start_time ), 2 );

        // Find all stored language registries
        $options = $wpdb->get_col(
            "SELECT option_name FROM {$wpdb->options}
             WHERE option_name LIKE 'dba_full_registry_%'"
        );

        $lang_counts = array();
        $total_pages = 0;
        $total_idf   = 0;
        $all_langs   = array();

        foreach ( $options as $opt_name ) {
            $lang = str_replace( 'dba_full_registry_', '', $opt_name );
            if ( strlen( $lang ) < 2 || strlen( $lang ) > 5 ) continue;

            $pages = get_option( $opt_name, array() );
            if ( ! is_array( $pages ) || empty( $pages ) ) continue;

            $all_langs[]           = $lang;
            $lang_counts[ $lang ]  = count( $pages );
            $total_pages          += count( $pages );

            // IDF
            $wc = array();
            foreach ( $pages as $p ) {
                foreach ( array_unique( $p['topic_words'] ?? array() ) as $w ) {
                    if ( ! isset( $wc[ $w ] ) ) $wc[ $w ] = 0;
                    $wc[ $w ]++;
                }
            }
            $td = max( count( $pages ), 1 );
            $idf = array();
            foreach ( $wc as $word => $dc ) {
                $idf[ $word ] = log( $td / max( $dc, 1 ) );
            }
            update_option( $this->idf_option_prefix . '_' . $lang, $idf, false );
            $total_idf += count( $idf );

            unset( $pages, $wc, $idf );
        }

        // Blog registry (English)
        $en = get_option( $this->registry_prefix . '_en', array() );
        $this->rebuild_blog_registry( is_array( $en ) ? $en : array() );

        arsort( $lang_counts );

        update_option( $this->scan_meta_option, array(
            'last_scan'        => current_time( 'mysql' ),
            'total_pages'      => $total_pages,
            'languages'        => $all_langs,
            'lang_counts'      => $lang_counts,
            'idf_terms'        => $total_idf,
            'scan_duration'    => $duration,
            'last_incremental' => current_time( 'mysql' ),
        ), false );

        delete_option( 'dba_scan_state' );
        delete_option( 'dba_full_sitemap_registry' );
        delete_option( 'dba_idf_weights' );
        $wpdb->query( "DELETE FROM {$wpdb->options} WHERE option_name LIKE 'dba_scan_chunk_%'" );

        $summary = array();
        foreach ( array_slice( $lang_counts, 0, 5, true ) as $l => $c ) $summary[] = strtoupper( $l ) . ':' . $c;
        error_log( sprintf(
            'DBA Scanner: Complete — %d pages, %d languages [%s], %.2fs',
            $total_pages, count( $all_langs ), implode( ', ', $summary ), $duration
        ) );
    }

    /* ================================================================
     *  MODE 2: INCREMENTAL REFRESH (before generation)
     * ================================================================ */

    public function incremental_refresh() {
        global $wpdb;

        $meta = $this->get_scan_meta();
        $last = $meta['last_scan'] ?? null;

        if ( empty( $last ) ) {
            // No prior scan — cannot do incremental. User must click button.
            error_log( 'DBA Scanner: Incremental skipped — no prior scan exists.' );
            return;
        }

        $posts = $wpdb->get_results( $wpdb->prepare(
            "SELECT ID, post_title, post_name, post_type, post_excerpt, post_date, post_modified
             FROM {$wpdb->posts}
             WHERE post_status = 'publish' AND post_type IN ('post', 'page')
             AND post_title != '' AND post_modified > %s
             ORDER BY ID ASC LIMIT 500",
            $last
        ) );

        if ( empty( $posts ) ) {
            error_log( 'DBA Scanner: Incremental — no changes since ' . $last );
            return;
        }

        $ids      = array_map( function( $p ) { return $p->ID; }, $posts );
        $lang_map = $this->bulk_get_languages( $ids );
        $meta_map = $this->bulk_get_meta( $ids, array(
            '_yoast_wpseo_focuskw', '_yoast_wpseo_metadesc', 'rank_math_focus_keyword',
            '_dba_subject', '_dba_theme',
        ) );
        $term_map = $this->bulk_get_terms( $ids );

        // Group by language
        $by_lang = array();
        foreach ( $posts as $post ) {
            $pid  = $post->ID;
            $lang = $lang_map[ $pid ] ?? 'en';
            $entry = $this->build_entry_raw( $post, $lang, $meta_map[ $pid ] ?? array(), $term_map[ $pid ] ?? array( 'categories' => array(), 'tags' => array() ) );
            if ( ! $entry ) continue;
            if ( ! isset( $by_lang[ $lang ] ) ) $by_lang[ $lang ] = array();
            $by_lang[ $lang ][] = $entry;
        }

        $added = 0; $updated = 0;

        foreach ( $by_lang as $lang => $new_entries ) {
            $existing = get_option( $this->registry_prefix . '_' . $lang, array() );
            if ( ! is_array( $existing ) ) $existing = array();

            // Index by URL
            $url_idx = array();
            foreach ( $existing as $i => $p ) $url_idx[ strtolower( rtrim( $p['url'], '/' ) ) ] = $i;

            foreach ( $new_entries as $entry ) {
                $uk = strtolower( rtrim( $entry['url'], '/' ) );
                if ( isset( $url_idx[ $uk ] ) ) {
                    $existing[ $url_idx[ $uk ] ] = $entry;
                    $updated++;
                } else {
                    $existing[] = $entry;
                    $added++;
                }
            }

            update_option( $this->registry_prefix . '_' . $lang, $existing, false );
        }

        // Update scan meta
        $meta['last_scan']        = current_time( 'mysql' );
        $meta['last_incremental'] = current_time( 'mysql' );
        foreach ( $by_lang as $lang => $_ ) {
            $p = get_option( $this->registry_prefix . '_' . $lang, array() );
            $meta['lang_counts'][ $lang ] = is_array( $p ) ? count( $p ) : 0;
        }
        $meta['total_pages'] = array_sum( $meta['lang_counts'] ?? array() );
        update_option( $this->scan_meta_option, $meta, false );

        error_log( sprintf( 'DBA Scanner: Incremental — %d added, %d updated', $added, $updated ) );
    }

    /* ================================================================
     *  MODE 3: SINGLE POST UPDATE
     * ================================================================ */

    public function update_single_entry( $post_id ) {
        $post = get_post( $post_id );
        if ( ! $post || $post->post_status !== 'publish' ) return;
        if ( ! in_array( $post->post_type, array( 'post', 'page' ) ) ) return;
        if ( empty( $post->post_title ) || ( defined( 'DOING_AUTOSAVE' ) && DOING_AUTOSAVE ) ) return;

        $lang_map = $this->bulk_get_languages( array( $post_id ) );
        $lang     = $lang_map[ $post_id ] ?? 'en';
        $meta_map = $this->bulk_get_meta( array( $post_id ), array(
            '_yoast_wpseo_focuskw', '_yoast_wpseo_metadesc', 'rank_math_focus_keyword',
            '_dba_subject', '_dba_theme',
        ) );
        $term_map = $this->bulk_get_terms( array( $post_id ) );

        $row = (object) array(
            'ID' => $post->ID, 'post_title' => $post->post_title,
            'post_name' => $post->post_name, 'post_type' => $post->post_type,
            'post_excerpt' => $post->post_excerpt, 'post_date' => $post->post_date,
            'post_modified' => $post->post_modified,
        );

        $entry = $this->build_entry_raw( $row, $lang, $meta_map[ $post_id ] ?? array(), $term_map[ $post_id ] ?? array( 'categories' => array(), 'tags' => array() ) );
        if ( ! $entry ) return;

        $pages   = get_option( $this->registry_prefix . '_' . $lang, array() );
        if ( ! is_array( $pages ) ) $pages = array();
        $url_key = strtolower( rtrim( $entry['url'], '/' ) );
        $found   = false;

        foreach ( $pages as $i => $p ) {
            if ( strtolower( rtrim( $p['url'], '/' ) ) === $url_key ) {
                $pages[ $i ] = $entry;
                $found = true;
                break;
            }
        }
        if ( ! $found ) $pages[] = $entry;

        update_option( $this->registry_prefix . '_' . $lang, $pages, false );
        error_log( 'DBA Scanner: Updated [' . $lang . '] "' . $entry['title'] . '"' );
    }

    public function remove_single_entry( $post_id ) {
        $post = get_post( $post_id );
        if ( ! $post || ! in_array( $post->post_type, array( 'post', 'page' ) ) ) return;

        $lang_map = $this->bulk_get_languages( array( $post_id ) );
        $lang     = $lang_map[ $post_id ] ?? 'en';
        $url      = $this->build_raw_permalink( $post );
        $url_key  = strtolower( rtrim( $url, '/' ) );

        $pages = get_option( $this->registry_prefix . '_' . $lang, array() );
        if ( ! is_array( $pages ) ) return;

        $filtered = array();
        foreach ( $pages as $p ) {
            if ( strtolower( rtrim( $p['url'], '/' ) ) !== $url_key ) $filtered[] = $p;
        }

        if ( count( $filtered ) < count( $pages ) ) {
            update_option( $this->registry_prefix . '_' . $lang, $filtered, false );
            error_log( 'DBA Scanner: Removed [' . $lang . '] ' . $url );
        }
    }

    /* ================================================================
     *  ENSURE FRESH / QUICK FULL SCAN
     * ================================================================ */

    public function ensure_fresh_registry() {
        $meta = $this->get_scan_meta();
        $last = $meta['last_scan'] ?? null;

        // NEVER do a full scan here — that kills shared hosting (35k posts = timeout).
        // Full scan is ONLY available via the AJAX "Scan Internal Pages" button.
        if ( empty( $last ) ) {
            error_log( 'DBA Scanner: No scan yet — skipping. Use "Scan Internal Pages" button.' );
            return;
        }

        $age = time() - strtotime( $last );
        if ( $age > HOUR_IN_SECONDS ) {
            $this->incremental_refresh();
        }
    }

    /**
     * Quick scan — NON-AJAX fallback. Strictly limited to avoid timeout.
     * Only scans up to 500 most recent posts. For full scan, use AJAX button.
     */
    private function quick_full_scan() {
        global $wpdb;

        $start = microtime( true );

        // SAFETY: Only scan 500 most recent posts — enough for linking, no timeout risk
        $posts = $wpdb->get_results(
            "SELECT ID, post_title, post_name, post_type, post_excerpt, post_date, post_modified
             FROM {$wpdb->posts}
             WHERE post_status = 'publish' AND post_type IN ('post', 'page') AND post_title != ''
             ORDER BY post_modified DESC
             LIMIT 500"
        );

        if ( empty( $posts ) ) {
            error_log( 'DBA Scanner: Quick scan — no posts found' );
            return;
        }

        $ids      = array_map( function( $p ) { return $p->ID; }, $posts );
        $lang_map = $this->bulk_get_languages( $ids );
        $meta_map = $this->bulk_get_meta( $ids, array(
            '_yoast_wpseo_focuskw', '_yoast_wpseo_metadesc', 'rank_math_focus_keyword',
            '_dba_subject', '_dba_theme',
        ) );
        $term_map = $this->bulk_get_terms( $ids );

        $by_lang = array();
        foreach ( $posts as $post ) {
            $pid  = $post->ID;
            $lang = $lang_map[ $pid ] ?? 'en';
            $entry = $this->build_entry_raw( $post, $lang, $meta_map[ $pid ] ?? array(), $term_map[ $pid ] ?? array( 'categories' => array(), 'tags' => array() ) );
            if ( ! $entry ) continue;
            if ( ! isset( $by_lang[ $lang ] ) ) $by_lang[ $lang ] = array();
            $by_lang[ $lang ][] = $entry;
        }

        // Save per-language
        $lang_counts = array();
        $total = 0;
        foreach ( $by_lang as $lang => $pages ) {
            update_option( $this->registry_prefix . '_' . $lang, $pages, false );
            $lang_counts[ $lang ] = count( $pages );
            $total += count( $pages );
        }

        // Build blog registry from English
        if ( isset( $by_lang['en'] ) ) {
            $this->rebuild_blog_registry( $by_lang['en'] );
        }

        update_option( $this->scan_meta_option, array(
            'last_scan'        => current_time( 'mysql' ),
            'total_pages'      => $total,
            'languages'        => array_keys( $by_lang ),
            'lang_counts'      => $lang_counts,
            'idf_terms'        => 0,
            'scan_duration'    => round( microtime( true ) - $start, 2 ),
            'last_incremental' => current_time( 'mysql' ),
            'partial'          => true,
        ), false );

        error_log( 'DBA Scanner: Quick scan — ' . $total . ' pages (500 most recent)' );
    }

    public function refresh_registry() {
        // Called by daily cron + settings save. MUST be lightweight.
        // If no prior scan: skip (user must click "Scan Internal Pages").
        // If prior scan exists: do incremental only.
        $meta = $this->get_scan_meta();
        if ( ! empty( $meta['last_scan'] ) ) {
            $this->incremental_refresh();
        } else {
            error_log( 'DBA Scanner: refresh_registry skipped — no prior scan. Use admin button.' );
        }
        return $this->get_registry();
    }

    /* ================================================================
     *  BLOG-ONLY REGISTRY (backward compat)
     * ================================================================ */

    private function rebuild_blog_registry( $pages ) {
        $cs = get_option( 'dba_content_settings', array() );
        $pillar = rtrim( $cs['pillar_url'] ?? 'https://www.damalion.com/blog/', '/' );

        $blog = array();
        foreach ( $pages as $p ) {
            if ( ( $p['is_blog'] ?? false ) && rtrim( $p['url'], '/' ) !== $pillar ) {
                $blog[] = array( 'url' => $p['url'], 'slug' => $p['slug'], 'title' => $p['title'], 'date' => $p['last_modified'] ?? '' );
            }
        }
        update_option( $this->registry_option, $blog, false );
    }

    /* ================================================================
     *  CONTEXTUAL LINK SELECTION — language-aware
     * ================================================================ */

    public function select_contextual_links( $exclude_slugs = array(), $theme = '', $subject = '', $limit = 8, $language = 'en' ) {
        $pages = $this->get_full_registry( $language );
        $idf   = get_option( $this->idf_option_prefix . '_' . $language, array() );

        if ( empty( $pages ) ) {
            error_log( 'DBA Sitemap: No pages for [' . $language . ']' );
            return array();
        }

        $blocked = array(
            'https://www.damalion.com/blog', 'https://www.damalion.com/blog/',
            'https://www.damalion.com', 'https://www.damalion.com/',
            'https://www.damalion.com/lets-connect', 'https://www.damalion.com/lets-connect/',
        );
        $blocked_pat = array( '/category/', '/tag/', '/author/', '/page/', '/feed/', '/wp-json/', '/wp-admin/', '/wp-content/' );

        $avail = array();
        foreach ( $pages as $p ) {
            $ul = strtolower( rtrim( $p['url'], '/' ) );
            if ( in_array( $ul, $blocked ) || in_array( $ul . '/', $blocked ) ) continue;
            $skip = false;
            foreach ( $blocked_pat as $bp ) { if ( strpos( $ul, $bp ) !== false ) { $skip = true; break; } }
            if ( $skip ) continue;
            $s = $p['slug'] ?? '';
            if ( in_array( $s, $exclude_slugs ) || strlen( $s ) < 4 ) continue;
            $avail[] = $p;
        }

        if ( empty( $avail ) ) return array();

        $qw = $this->extract_topic_words( strtolower( str_replace( ' ', '-', $theme . ' ' . $subject ) ) );

        if ( empty( $qw ) ) { shuffle( $avail ); return array_slice( $avail, 0, $limit ); }

        $scored = array();
        foreach ( $avail as $p ) {
            $ptw = $p['topic_words'] ?? array();
            $sc = 0; $m = array();
            foreach ( $qw as $w ) {
                if ( in_array( $w, $ptw ) ) { $sc += ( $idf[ $w ] ?? 1.0 ); $m[] = $w; }
            }
            if ( ! empty( $p['yoast_keyphrase'] ) ) {
                $yw = $this->extract_topic_words( strtolower( str_replace( ' ', '-', $p['yoast_keyphrase'] ) ) );
                if ( ! empty( array_intersect( $qw, $yw ) ) ) $sc *= 1.5;
            }
            if ( count( $m ) >= 2 ) $sc *= ( 1 + count( $m ) * 0.3 );
            $scored[] = array( 'p' => $p, 's' => $sc, 'm' => $m );
        }

        usort( $scored, function( $a, $b ) { return $b['s'] <=> $a['s']; } );

        $res = array(); $used = array();
        foreach ( $scored as $e ) {
            if ( count( $res ) >= $limit || $e['s'] <= 0.1 ) break;
            if ( in_array( $e['p']['slug'], $used ) ) continue;
            $res[] = $e['p']; $used[] = $e['p']['slug'];
        }

        if ( count( $res ) < $limit ) {
            $rem = array();
            foreach ( $avail as $p ) { if ( ! in_array( $p['slug'], $used ) ) $rem[] = $p; }
            shuffle( $rem );
            while ( count( $res ) < $limit && ! empty( $rem ) ) { $res[] = array_shift( $rem ); }
        }

        error_log( 'DBA: [' . $language . '] ' . count( $res ) . ' links selected (' . count( $avail ) . ' candidates)' );
        return $res;
    }

    public function select_satellite_links( $exclude_slugs = array(), $theme = '', $subject = '' ) {
        return $this->select_contextual_links( $exclude_slugs, $theme, $subject, 8, 'en' );
    }

    /* ================================================================
     *  TOPIC WORD EXTRACTION
     * ================================================================ */

    public function extract_topic_words( $slug ) {
        $words = preg_split( '/[\s\-_\/]+/', strtolower( trim( $slug ) ) );
        $noise = array(
            'the','and','for','with','from','this','that','into','about',
            'how','what','why','when','where','are','was','were','been','have','has',
            'will','can','may','should','would','could','its','our','your','their',
            'a','an','in','on','at','to','of','by','as','is','it','or','be','do','no',
            'not','but','all','more','some','such','than','too','very','just','also',
            'set','up','per','via','use','new','way','get','let','try','run','put',
            'see','own','out','off','yet','so','if','my','we','he','she','they',
            'company','companies','services','service','guide','blog','page',
            'key','benefits','benefit','overview','introduction','complete',
            'step','steps','tips','best','top','list','things','know',
            'need','understanding','explained','everything','important',
        );
        $r = array();
        foreach ( $words as $w ) {
            $w = preg_replace( '/[^a-z0-9]/', '', $w );
            if ( strlen( $w ) >= 3 && ! in_array( $w, $noise ) && ! is_numeric( $w ) ) $r[] = $w;
        }
        return $r;
    }

    /* ================================================================
     *  KEYPHRASE BUILDER
     * ================================================================ */

    private function build_smart_keyphrases( $slug, $title ) {
        $kp = array( $title );
        $tw = $this->extract_topic_words( $slug );
        if ( empty( $tw ) ) return $kp;

        $sw = explode( '-', strtolower( $slug ) );
        $conn = array( 'of','in','for','and','the','to','a','an','on','with','from','by','or','as','at','how','set','up','your','its','our','per','via','is','are','was' );

        $parts = array();
        for ( $i = 0; $i < count( $sw ); $i++ ) {
            $w = $sw[ $i ];
            if ( in_array( $w, $conn ) || strlen( $w ) < 2 ) {
                if ( ! empty( $parts ) && count( $parts ) < 5 ) { $parts[] = $w; }
                else {
                    if ( count( $parts ) >= 2 ) {
                        $p = implode( ' ', $parts );
                        $p = trim( preg_replace( '/^(' . implode( '|', $conn ) . ')\s+/i', '', $p ) );
                        $p = trim( preg_replace( '/\s+(' . implode( '|', $conn ) . ')$/i', '', $p ) );
                        if ( str_word_count( $p ) >= 2 ) $kp[] = ucwords( $p );
                    }
                    $parts = array();
                }
                continue;
            }
            $parts[] = $w;
            if ( count( $parts ) >= 5 ) {
                $p = implode( ' ', $parts );
                $p = trim( preg_replace( '/^(' . implode( '|', $conn ) . ')\s+/i', '', $p ) );
                $p = trim( preg_replace( '/\s+(' . implode( '|', $conn ) . ')$/i', '', $p ) );
                if ( str_word_count( $p ) >= 2 ) $kp[] = ucwords( $p );
                $parts = array_slice( $parts, -2 );
            }
        }
        if ( count( $parts ) >= 2 ) {
            $p = implode( ' ', $parts );
            $p = trim( preg_replace( '/^(' . implode( '|', $conn ) . ')\s+/i', '', $p ) );
            $p = trim( preg_replace( '/\s+(' . implode( '|', $conn ) . ')$/i', '', $p ) );
            if ( str_word_count( $p ) >= 2 ) $kp[] = ucwords( $p );
        }
        for ( $i = 0; $i < count( $tw ) - 1; $i++ ) {
            $kp[] = $tw[ $i ] . ' ' . $tw[ $i + 1 ];
            if ( $i < count( $tw ) - 2 ) $kp[] = $tw[ $i ] . ' ' . $tw[ $i + 1 ] . ' ' . $tw[ $i + 2 ];
        }
        return array_unique( $kp );
    }

    /* ================================================================
     *  SITEMAP PING
     * ================================================================ */

    public function ping_sitemap() {
        $cs = get_option( 'dba_content_settings', array() );
        $url = $cs['sitemap_url'] ?? 'https://www.damalion.com/sitemap.xml';
        wp_remote_get( 'https://www.google.com/ping?sitemap=' . urlencode( $url ), array( 'timeout' => 10, 'blocking' => false ) );
    }
}
