import os
import json
from typing import List, Optional, Dict, Tuple
import sys
from datetime import datetime, timedelta
import requests
from bs4 import BeautifulSoup
import feedparser
from dotenv import load_dotenv
from trafilatura import fetch_url, extract
import openai
from urllib.parse import urlparse
from collections import Counter
import time
import re
from docx import Document
from docx.shared import Pt, RGBColor, Inches
from docx.enum.text import WD_ALIGN_PARAGRAPH
from docx.oxml import OxmlElement
from docx.opc.constants import RELATIONSHIP_TYPE as RT
from docx.oxml.ns import qn
import html5lib
import random
from websearch import web_search
from flask import Flask, request, jsonify
from io import BytesIO
import boto3
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.application import MIMEApplication


# Load environment variables
load_dotenv()


def add_hyperlink(paragraph, text, url):
    """Create a hyperlink in a paragraph."""
    # First create the w:hyperlink tag
    hyperlink = OxmlElement('w:hyperlink')

    # Create the relationship
    r_id = paragraph.part.relate_to(url, RT.HYPERLINK, is_external=True)
    hyperlink.set(qn('r:id'), r_id)

    # Create a new run
    new_run = OxmlElement('w:r')
    rPr = OxmlElement('w:rPr')

    # Add color
    c = OxmlElement('w:color')
    c.set(qn('w:val'), '0000FF')
    rPr.append(c)

    # Add underline
    u = OxmlElement('w:u')
    u.set(qn('w:val'), 'single')
    rPr.append(u)

    new_run.append(rPr)

    # Add text
    t = OxmlElement('w:t')
    t.text = text
    new_run.append(t)

    hyperlink.append(new_run)
    return hyperlink


class ContentAnalyzer:
    def __init__(self):
        self.openai_api_key = os.getenv('OPENAI_API_KEY')
        if not self.openai_api_key:
            raise ValueError("OPENAI_API_KEY environment variable is not set")

        # Set OpenAI API key
        openai.api_key = self.openai_api_key

        # Validate Google API keys
        self.google_api_key = os.getenv('GOOGLE_API_KEY')
        self.google_cx = os.getenv('GOOGLE_CX')
        if not self.google_api_key or not self.google_cx:
            raise ValueError("GOOGLE_API_KEY and GOOGLE_CX environment variables must be set")

        print("\n🔑 Validating API keys...")
        self.validate_api_key()

    def validate_api_key(self):
        """Validate OpenAI and Google API keys are working."""
        try:
            # Test OpenAI
            openai.ChatCompletion.create(
                model="gpt-3.5-turbo",
                messages=[{"role": "user", "content": "test"}],
                max_tokens=5
            )

            # Test Google Custom Search API
            base_url = "https://www.googleapis.com/customsearch/v1"
            params = {
                'key': self.google_api_key,
                'cx': self.google_cx,
                'q': 'test',
                'num': 1
            }
            response = requests.get(base_url, params=params)
            response.raise_for_status()

            print("✅ API keys validated successfully")
        except Exception as e:
            print(f"❌ Error validating API keys: {str(e)}")
            raise

    def get_user_input(self) -> tuple[str, str, str]:
        """Get keyword, industry, and location from user with input validation."""
        while True:
            keyword = input("🔑 Enter your keyword/topic: ").strip()
            if keyword:
                break
            print("❌ Keyword cannot be empty. Please try again.")

        while True:
            industry = input("🏭 Enter the industry (e.g., ev, healthcare, fintech): ").strip().lower()
            if industry:
                break
            print("❌ Industry cannot be empty. Please try again.")

        while True:
            location = input("📍 Enter the location (e.g., us, in, uk): ").strip().lower()
            if location and len(location) <= 3:
                break
            print("❌ Please enter a valid 2-3 letter country code.")

        return keyword, industry, location

    def extract_headings(self, html_content: str) -> Dict[str, List[str]]:
        """Extract all headings (h1-h6) from HTML content."""
        soup = BeautifulSoup(html_content, 'html.parser')
        headings = {
            'h1': [h.get_text().strip() for h in soup.find_all('h1')],
            'h2': [h.get_text().strip() for h in soup.find_all('h2')],
            'h3': [h.get_text().strip() for h in soup.find_all('h3')],
            'h4': [h.get_text().strip() for h in soup.find_all('h4')]
        }
        return headings

    def analyze_common_headings(self, all_headings: List[Dict[str, List[str]]]) -> Dict[str, List[str]]:
        """Analyze common themes and patterns in headings across ranking content."""
        combined_headings = {
            'h1': [],
            'h2': [],
            'h3': [],
            'h4': []
        }

        for page_headings in all_headings:
            for level in combined_headings:
                combined_headings[level].extend(page_headings.get(level, []))

        response = openai.ChatCompletion.create(
            model="gpt-3.5-turbo",
            messages=[
                {"role": "system",
                 "content": "Analyze these headings from top-ranking content and identify common themes, patterns, and important topics that should be covered."},
                {"role": "user",
                 "content": f"Analyze these headings and return common themes and topics that should be covered:\n\n{json.dumps(combined_headings, indent=2)}"}
            ]
        )

        try:
            return json.loads(response.choices[0].message.content)
        except:
            # Return a default structure if parsing fails
            return {
                "main_themes": [],
                "subtopics": [],
                "common_patterns": []
            }

    def classify_page_type(self, url: str, content: str) -> str:
        """Classify the page type based on URL and content analysis."""
        response = openai.ChatCompletion.create(
            model="gpt-3.5-turbo",
            messages=[
                {"role": "system",
                 "content": "Classify the given content as either 'service', 'blog', or 'tool' page."},
                {"role": "user", "content": f"URL: {url}\n\nContent snippet: {content[:1000]}"}
            ]
        )

        try:
            page_type = response.choices[0].message.content.lower().strip()
            if page_type not in ['service', 'blog', 'tool']:
                return 'blog'  # default to blog if classification is unclear
            return page_type
        except:
            return 'blog'  # default to blog if there's an error

    def validate_search_results(self, urls: List[str], keyword: str, industry: str) -> tuple[bool, List[str]]:
        """Validate if search results are relevant to the intended industry."""
        try:
            # Create a validation prompt with detailed industry context
            validation_prompt = f"""
            Analyze if these URLs and their content are relevant to the keyword "{keyword}" specifically in the {industry} industry.

            Keyword: {keyword}
            Industry: {industry}
            URLs to analyze: {urls[:3]}
            """

            response = openai.ChatCompletion.create(
                model="gpt-3.5-turbo",
                messages=[
                    {"role": "system",
                     "content": "You are an expert content analyst specializing in industry-specific search result validation."},
                    {"role": "user", "content": validation_prompt}
                ],
                max_tokens=200,
                temperature=0.3
            )

            analysis = response.choices[0].message.content.strip()

            # Simple validation based on response
            is_relevant = "relevant" in analysis.lower() and "not relevant" not in analysis.lower()

            return is_relevant, urls[:5]  # Return top 5 validated URLs

        except Exception as e:
            print(f"⚠️ Error in validation: {str(e)}")
            return True, urls  # Default to accepting results if validation fails

    def search_google(self, query: str, industry: str, location: str = "us", search_type: str = "organic") -> Dict:
        """Perform web search and return both search results and related questions."""
        try:
            # First try with original keyword
            full_query = f"{query} {industry}" if industry else query
            if search_type == "news":
                full_query += " news"

            print(f"\n🔍 Searching for: {full_query}")

            # Use web search to get results
            search_results = web_search(full_query)

            # Extract URLs and snippets from search results
            if search_results:
                urls = [result.get('url', '') for result in search_results if result.get('url')]
                snippets = [result.get('snippet', '') for result in search_results if result.get('snippet')]

                print(f"✅ Found {len(search_results)} initial results")

                # Look for related questions in snippets
                related_questions = self.extract_related_questions(snippets)
                print(f"✅ Found {len(related_questions)} related questions")

                # Validate search results relevance to industry
                is_relevant, validated_urls = self.validate_search_results(urls, query, industry)

                if not is_relevant:
                    print(f"⚠️ Initial results may not be relevant to {industry} industry.")
                    print("🔄 Refining search with industry context...")

                    # Try refined search
                    refined_query = f"{query} {industry} {location}"
                    print(f"🔍 Searching with refined query: {refined_query}")

                    refined_results = web_search(refined_query)
                    if refined_results:
                        search_results = refined_results
                        print(f"✅ Found {len(refined_results)} relevant results with industry context")

                return {
                    'results': search_results,
                    'related_questions': related_questions,
                    'validated_urls': validated_urls
                }
            else:
                print("⚠️ No search results found")
                return {'results': [], 'related_questions': [], 'validated_urls': []}

        except Exception as e:
            print(f"❌ Error in search: {str(e)}")
            return {'results': [], 'related_questions': [], 'validated_urls': []}

    def extract_questions_from_content(self, content: str) -> List[str]:
        """Extract potential 'People Also Ask' style questions from content."""
        response = openai.ChatCompletion.create(
            model="gpt-3.5-turbo",
            messages=[
                {"role": "system",
                 "content": "Extract relevant questions that people might ask about this topic from the given content. Return as JSON array of strings."},
                {"role": "user", "content": f"Content: {content[:2000]}"}
            ]
        )

        try:
            return json.loads(response.choices[0].message.content)
        except:
            return []  # Return empty list if parsing fails

    def get_recent_content(self, keyword: str, industry: str, location: str) -> List[Dict]:
        """Get recent content about the topic from various sources."""
        recent_content = []

        # Search news with the same location parameter
        news_urls = self.search_google(keyword, industry, location, search_type="news")
        for url in news_urls:
            try:
                content = extract(fetch_url(url))
                if content:
                    recent_content.append({
                        "title": self.extract_title(content),
                        "link": url,
                        "summary": content[:200]
                    })
            except Exception as e:
                print(f"⚠️ Error fetching news content from {url}: {e}")

        # RSS feeds
        rss_feeds = [
            "https://techcrunch.com/feed/",
            "https://www.theverge.com/rss/index.xml",
            "https://www.wired.com/feed/rss"
        ]

        for feed_url in rss_feeds:
            try:
                feed = feedparser.parse(feed_url)
                for entry in feed.entries:
                    if keyword.lower() in entry.title.lower() or keyword.lower() in entry.summary.lower():
                        recent_content.append({
                            "title": entry.title,
                            "link": entry.link,
                            "date": entry.published if 'published' in entry else '',
                            "summary": entry.summary
                        })
            except Exception as e:
                print(f"⚠️ Error fetching RSS feed {feed_url}: {e}")

        return recent_content

    def extract_title(self, content: str) -> str:
        """Extract title from content."""
        first_line = content.split('\n')[0]
        return first_line[:100]

    def generate_schema_markup(self, content_type: str, data: Dict) -> str:
        """Generate simplified schema markup based on content type."""
        schema = {
            "@context": "https://schema.org",
            "@type": "Article" if content_type == "blog" else "Service",
            "headline": data["title"],
            "datePublished": datetime.now().isoformat(),
            "description": data["meta_description"]
        }

        return json.dumps(schema, indent=2)

    def validate_content_quality(self, content: str) -> Dict[str, any]:
        """Validate content for active voice, conversational tone, redundancy, and table formatting."""
        try:
            response = openai.ChatCompletion.create(
                model="gpt-3.5-turbo",
                messages=[
                    {"role": "system", "content": """You are a content quality analyzer. Analyze the given content for:
                        1. Active voice usage
                        2. Conversational tone
                        3. Redundant phrases or ideas
                        4. Sentence variety
                        5. Table formatting and usage:
                           - Check if tables are properly formatted in markdown
                           - Verify if tables are used for appropriate content structuring
                           - Ensure tables have clear headers and aligned columns
                           - Check if tables have explanatory text
                        Return specific issues found and suggested improvements."""},
                    {"role": "user", "content": content}
                ]
            )

            analysis = json.loads(response.choices[0].message.content)
            return analysis
        except Exception as e:
            print(f"⚠️ Error in content validation: {e}")
            return {
                "errors": [],
                "suggestions": [],
                "table_issues": []
            }

    def enhance_content_style(self, content: str, analysis: Dict) -> str:
        """Improve content based on style analysis and ensure proper table formatting."""
        try:
            prompt = f"""
            Enhance this content by:
            1. Converting passive voice to active voice
            2. Making the tone more conversational
            3. Removing identified redundancies: {analysis.get('redundancies', [])}
            4. Improving sentence variety
            5. Enhancing table formatting and structure:
               - Ensure all tables use proper markdown syntax
               - Add clear column headers
               - Align numeric columns to the right
               - Add explanatory text before each table
               - Convert appropriate list-like content into tables
               - Use tables for:
                 * Feature comparisons
                 * Technical specifications
                 * Pros and cons
                 * Step-by-step processes
                 * Key metrics

            Table formatting rules:
            1. Use | for column separation
            2. Use proper header row separation with hyphens
            3. Add alignment indicators in header separation row
               - :--- for left alignment
               - :---: for center alignment
               - ---: for right alignment
            4. Keep tables focused and scannable

            Original content:
            {content}

            Table issues to fix: {analysis.get('table_issues', [])}
            """

            response = openai.ChatCompletion.create(
                model="gpt-3.5-turbo",
                messages=[
                    {"role": "system"},
                    {"role": "user", "content": prompt}
                ]
            )

            return response.choices[0].message.content
        except Exception as e:
            print(f"⚠️ Error in content enhancement: {e}")
            return content

    def get_people_also_ask_questions(self, keyword: str, industry: str) -> List[Dict]:
        """Extract 'People Also Ask' questions using web search."""
        try:
            # Use web search to get results
            search_query = f"{keyword} {industry} frequently asked questions"
            search_results = web_search(search_query)

            questions = []
            for result in search_results:
                # Extract questions from snippets
                snippet = result.get("snippet", "")
                if "?" in snippet:
                    sentences = snippet.split(".")
                    for sentence in sentences:
                        if "?" in sentence:
                            questions.append({
                                "question": sentence.strip(),
                                "source": result.get("url", "")
                            })

            # Validate and filter questions
            if questions:
                validation_prompt = f"""
                Analyze these questions for relevance to "{keyword}" in the {industry} industry.
                Questions: {json.dumps([q.get('question', '') for q in questions])}

                Return a JSON array of objects containing:
                1. "question": The original question
                2. "is_relevant": boolean indicating if it's relevant to our topic
                3. "relevance_score": 1-10 score of relevance
                4. "category": Categorize the question (e.g., "technical", "business", "comparison", etc.)
                """

                response = openai.ChatCompletion.create(
                    model="gpt-3.5-turbo",
                    messages=[
                        {"role": "system",
                         "content": "You are a content relevance analyzer specializing in question evaluation."},
                        {"role": "user", "content": validation_prompt}
                    ]
                )

                validated_questions = json.loads(response.choices[0].message.content)

                # Filter and sort questions by relevance
                relevant_questions = [
                    q for q in validated_questions
                    if q['is_relevant'] and q['relevance_score'] >= 7
                ]

                # Sort by relevance score
                relevant_questions.sort(key=lambda x: x['relevance_score'], reverse=True)

                return relevant_questions

            return []

        except Exception as e:
            print(f"⚠️ Error fetching People Also Ask questions: {e}")
            return []

    def analyze_content_length(self, content_data: List[Dict], content_type: str) -> Dict[str, int]:
        """Analyze content length and determine optimal word count."""
        try:
            # For blog content type, use fixed length of 1500 words
            if content_type == "blog":
                return {
                    "optimal_total": 1500,
                    "range": {
                        "min": 1400,  # Allow slight variation
                        "max": 1600
                    },
                    "section_targets": {
                        "introduction": 225,  # 15% of total
                        "main_content": 975,  # 65% of total
                        "conclusion": 150,  # 10% of total
                        "faq": 150  # 10% of total
                    }
                }

            # For other content types, calculate based on existing content
            word_counts = []
            for data in content_data:
                if data.get("type") == content_type and data.get("content"):
                    content = data["content"]
                    total_words = len(content.split())
                    word_counts.append(total_words)

            if word_counts:
                avg_total = sum(word_counts) / len(word_counts)
                max_total = max(word_counts)
                min_total = min(word_counts)
                median_total = sorted(word_counts)[len(word_counts) // 2]

                print(f"\n📊 Word count analysis from top results:")
                print(f"Average: {int(avg_total)} words")
                print(f"Median: {median_total} words")
                print(f"Range: {min_total} - {max_total} words")

                buffer = median_total * 0.2

                return {
                    "optimal_total": int(median_total),
                    "range": {
                        "min": max(int(median_total - buffer), min_total),
                        "max": min(int(median_total + buffer), max_total)
                    },
                    "section_targets": {
                        "introduction": int(median_total * 0.15),
                        "main_content": int(median_total * 0.65),
                        "conclusion": int(median_total * 0.10),
                        "faq": int(median_total * 0.10)
                    }
                }

            # Fallback values for non-blog content types
            default_counts = {
                "service": 2000,
                "tool": 2500
            }
            default_total = default_counts.get(content_type, 2000)

            return {
                "optimal_total": default_total,
                "range": {
                    "min": int(default_total * 0.8),
                    "max": int(default_total * 1.2)
                },
                "section_targets": {
                    "introduction": int(default_total * 0.15),
                    "main_content": int(default_total * 0.65),
                    "conclusion": int(default_total * 0.10),
                    "faq": int(default_total * 0.10)
                }
            }

        except Exception as e:
            print(f"⚠️ Error in content length analysis: {e}")
            return {
                "optimal_total": 1500,  # Default to 1500 words
                "range": {"min": 1400, "max": 1600},
                "section_targets": {
                    "introduction": 225,
                    "main_content": 975,
                    "conclusion": 150,
                    "faq": 150
                }
            }

    def validate_content_length(self, content: str, target_lengths: Dict[str, int]) -> Dict[str, any]:
        """Validate if content meets target length requirements and suggest adjustments."""
        try:
            current_length = len(content.split())
            target_range = target_lengths["range"]

            analysis = {
                "current_length": current_length,
                "target_length": target_lengths["optimal_total"],
                "within_range": target_range["min"] <= current_length <= target_range["max"],
                "adjustment_needed": 0,
                "suggestions": []
            }

            # Calculate needed adjustment
            if current_length < target_range["min"]:
                analysis["adjustment_needed"] = target_range["min"] - current_length
                analysis["suggestions"].append(
                    f"Content is {analysis['adjustment_needed']} words short of minimum length")
            elif current_length > target_range["max"]:
                analysis["adjustment_needed"] = current_length - target_range["max"]
                analysis["suggestions"].append(f"Content is {analysis['adjustment_needed']} words over maximum length")

            return analysis

        except Exception as e:
            print(f"⚠️ Error in content length validation: {e}")
            return {
                "current_length": 0,
                "target_length": target_lengths["optimal_total"],
                "within_range": False,
                "adjustment_needed": 0,
                "suggestions": []
            }

    def adjust_content_length(self, content: str, length_analysis: Dict[str, any]) -> str:
        """Adjust content length based on analysis."""
        if not length_analysis["within_range"]:
            try:
                adjustment_prompt = f"""
                {'Expand' if length_analysis['adjustment_needed'] > 0 else 'Trim'} this content by approximately {abs(length_analysis['adjustment_needed'])} words while maintaining quality and key information.

                Current content:
                {content}
                """

                response = openai.ChatCompletion.create(
                    model="gpt-3.5-turbo",
                    messages=[
                        {"role": "system",
                         "content": "You are an expert content editor. Adjust content length while maintaining quality and key information."},
                        {"role": "user", "content": adjustment_prompt}
                    ]
                )

                return response.choices[0]['message']['content']

            except Exception as e:
                print(f"⚠️ Error adjusting content length: {e}")
                return content

        return content

    def analyze_interlinking_opportunities(self, keyword: str, content: str, industry: str) -> List[Dict]:
        """Analyze and find interlinking opportunities from existing blog content."""
        try:
            print("\n🔍 Starting interlinking analysis...")
            print("Analyzing Pulse Energy blog content...")

            blog_urls = []

            # Strategy 1: Try exact keyword match on site
            search_query = f'site:pulseenergy.io/blog "{keyword}"'

            try:
                # Use web search to get results
                search_results = web_search(search_query)

                if search_results:
                    for result in search_results:
                        if "url" in result and "/blog/" in result["url"].lower():
                            blog_urls.append({
                                "url": result["url"],
                                "title": result.get("title", ""),
                                "snippet": result.get("snippet", "")
                            })

                print(f"Strategy 1: Found {len(blog_urls)} exact matches")

            except Exception as e:
                print(f"⚠️ Error in exact search: {e}")

            # Strategy 2: Try related terms if we don't have enough
            if len(blog_urls) < 3:
                # Extract key terms from the keyword
                terms = keyword.lower().split()
                for term in terms:
                    if len(term) > 2:  # Only use meaningful terms
                        related_query = f'site:pulseenergy.io/blog "{term}"'

                        try:
                            related_results = web_search(related_query)

                            if related_results:
                                for result in related_results:
                                    if "url" in result and "/blog/" in result["url"].lower():
                                        # Check if this URL is already in our list
                                        if not any(b["url"] == result["url"] for b in blog_urls):
                                            blog_urls.append({
                                                "url": result["url"],
                                                "title": result.get("title", ""),
                                                "snippet": result.get("snippet", "")
                                            })
                        except Exception as e:
                            print(f"⚠️ Error in related term search: {e}")
                            continue

                print(f"Strategy 2: Found {len(blog_urls)} total after related terms")

            # Strategy 3: Try broader industry terms if still not enough
            if len(blog_urls) < 2:
                broader_terms = []
                if 'ev' in keyword.lower() or industry == 'ev':
                    broader_terms = ['charging', 'electric vehicle', 'fleet', 'station', 'energy']
                else:
                    broader_terms = [industry, 'management', 'solution', 'system']

                for term in broader_terms:
                    try:
                        broader_query = f'site:pulseenergy.io/blog "{term}"'
                        broader_results = web_search(broader_query)

                        if broader_results:
                            for result in broader_results:
                                if "url" in result and "/blog/" in result["url"].lower():
                                    if not any(b["url"] == result["url"] for b in blog_urls):
                                        blog_urls.append({
                                            "url": result["url"],
                                            "title": result.get("title", ""),
                                            "snippet": result.get("snippet", "")
                                        })
                                        if len(blog_urls) >= 5:  # Stop once we have enough
                                            break
                    except Exception as e:
                        print(f"⚠️ Error in broader term search: {e}")
                        continue

                print(f"Strategy 3: Found {len(blog_urls)} total after broader terms")

            # Strategy 4: Fallback to sample relevant blog posts if no results
            if not blog_urls:
                print("🔄 No blog results found, using sample relevant posts...")
                sample_posts = [
                    {
                        "url": "https://pulseenergy.io/blog/ev-charging-management-systems",
                        "title": "Complete Guide to EV Charging Management Systems",
                        "snippet": "Learn about EV charging management systems, OCPP protocols, and smart charging solutions for CPOs and fleet operators."
                    },
                    {
                        "url": "https://pulseenergy.io/blog/ocpp-protocol-guide",
                        "title": "Understanding OCPP Protocol for EV Charging",
                        "snippet": "Comprehensive guide to OCPP 1.6 and 2.0.1 implementation for charging station management and interoperability."
                    },
                    {
                        "url": "https://pulseenergy.io/blog/smart-charging-solutions",
                        "title": "Smart Charging Solutions for Fleet Management",
                        "snippet": "Explore intelligent charging strategies, load balancing, and energy management for electric vehicle fleets."
                    },
                    {
                        "url": "https://pulseenergy.io/blog/charging-station-monitoring",
                        "title": "Real-time Charging Station Monitoring",
                        "snippet": "Monitor your charging infrastructure with advanced analytics, alerts, and performance tracking capabilities."
                    },
                    {
                        "url": "https://pulseenergy.io/blog/payment-solutions-ev-charging",
                        "title": "Payment Solutions for EV Charging Networks",
                        "snippet": "Streamline payment processing and billing for your charging network with integrated payment gateways."
                    }
                ]

                # Select most relevant sample posts based on keyword
                keyword_lower = keyword.lower()
                for post in sample_posts:
                    # Simple relevance check
                    title_lower = post["title"].lower()
                    snippet_lower = post["snippet"].lower()

                    # Check for keyword overlap
                    keyword_words = set(keyword_lower.split())
                    title_words = set(title_lower.split())
                    snippet_words = set(snippet_lower.split())

                    overlap = len(keyword_words.intersection(title_words.union(snippet_words)))
                    if overlap > 0 or any(word in title_lower + snippet_lower for word in keyword_words):
                        blog_urls.append(post)
                        if len(blog_urls) >= 3:  # Limit to top 3 most relevant
                            break

                print(f"Strategy 4: Using {len(blog_urls)} sample posts")

            if not blog_urls:
                print("⚠️ No relevant blog posts found")
                return []

            print(f"✅ Found {len(blog_urls)} potential blog posts")

            # Calculate relevance scores and filter
            interlink_suggestions = []
            for blog in blog_urls:
                try:
                    # Calculate relevance score
                    relevance_score = self.calculate_relevance_score(
                        keyword,
                        blog["title"],
                        blog["snippet"]
                    )

                    if relevance_score >= 5.0:  # Lower threshold for sample posts
                        suggestion = {
                            "url": blog["url"],
                            "title": blog["title"],
                            "description": blog["snippet"][:200],
                            "relevance_score": relevance_score
                        }
                        interlink_suggestions.append(suggestion)
                        print(f"✅ Found relevant blog post: {blog['title']}")
                        print(f"📊 Relevance score: {relevance_score:.1f}/10")

                except Exception as e:
                    print(f"⚠️ Error analyzing blog {blog['url']}: {e}")
                    continue

            # Sort by relevance score
            interlink_suggestions.sort(key=lambda x: x["relevance_score"], reverse=True)

            # Take top 5 most relevant suggestions
            return interlink_suggestions[:5]

        except Exception as e:
            print(f"⚠️ Error analyzing interlinking opportunities: {e}")
            return []

    def enhance_content_with_links(self, content: str, link_suggestions: List[Dict], can_add_cta: bool) -> str:
        """Add internal links and natural CTAs to the content in appropriate contexts."""
        try:
            if not link_suggestions:
                return content

            # First, analyze content for CTA opportunities
            website_analyzer = WebsiteAnalyzer()
            content_analysis = website_analyzer.verify_content_accuracy(content)

            # Create a prompt that focuses on natural content enhancement
            enhancement_prompt = f"""
            Enhance this content by adding relevant blog post links and natural CTAs where appropriate.
            The content should flow naturally and maintain its educational value first.

            Content:
            {content}

            Blog posts to link (in order of relevance):
            {json.dumps(link_suggestions, indent=2)}

            CTA Opportunities:
            {json.dumps(content_analysis.get('cta_opportunities', []), indent=2)}

            Rules:
            1. Keep the content flow natural and educational
            2. Only add links where contextually relevant
            3. Use varied anchor text that matches the context
            4. Space out links throughout the content
            5. Add at least one link per blog post if context allows
            6. Don't add links in headings
            7. Don't break existing links
            8. Place CTAs only where they naturally fit the discussion
            9. Limit to 2-3 CTAs maximum
            10. Ensure each CTA adds value to the reader
            11. Return the entire enhanced content

            Can add CTAs: {"Yes" if can_add_cta and content_analysis.get('can_add_cta', False) else "No"}
            """

            response = openai.ChatCompletion.create(
                model="gpt-3.5-turbo",
                messages=[
                    {"role": "system",
                     "content": "You are an expert at naturally integrating educational content with relevant links and value-adding CTAs."},
                    {"role": "user", "content": enhancement_prompt}
                ]
            )

            enhanced_content = response.choices[0].message.content

            # Verify links were actually added
            links_added = False
            for suggestion in link_suggestions:
                if suggestion["url"] in enhanced_content:
                    links_added = True
                    break

            if not links_added:
                print("⚠️ No links were added to the content. Trying again with simplified approach...")
                # Try a simpler approach
                for suggestion in link_suggestions:
                    keyword = suggestion["title"].lower()
                    url = suggestion["url"]
                    # Look for the keyword in the content
                    pattern = re.compile(re.escape(keyword), re.IGNORECASE)
                    if pattern.search(content):
                        # Replace first occurrence with linked version
                        enhanced_content = pattern.sub(f"[{keyword}]({url})", content, count=1)
                        print(f"✅ Added link for: {keyword}")
                        links_added = True
                        break

            # If CTAs are allowed, verify they were added naturally
            if can_add_cta and content_analysis.get('can_add_cta', False):
                ctas_found = sum(1 for opp in content_analysis['cta_opportunities'] if
                                 website_analyzer.generate_natural_cta(
                                     {'term': opp['term'], 'type': opp['type']}) in enhanced_content)

                if ctas_found == 0:
                    print("⚠️ No CTAs were added. Adding one natural CTA...")
                    # Add one natural CTA at the most appropriate spot
                    best_opportunity = max(content_analysis['cta_opportunities'],
                                           key=lambda x: len(x['context'].split()))
                    natural_cta = website_analyzer.generate_natural_cta(best_opportunity)

                    # Find a natural break in the content (like end of a section) to add the CTA
                    sections = enhanced_content.split('\n\n')
                    if len(sections) > 2:  # Ensure we have enough sections
                        # Add CTA after a relevant section
                        for i, section in enumerate(sections[1:-1], 1):  # Skip first and last sections
                            if best_opportunity['term'].lower() in section.lower():
                                sections.insert(i + 1, f"\n{natural_cta}\n")
                                enhanced_content = '\n\n'.join(sections)
                                print("✅ Added natural CTA in relevant context")
                                break

            if links_added:
                print("✅ Successfully enhanced content with links and natural CTAs")
            else:
                print("⚠️ Could not find suitable contexts for enhancement")
                enhanced_content = content

            return enhanced_content

        except Exception as e:
            print(f"⚠️ Error enhancing content: {e}")
            return content

    def generate_interlinking_report(self, link_suggestions: List[Dict], keyword: str) -> str:
        """Generate a detailed interlinking implementation report."""
        try:
            report_prompt = f"""
            Create a detailed internal linking implementation report for content about "{keyword}" on pulseenergy.io.

            Link suggestions:
            {json.dumps(link_suggestions, indent=2)}

            Include in the report:
            1. Executive Summary
            2. Linking Strategy Overview
            3. Detailed Implementation Instructions for each link
            4. Expected SEO Benefits
            5. Priority Order for Implementation

            Format the report in Markdown.
            """

            response = openai.ChatCompletion.create(
                model="gpt-3.5-turbo",
                messages=[
                    {"role": "system",
                     "content": "You are an SEO expert creating internal linking implementation reports."},
                    {"role": "user", "content": report_prompt}
                ]
            )

            return response.choices[0]['message']['content']

        except Exception as e:
            print(f"⚠️ Error generating interlinking report: {e}")
            return "Error generating report"

    def polish_generated_document(self, doc_path: str, keyword: str, location: str = "us") -> str:
        """Polish the generated document with specific requirements."""
        try:
            print("\n✨ Polishing the generated document...")

            # Read the generated document
            doc = Document(doc_path)

            # Extract content from different sections
            extracted_content = {
                'title': '',
                'meta_description': '',
                'content': '',
                'faq_section': '',
                'schema_markup': '',
                'interlinking_opportunities': ''
            }

            current_section = None
            content_parts = []

            for paragraph in doc.paragraphs:
                text = paragraph.text.strip()
                if not text:
                    continue

                # Identify sections based on headings
                if text.lower().startswith('meta description'):
                    current_section = 'meta_description'
                    continue
                elif text.lower().startswith('content'):
                    current_section = 'content'
                    continue
                elif text.lower().startswith('frequently asked questions') or text.lower().startswith('faq'):
                    current_section = 'faq_section'
                    continue
                elif text.lower().startswith('schema markup'):
                    current_section = 'schema_markup'
                    continue
                #elif text.lower().startswith('blog interlinking'):
                    #current_section = 'interlinking_opportunities'
                    #continue
                elif text.lower().startswith('content accuracy'):
                    continue  # Skip accuracy report section
                else:
                    # First non-empty paragraph is likely the title
                    if not extracted_content['title'] and current_section is None:
                        extracted_content['title'] = text
                        continue

                # Add content to appropriate section
                if current_section == 'content':
                    content_parts.append(text)
                elif current_section and current_section != 'content':
                    if extracted_content[current_section]:
                        extracted_content[current_section] += f"\n{text}"
                    else:
                        extracted_content[current_section] = text

            # Combine main content
            extracted_content['content'] = '\n\n'.join(content_parts)

            if not extracted_content['content']:
                raise Exception("Could not extract content from document")

            print("✅ Content extracted from document")

            # Generate topic-relevant FAQ questions first
            faq_prompt = f"""
            Generate 5 relevant, practical FAQ questions about "{keyword}".
            Location context: {location.upper()}

            Return ONLY a JSON array of strings with the questions:
            ["question 1", "question 2", "question 3", "question 4", "question 5"]

            Make them practical, specific to the topic, and relevant to the target audience's real concerns.
            """

            print("🤔 Generating topic-relevant FAQ questions...")

            faq_response = openai.ChatCompletion.create(
                model="gpt-3.5-turbo",
                messages=[
                    {"role": "system",
                     "content": "You are an expert at creating relevant FAQ questions for EV industry professionals. Return only JSON arrays."},
                    {"role": "user", "content": faq_prompt}
                ],
                temperature=0.3,
                max_tokens=300
            )

            try:
                relevant_faq_questions = json.loads(faq_response.choices[0].message.content.strip())
            except json.JSONDecodeError:
                print("⚠️ Using fallback FAQ questions")
                relevant_faq_questions = [
                    f"What are the key benefits of {keyword}",
                    f"How does {keyword} impact operational costs?",
                    f"What should everyone know about {keyword} implementation?",
                    f"What are the common challenges with {keyword}?",
                    f"How do you choose the right {keyword} solution?"
                ]

            # Create enhanced polishing prompt with interlinking information
            polishing_prompt = f"""
            You are transforming content about "{keyword}" in {location.upper()}.

            CRITICAL INSTRUCTIONS:
            1. Write EXACTLY 1500 words in the main content
            2. Title format: " [Topic] in [Location]?"
            3. Use conversational tone with "you", "your", direct address
            4. Include practical, actionable advice
            5. PRESERVE ALL EXISTING INTERNAL LINKS - do not remove any links that are already in the content
            6. Return ONLY valid JSON (no extra text before or after)

            Original content to transform (ALREADY HAS INTERNAL LINKS INTEGRATED):
            {extracted_content['content']}

            FAQ questions to answer:
            {json.dumps(relevant_faq_questions, indent=2)}

            WRITING STYLE:
            - "When you're setting up solar finance..."
            - "Your operations depend on solar finance..."
            - "Here's what you need to consider..."
            - "Let's break down the key factors..."

            LINK PRESERVATION INSTRUCTIONS:
            - Keep ALL existing links in the content exactly as they are
            - Maintain any [text](url) format links
            - Do not remove or modify existing internal links
            - You can enhance the context around existing links to make them more natural

            Return this exact JSON structure with NO other text:
            {{
                "polished_title": "{keyword.title()}",
                "polished_meta_description": "Essential guide  on {keyword} in India. Practical insights and actionable advice.",
                "polished_content": "EXACTLY 1500 words of conversational content addressing directly with practical advice, actionable insights about {keyword} in india"
                "polished_faq": "Brief FAQ section answering the key questions with practical, actionable answers"
            }}
            """

            print("🔄 Applying advanced polishing with gpt-3.5-turbo...")

            response = openai.ChatCompletion.create(
                model="gpt-3.5-turbo",
                messages=[
                    {"role": "system",
                     "content": "You are an expert content editor. You MUST return valid JSON with exactly 1500 words in the polished_content field. No exceptions."},
                    {"role": "user", "content": polishing_prompt}
                ],
                temperature=0.1,  # Even lower temperature for consistency
                max_tokens=4000  # Increased for longer content
            )

            # Enhanced JSON parsing with multiple fallback strategies
            response_text = response.choices[0].message.content.strip()

            # Clean the response text
            response_text = response_text.replace('```json', '').replace('```', '').strip()

            # Try to extract JSON from response
            polished_result = None

            # Strategy 1: Direct JSON parsing
            try:
                polished_result = json.loads(response_text)
                print("✅ Successfully parsed JSON response")
            except json.JSONDecodeError as e:
                print(f"⚠️ Direct JSON parsing failed: {str(e)}")

                # Strategy 2: Find JSON object in text
                json_pattern = r'\{[^{}]*"polished_title"[^{}]*"polished_meta_description"[^{}]*"polished_content"[^{}]*"polished_faq"[^{}]*\}'
                json_match = re.search(json_pattern, response_text, re.DOTALL)

                if json_match:
                    try:
                        # Extract the matched JSON and try to parse it
                        json_text = json_match.group(0)
                        polished_result = json.loads(json_text)
                        print("✅ Extracted JSON using pattern matching")
                    except json.JSONDecodeError:
                        print("⚠️ Pattern matching failed")

                # Strategy 3: Manual extraction
                if not polished_result:
                    print("⚠️ Attempting manual JSON extraction...")
                    try:
                        # Find start and end of JSON object
                        start = response_text.find('{')
                        end = response_text.rfind('}') + 1
                        if start != -1 and end > start:
                            json_text = response_text[start:end]
                            polished_result = json.loads(json_text)
                            print("✅ Manual extraction successful")
                    except:
                        print("⚠️ Manual extraction failed")

            # Strategy 4: Force regeneration if parsing completely fails
            if not polished_result:
                print("🔄 JSON parsing failed completely. Regenerating with simpler prompt...")

                simple_prompt = f"""
                Create 1500-word conversational content about {keyword} in {location.upper()}.

                Title: {keyword.title()} in {location.upper()}?

                Write in conversational tone addressing the reader directly. Include practical advice.
                """

                simple_response = openai.ChatCompletion.create(
                    model="gpt-3.5-turbo",
                    messages=[
                        {"role": "system", "content": "Write conversational content for solar finance ."},
                        {"role": "user", "content": simple_prompt}
                    ],
                    temperature=0.1
                )

                polished_result = {
                    "polished_title": f"{keyword.title()} in {location.upper()}?",
                    "polished_meta_description": f"Essential guide on {keyword} in {location.upper()}. Get practical insights and actionable advice.",
                    "polished_content": simple_response.choices[0].message.content,
                    "polished_faq": "\n\n".join([
                                                    f"**Q: {q}**\nA: This depends on your specific operational requirements and local regulations in {location.upper()}."
                                                    for q in relevant_faq_questions[:3]])
                }
                print("✅ Generated content with fallback approach")

            # Aggressive content expansion to hit 1500 words
            word_count = len(polished_result["polished_content"].split())
            print(f"📊 Initial content word count: {word_count}")

            if word_count < 1400:  # Aggressive expansion
                print("🔄 Content significantly under target, applying aggressive expansion...")

                expansion_prompt = f"""
                Expand this {word_count}-word content to EXACTLY 1500 words.

                EXPANSION STRATEGY:
                - Add more detailed examples
                - Include specific benefits for solar finance
                - Add implementation steps
                - Include cost considerations
                - Add regulatory context for {location.upper()}
                - Include case studies or scenarios
                - Include best practices

                Original content:
                {polished_result["polished_content"]}

                Target: EXACTLY 1500 words
                Tone: Keep conversational and practical
                Location: {location.upper()}

                Return ONLY the expanded content, nothing else.
                """

                expansion_response = openai.ChatCompletion.create(
                    model="gpt-3.5-turbo",
                    messages=[
                        {"role": "system",
                         "content": "You are an expert at expanding content to exact word counts while maintaining quality and relevance."},
                        {"role": "user", "content": expansion_prompt}
                    ],
                    temperature=0.1,
                    max_tokens=4000
                )

                polished_result["polished_content"] = expansion_response.choices[0].message.content.strip()
                final_word_count = len(polished_result["polished_content"].split())
                print(f"📊 After expansion: {final_word_count} words")

                # If still short, do one more expansion
                if final_word_count < 1400:
                    print("🔄 Still short, doing final expansion...")
                    additional_content = f"""

## Additional Considerations for {location.upper()}

When you're implementing {keyword} in {location.upper()}, you'll need to consider several additional factors that are specific to the region. Understanding these nuances will help you make better decisions.

### Regulatory Environment in {location.upper()}

The regulatory landscape in {location.upper()} continues to evolve. You'll want to stay updated on local requirements and standards that may affect your charging infrastructure decisions.

### Best Practices for Implementation

Based on successful deployments in {location.upper()}, here are proven strategies that work:

1. **Planning Phase**: Start with a comprehensive site assessment
2. **Installation**: Work with certified local installers
3. **Operations**: Establish clear maintenance protocols
4. **Monitoring**: Implement real-time performance tracking

### Cost Optimization Strategies

Managing costs effectively requires understanding both upfront investments and ongoing operational expenses. Consider factors like energy costs, maintenance requirements, and potential government incentives available in {location.upper()}.

### Future-Proofing Your Investment

Technology in this space evolves rapidly. When you're making decisions about {keyword}, think about how your choices will serve your fleet not just today, but in the years to come.
                    """

                    polished_result["polished_content"] += additional_content
                    final_word_count = len(polished_result["polished_content"].split())
                    print(f"📊 Final word count after padding: {final_word_count} words")

            # Create polished document with clean formatting
            polished_doc = Document()

            # Add polished title (clean, no JSON artifacts)
            title = polished_doc.add_heading(polished_result["polished_title"], 0)
            title.alignment = WD_ALIGN_PARAGRAPH.CENTER

            # Add polished meta description
            polished_doc.add_heading('Meta Description', level=1)
            polished_doc.add_paragraph(polished_result["polished_meta_description"])
            polished_doc.add_paragraph()

            # Add polished content with proper formatting and interlinking
            #polished_doc.add_heading('Final Content (Ready to Publish)', level=1)

            # Process the content to actively hyperlink relevant keywords
            content_text = polished_result["polished_content"]

            # Extract interlinking opportunities and create keyword-to-URL mapping
            if extracted_content.get('interlinking_opportunities'):
                print("🔗 Processing internal links for keyword hyperlinking...")

                # Parse interlinking opportunities to extract blog post info
                interlink_lines = extracted_content['interlinking_opportunities'].split('\n')
                blog_posts = []
                current_post = {}

                for line in interlink_lines:
                    line = line.strip()
                    if line.startswith('•') or line.startswith('-'):
                        # Save previous post if exists
                        if current_post.get('title') and current_post.get('url'):
                            blog_posts.append(current_post)
                        # Start new post
                        current_post = {'title': line.lstrip('•-').strip()}
                    elif line.startswith('URL:'):
                        current_post['url'] = line.replace('URL:', '').strip()
                    elif 'Relevance Score:' in line:
                        score_text = line.split('Relevance Score:', 1)[1].strip()
                        try:
                            current_post['relevance_score'] = float(score_text.split('/')[0])
                        except:
                            current_post['relevance_score'] = 6.0

                # Add last post
                if current_post.get('title') and current_post.get('url'):
                    blog_posts.append(current_post)

                if blog_posts:
                    print(f"📋 Found {len(blog_posts)} blog posts to link")

                    # Create keyword linking prompt
                    linking_prompt = f"""
                    You are an expert at identifying relevant keywords in content and creating natural hyperlinks.

                    TASK: Identify 2-3 highly relevant keywords/phrases in this content that relate to the blog posts below, and create hyperlinks for them.

                    CONTENT:
                    {content_text}

                    AVAILABLE BLOG POSTS TO LINK:
                    {json.dumps([{'title': post['title'], 'url': post['url'], 'relevance': post.get('relevance_score', 6.0)} for post in blog_posts], indent=2)}

                    INSTRUCTIONS:
                    1. Find 2-3 relevant keywords or phrases in the content that naturally relate to these blog posts
                    2. Replace those keywords with markdown links: [keyword](url)
                    3. Only link keywords that are genuinely relevant - don't force links
                    4. Choose the most contextually appropriate blog post URL for each keyword
                    5. Vary the anchor text (don't use the exact blog post title)
                    6. Make the links feel natural in the content flow

                    EXAMPLES of good linking:
                    - "EV charging connectors" → [EV charging connectors](https://pulseenergy.io/blog/ev-connectors-guide)
                    - "OCPP protocol" → [OCPP implementation](https://pulseenergy.io/blog/ocpp-guide) 
                    - "charging station management" → [managing charging infrastructure](https://pulseenergy.io/blog/station-management)

                    Return ONLY the content with the hyperlinks added. Do not change anything else about the content.
                    """

                    print("🔗 Creating hyperlinks for relevant keywords...")

                    linking_response = openai.ChatCompletion.create(
                        model="gpt-3.5-turbo",
                        messages=[
                            {"role": "system",
                             "content": "You are an expert at creating natural, contextual hyperlinks in content. You only modify content to add relevant hyperlinks and change nothing else."},
                            {"role": "user", "content": linking_prompt}
                        ],
                        temperature=0.1
                    )

                    content_text = linking_response.choices[0].message.content.strip()

                    # Count how many links were added
                    link_count = content_text.count('](https://pulseenergy.io')
                    print(f"✅ Added {link_count} contextual hyperlinks to relevant keywords")
                else:
                    print("⚠️ No valid blog posts found for linking")

            # Split content into paragraphs and add to document with hyperlink processing
            content_paragraphs = content_text.split('\n\n')

            for paragraph_text in content_paragraphs:
                if paragraph_text.strip():
                    # Check if it's a heading (starts with #)
                    if paragraph_text.strip().startswith('#'):
                        # Remove markdown heading markers and add as heading
                        heading_text = paragraph_text.strip().lstrip('#').strip()
                        heading_level = min(paragraph_text.count('#'), 3)  # Max level 3
                        polished_doc.add_heading(heading_text, level=heading_level)
                    else:
                        # Add as regular paragraph with hyperlink processing
                        para = polished_doc.add_paragraph()

                        # Process markdown links [text](url)
                        link_pattern = r'\[([^\]]+)\]\(([^)]+)\)'

                        # Split text by links
                        parts = re.split(link_pattern, paragraph_text)

                        for i, part in enumerate(parts):
                            if i % 3 == 0:  # Regular text
                                if part:
                                    para.add_run(part)
                            elif i % 3 == 1:  # Link text
                                link_text = part
                                if i + 1 < len(parts):
                                    link_url = parts[i + 1]
                                    # Create hyperlink
                                    hyperlink = add_hyperlink(para, link_text, link_url)
                                    para._p.append(hyperlink)
                                    print(f"🔗 Created hyperlink: {link_text} → {link_url}")
                            # Skip i % 3 == 2 as it's the URL part already processed

            polished_doc.add_paragraph()

            # Add clean FAQ section (no JSON artifacts)
            if polished_result.get("polished_faq"):
                faq_data = polished_result["polished_faq"]

                # Handle both string and list formats for FAQ
                if isinstance(faq_data, list):
                    # FAQ is already a list of questions
                    polished_doc.add_heading('Frequently Asked Questions', level=1)

                    for item in faq_data:
                        if isinstance(item, dict) and 'question' in item and 'answer' in item:
                            # Structured Q&A format
                            q_para = polished_doc.add_paragraph()
                            q_run = q_para.add_run(f"Q: {item['question']}")
                            q_run.bold = True

                            a_para = polished_doc.add_paragraph()
                            a_para.add_run(f"A: {item['answer']}")

                            polished_doc.add_paragraph()  # Space between Q&As
                        elif isinstance(item, str) and item.strip():
                            # Plain string format
                            if item.strip().startswith('Q:') or item.strip().startswith('**Q:'):
                                q_para = polished_doc.add_paragraph()
                                q_run = q_para.add_run(item.strip().replace('**', ''))
                                q_run.bold = True
                            else:
                                a_para = polished_doc.add_paragraph()
                                a_para.add_run(item.strip().replace('**', ''))
                                polished_doc.add_paragraph()

                elif isinstance(faq_data, str) and faq_data.strip():
                    # FAQ is a string, process it
                    polished_doc.add_heading('Frequently Asked Questions', level=1)

                    faq_content = faq_data
                    # Remove any JSON artifacts from FAQ
                    faq_content = faq_content.replace('"polished_faq":', '').replace('[', '').replace(']', '').replace(
                        '{', '').replace('}', '')

                    # Split FAQ by Q: markers
                    faq_items = faq_content.split('**Q:')

                    for item in faq_items[1:]:  # Skip first empty item
                        if '**' in item and 'A:' in item:
                            parts = item.split('**')
                            if len(parts) >= 2:
                                question = parts[0].strip()
                                answer_part = parts[1] if len(parts) > 1 else item
                                if 'A:' in answer_part:
                                    answer = answer_part.split('A:', 1)[1].strip()

                                    # Add question
                                    q_para = polished_doc.add_paragraph()
                                    q_run = q_para.add_run(f"Q: {question}")
                                    q_run.bold = True

                                    # Add answer
                                    a_para = polished_doc.add_paragraph()
                                    a_para.add_run(f"A: {answer}")

                                    polished_doc.add_paragraph()  # Space between Q&As

            # Add internal links reference section
            if extracted_content.get('interlinking_opportunities'):
                polished_doc.add_heading('Internal Links Used in This Content', level=1)

                interlink_lines = extracted_content['interlinking_opportunities'].split('\n')
                current_link = {}

                for line in interlink_lines:
                    line = line.strip()
                    if line.startswith('•') or line.startswith('-'):
                        # Start of a new link
                        if current_link:
                            # Add previous link
                            link_para = polished_doc.add_paragraph(style='List Bullet')
                            link_para.add_run(f"{current_link.get('title', 'Link')}\n").bold = True
                            if current_link.get('url'):
                                link_para.add_run(f"URL: {current_link['url']}\n")
                            if current_link.get('relevance'):
                                link_para.add_run(f"Relevance Score: {current_link['relevance']}\n")
                        current_link = {'title': line.lstrip('•-').strip()}
                    elif line.startswith('URL:'):
                        current_link['url'] = line.replace('URL:', '').strip()
                    elif 'Relevance Score:' in line:
                        current_link['relevance'] = line.split('Relevance Score:', 1)[1].strip()

                # Add last link
                if current_link:
                    link_para = polished_doc.add_paragraph(style='List Bullet')
                    link_para.add_run(f"{current_link.get('title', 'Link')}\n").bold = True
                    if current_link.get('url'):
                        link_para.add_run(f"URL: {current_link['url']}\n")
                    if current_link.get('relevance'):
                        link_para.add_run(f"Relevance Score: {current_link['relevance']}\n")

            # Add schema markup (technical reference, not part of main content)
            if extracted_content.get('schema_markup'):
                polished_doc.add_heading('Technical: Schema Markup', level=1)
                schema_paragraph = polished_doc.add_paragraph()
                schema_run = schema_paragraph.add_run(extracted_content['schema_markup'])
                schema_run.font.name = 'Courier New'
                schema_run.font.size = Pt(10)

            # Save polished document
            base_name = os.path.splitext(doc_path)[0]
            polished_path = f"{base_name}_POLISHED.docx"
            polished_doc.save(polished_path)

            print(f"✅ Clean, publishable document saved as: {polished_path}")

            # Final word count verification
            final_word_count = len(polished_result["polished_content"].split())
            print(f"📊 Final word count: {final_word_count} words")

            if abs(final_word_count - 1500) <= 100:  # Within tolerance
                print("✅ Word count target achieved!")
            else:
                print(f"⚠️ Word count deviation: Target was 1500, got {final_word_count}")

            return polished_path

        except Exception as e:
            print(f"❌ Error polishing document: {str(e)}")
            raise
          
        
    def send_email_with_doc(self,receiver_email, topic, filename, doc_stream):

        try:
            # Create SES client
            client = boto3.client(
                'ses',
                region_name=os.getenv('AWS_REGION_NAME'),
                aws_access_key_id=os.getenv("SES_KEY"),
                aws_secret_access_key=os.getenv("SES_KEY_SECRET")
            )
            

            sender_email = os.getenv("MAIL_USERNAME")

            subject = f"Blog content: {topic}"
            body_text = (
            f"Hello,\n\n"
            f"Thank you for requesting information about \"{topic}\".\n\n"
            "Please find the detailed document attached. \n\n"
            "Best regards,\n"
            "Credit Fair Team"
        )

            # Build email
            msg = MIMEMultipart()
            msg['Subject'] = subject
            msg['From'] = sender_email
            msg['To'] = receiver_email

            # Add plain text body
            msg.attach(MIMEText(body_text, 'plain'))

            # Reset stream pointer and attach file
            doc_stream.seek(0)
            part = MIMEApplication(doc_stream.read(), _subtype="vnd.openxmlformats-officedocument.wordprocessingml.document")
            part.add_header('Content-Disposition', 'attachment', filename=filename)
            msg.attach(part)

            # Send via SES
            response = client.send_raw_email(
                Source=sender_email,
                Destinations=[receiver_email],
                RawMessage={'Data': msg.as_string()}
            )

            print("✅ Email sent! Message ID:", response['MessageId'])
            return True

        except Exception as ex:
            print(f"❌ An error occurred: {ex}")
            return False
             
    
    def save_content(self, content: Dict, keyword: str, location: str = "us", receiver_email: str = None) -> None:
        """Save the final polished content and interlinking report in a keyword-specific folder."""
        try:
            # Create base directory
            base_dir = os.path.expanduser("~/Documents/ev_content")
            os.makedirs(base_dir, exist_ok=True)

            # Create keyword-specific folder
            safe_keyword = "".join(c if c.isalnum() else "_" for c in keyword.lower())
            keyword_dir = os.path.join(base_dir, safe_keyword)
            os.makedirs(keyword_dir, exist_ok=True)

            # Create safe filename from title
            safe_title = "".join(c if c.isalnum() else "_" for c in content['title'].lower())

            # Create Word document in memory (not saved to disk yet)
            doc = Document()

            # Add title
            title = doc.add_heading(content['title'], 0)
            title.alignment = WD_ALIGN_PARAGRAPH.CENTER

            # Add meta description
            doc.add_heading('Meta Description', level=1)
            doc.add_paragraph(content['meta_description'])
            doc.add_paragraph()

            # Fact check content against website
            print("\n🔍 Fact checking content against Pulse Energy website...")
            website_analyzer = WebsiteAnalyzer()
            website_data = website_analyzer.scrape_website()

            if website_data:
                print("✅ Website data collected successfully")
                accuracy_analysis = website_analyzer.verify_content_accuracy(content['content'])

                if not accuracy_analysis['accurate']:
                    print("\n⚠️ Content accuracy issues found:")
                    for issue in accuracy_analysis['issues']:
                        print(f"  • {issue}")
                    print("\nSuggested changes:")
                    for change in accuracy_analysis['suggested_changes']:
                        print(f"  • {change}")

                    doc.add_heading('Content Accuracy Report', level=1)
                    doc.add_paragraph("The following accuracy issues were identified:")
                    for issue in accuracy_analysis['issues']:
                        doc.add_paragraph(f"• {issue}", style='List Bullet')
                    doc.add_paragraph("\nSuggested changes:")
                    for change in accuracy_analysis['suggested_changes']:
                        doc.add_paragraph(f"• {change}", style='List Bullet')
                    doc.add_paragraph()
                else:
                    print("✅ Content accuracy verified")

            # Add main content
            doc.add_heading('Content', level=1)
            clean_content = re.sub('<[^<]+?>', '', content['content'])
            doc.add_paragraph(clean_content)
            doc.add_paragraph()

            # Add FAQ section
            if content.get('faq_section'):
                doc.add_heading('Frequently Asked Questions', level=1)
                for qa in content['faq_section']:
                    question = doc.add_paragraph()
                    question.add_run(f"Q: {qa['question']}").bold = True
                    answer = doc.add_paragraph()
                    answer.add_run(f"A: {qa['answer']}")
                    doc.add_paragraph()

            # Add interlinking opportunities
            print("\n🔗 Analyzing blog posts for interlinking opportunities...")
            link_suggestions = self.analyze_interlinking_opportunities(keyword, content['content'], 'blog')
            if link_suggestions:
                print(f"Found {len(link_suggestions)} relevant blog posts for interlinking")
                doc.add_heading('Blog Interlinking Opportunities', level=1)
                for link in link_suggestions:
                    link_para = doc.add_paragraph(style='List Bullet')
                    link_para.add_run(f"{link['title']}\n").bold = True
                    link_para.add_run(f"URL: {link['url']}\n")
                    link_para.add_run(f"Relevance Score: {link['relevance_score']:.1f}/10\n")
                    link_para.add_run(f"Description: {link['description']}")
                    doc.add_paragraph()

            # Add Schema Markup section
            doc.add_heading('Schema Markup', level=1)
            schema_paragraph = doc.add_paragraph()
            schema_run = schema_paragraph.add_run(content['schema_markup'])
            schema_run.font.name = 'Courier New'
            schema_run.font.size = Pt(10)

            # Save to temporary file
            temp_doc_path = os.path.join(keyword_dir, "temp.docx")
            doc.save(temp_doc_path)

            # Polish the document and overwrite with final one
            print("\n🎨 Starting document polishing process...")
            polished_path = self.polish_generated_document(temp_doc_path, keyword, location)

            # Final output path
            final_doc_path = os.path.join(keyword_dir, f"{safe_title}.docx")
            os.rename(polished_path, final_doc_path)

            # Clean up temp file
            if os.path.exists(temp_doc_path):
                os.remove(temp_doc_path)

            print(f"\n✅ Final polished document saved as: {final_doc_path}")
            
            
                        # Read the polished document into memory
            with open(final_doc_path, "rb") as f:
                doc_stream = BytesIO(f.read())

            # Send the document as an email attachment
            self.send_email_with_doc(receiver_email, keyword, safe_title + ".docx", doc_stream)

            # Optionally, remove the local file if you don't want to keep it
            os.remove(final_doc_path)
                
                

        except Exception as e:
            print(f"❌ Error saving content: {e}")
            raise
        
    
    def search_existing_content(self, keyword: str) -> List[Dict]:
        """Search for content within pulseenergy.io using web search."""
        try:
            # Setup search query for site-specific content
            search_query = f'site:pulseenergy.io "{keyword}"'

            # Use web search to get results
            search_results = web_search(search_query)

            # Process search results
            if search_results:
                best_match = search_results[0]  # Get the first (most relevant) result

                return [{
                    "url": best_match["url"],
                    "title": best_match.get("title", ""),
                    "snippet": best_match.get("snippet", ""),
                    "relevance_score": 9.0  # High relevance since it's an exact match
                }]

            print(f"⚠️ No matching content found for keyword: {keyword}")
            return []

        except Exception as e:
            print(f"⚠️ Error searching content: {e}")
            return []

    def calculate_relevance_score(self, keyword: str, title: str, description: str) -> float:
        """Calculate relevance score based on title and description match."""
        try:
            prompt = f"""
            Analyze how relevant this content is to the keyword "{keyword}" in the context of EV charging.

            Content:
            Title: {title}
            Description: {description}

            Return a JSON object with:
            {{
                "score": float between 0-10,
                "reason": "Brief explanation of the score"
            }}

            Consider:
            1. Direct keyword match
            2. Semantic relevance
            3. Technical context
            4. User intent match
            """

            response = openai.ChatCompletion.create(
                model="gpt-3.5-turbo",
                messages=[
                    {"role": "system",
                     "content": "You are an expert at analyzing content relevance for EV charging topics."},
                    {"role": "user", "content": prompt}
                ]
            )

            try:
                result = json.loads(response.choices[0].message.content)
                return float(result["score"])
            except (json.JSONDecodeError, KeyError):
                # Return a default score if parsing fails
                return 5.0

        except Exception as e:
            print(f"⚠️ Error calculating relevance: {e}")
            # Default to medium score if analysis fails
            return 5.0

    def extract_keywords(self, text: str) -> List[str]:
        """Extract the most relevant keywords from the text."""
        try:
            # Define EV charging specific stopwords to exclude
            domain_stopwords = {
                'the', 'and', 'or', 'in', 'at', 'on', 'to', 'for', 'of', 'with',
                'by', 'from', 'up', 'about', 'into', 'over', 'after'
            }

            # Define high-priority EV charging terms
            priority_terms = {

                    'solar finance', 'solar loan', 'solar panel financing', 'solar emi',
                    'solar subsidy', 'renewable energy loan', 'solar rooftop loan',
                    'solar installation cost', 'solar financing scheme', 'solar credit',
                    'green energy loan', 'solar leasing', 'solar ppa', 'net metering',
                    'solar interest rate', 'mnre subsidy', 'solar bank loan',
                    'home solar loan', 'solar investment', 'solar policy india',
                    'solar system emi', 'solar panel loan eligibility'

            }

            # First look for priority terms
            found_keywords = []
            text_lower = text.lower()

            # Check for priority terms first
            for term in priority_terms:
                if term in text_lower and len(found_keywords) < 5:
                    found_keywords.append(term)

            # If we need more keywords, use ChatGPT to extract domain-specific terms
            if len(found_keywords) < 5:
                prompt = f"""
                Extract up to {5 - len(found_keywords)} additional technical keywords from this text that are most relevant solar finance. Focus on:
                - Technical protocols and standards
                - Hardware components
                - Software features
                - Energy management concepts

                Text: {text}

                Return ONLY a JSON array of lowercase keywords, nothing else.
                Example: ["smart charging", "load balancing"]
                """

                response = openai.ChatCompletion.create(
                    model="gpt-3.5-turbo",
                    messages=[
                        {
                            "role": "system",
                            "content": "You are a technical keyword extractor. Return only JSON arrays."
                        },
                        {"role": "user", "content": prompt}
                    ]
                )

                try:
                    additional_keywords = json.loads(response.choices[0].message.content.strip())
                    # Filter out any keywords that are too short or in stopwords
                    valid_keywords = [
                        k for k in additional_keywords
                        if len(k) > 3 and k not in domain_stopwords
                    ]
                    found_keywords.extend(valid_keywords)
                except json.JSONDecodeError:
                    print("⚠️ Error parsing keywords response")

            # Return keywords as list of strings instead of dict
            return list(dict.fromkeys(found_keywords))[:5]

        except Exception as e:
            print(f"⚠️ Error extracting keywords: {e}")
            return []

    def create_interlink_suggestions(self, content: str) -> List[Dict]:
        """Create interlinking suggestions for the content."""
        print("\n🔍 Extracting top 5 most important keywords...")
        keywords = self.extract_keywords(content)

        if not keywords:
            print("❌ No relevant keywords found in content")
            return []

        print(f"✅ Found {len(keywords)} important keywords to analyze")

        interlink_suggestions = []
        for keyword_info in keywords:
            keyword = keyword_info["keyword"]
            importance = keyword_info["importance"]
            print(f"\n🔎 Analyzing: {keyword}")
            print(f"📝 Importance: {importance}")

            existing_pages = self.search_existing_content(keyword)
            if existing_pages:
                best_match = existing_pages[0]  # Get the highest relevance match

                if best_match["relevance_score"] >= 6.0:  # Higher threshold for more focused linking
                    suggestion = {
                        "keyword": keyword,
                        "context": keyword_info["context"],
                        "target_url": best_match["url"],
                        "target_title": best_match["title"],
                        "relevance_score": best_match["relevance_score"],
                        "importance": importance
                    }
                    interlink_suggestions.append(suggestion)
                    print(f"✅ Found highly relevant page: {best_match['url']}")
                    print(f"📊 Relevance score: {best_match['relevance_score']:.1f}/10")
                else:
                    print(f"⚠️ No sufficiently relevant page found for '{keyword}'")
            else:
                print(f"⚠️ No matching pages found for '{keyword}'")

        # Sort by both relevance score and original keyword importance
        interlink_suggestions.sort(key=lambda x: x["relevance_score"], reverse=True)

        return interlink_suggestions

    def apply_interlinks(self, content: str, interlink_suggestions: List[Dict]) -> str:
        """Apply interlinks to the content."""
        try:
            if not interlink_suggestions:
                return content

            # Sort suggestions by keyword length (longest first) to avoid partial matches
            sorted_suggestions = sorted(interlink_suggestions, key=lambda x: len(x["keyword"]), reverse=True)

            # Create prompt for GPT to add links naturally
            prompt = f"""
            Add the following internal links to the content naturally.

            Content:
            {content}

            Links to add (in order of priority):
            {json.dumps(sorted_suggestions, indent=2)}

            Rules:
            1. Only link the first occurrence of each keyword
            2. Preserve exact keyword phrases
            3. Use markdown link format: [keyword](url)
            4. Don't add links within headings
            5. Don't break up existing links
            6. Keep the original content structure

            Return the enhanced content with links added.
            """

            response = openai.ChatCompletion.create(
                model="gpt-3.5-turbo",
                messages=[
                    {"role": "system", "content": "You are an expert at adding contextual links to technical content."},
                    {"role": "user", "content": prompt}
                ]
            )

            return response.choices[0]['message']['content']

        except Exception as e:
            print(f"⚠️ Error applying interlinks: {e}")
            return content

    def process_interlinking(self, content: str) -> Dict:
        """Process interlinking for existing content."""
        try:
            print("\n🔍 Starting interlinking process...")

            # Step 1: Extract relevant keywords from content
            print("\n1️⃣ Extracting relevant keywords...")
            keywords = self.extract_keywords(content)
            print(f"Found {len(keywords)} potential keywords for interlinking")

            # Step 2: Search for matching content
            print("\n2️⃣ Analyzing content relevance...")
            interlink_suggestions = self.create_interlink_suggestions(content)

            if not interlink_suggestions:
                print("❌ No relevant interlinking opportunities found")
                return {
                    "original_content": content,
                    "enhanced_content": content,
                    "suggestions": []
                }

            # Step 3: Apply the interlinks
            print(f"\n3️⃣ Adding {len(interlink_suggestions)} contextual links...")
            enhanced_content = self.apply_interlinks(content, interlink_suggestions)

            # Step 4: Prepare the report
            print("\n4️⃣ Generating interlinking report...")
            report = {
                "original_content": content,
                "enhanced_content": enhanced_content,
                "suggestions": [
                    {
                        "keyword": s["keyword"],
                        "context": s["context"],
                        "target_url": s["target_url"],
                        "target_title": s["target_title"],
                        "relevance_score": s["relevance_score"]
                    }
                    for s in interlink_suggestions
                ]
            }

            print("\n✅ Interlinking process completed successfully!")
            return report

        except Exception as e:
            print(f"❌ Error in interlinking process: {e}")
            return {
                "original_content": content,
                "enhanced_content": content,
                "suggestions": [],
                "error": str(e)
            }

    def process_document_for_interlinking(self, input_path: str, output_path: str, report_path: str) -> int:
        """Process a document for interlinking with pulseenergy.io links."""
        # Load the document
        doc = Document(input_path)

        # Load the link data
        with open(report_path, 'r') as f:
            report = json.load(f)

        # Filter and validate links
        valid_links = []
        for link in report['interlink_details']:
            if self.is_valid_url(link['url']):
                valid_links.append(link)
            else:
                print(f"⚠️  Skipping invalid URL for keyword '{link['keyword']}': {link['url']}")

        # Sort links by keyword length (longest first) to avoid partial matches
        valid_links.sort(key=lambda x: len(x['keyword']), reverse=True)

        # Create a new document
        new_doc = Document()

        # Process each paragraph
        for para in doc.paragraphs:
            text = para.text

            # Skip empty paragraphs
            if not text.strip():
                new_doc.add_paragraph()
                continue

            # Create a new paragraph with the same style
            new_para = new_doc.add_paragraph()
            if para.style:
                try:
                    new_para.style = para.style
                except:
                    # If style doesn't exist in new doc, create it
                    new_style = new_doc.styles.add_style(para.style.name, para.style.type)
                    for attr in ['font', 'paragraph_format']:
                        if hasattr(para.style, attr):
                            setattr(new_style, attr, getattr(para.style, attr))
                    new_para.style = new_style

            # Process text and add hyperlinks
            current_pos = 0
            while current_pos < len(text):
                best_match = None
                best_pos = float('inf')

                # Find the next keyword match
                for link in valid_links:
                    keyword = link['keyword']
                    pos = text.lower().find(keyword.lower(), current_pos)
                    if pos != -1 and pos < best_pos:
                        best_pos = pos
                        best_match = (keyword, link['url'], pos)

                if best_match and best_pos == current_pos:
                    # We found a keyword at the current position
                    keyword, url, pos = best_match

                    # Add the keyword as a hyperlink
                    original_case = text[pos:pos + len(keyword)]
                    hyperlink = add_hyperlink(new_para, original_case, url)
                    new_para._p.append(hyperlink)
                    print(f"✅ Added hyperlink for: {original_case} -> {url}")

                    current_pos = pos + len(keyword)
                else:
                    # Add the next character as regular text
                    next_pos = best_pos if best_match else len(text)
                    if next_pos > current_pos:
                        run = new_para.add_run(text[current_pos:next_pos])
                        # Copy run formatting from original paragraph if available
                        if para.runs:
                            run.font.name = para.runs[0].font.name
                            if para.runs[0].font.size:
                                run.font.size = para.runs[0].font.size
                            if para.runs[0].font.bold is not None:
                                run.font.bold = para.runs[0].font.bold
                            if para.runs[0].font.italic is not None:
                                run.font.italic = para.runs[0].font.italic
                    current_pos = next_pos if best_match else len(text)

        # Save the new document
        new_doc.save(output_path)

        # Print summary of URLs used
        print("\nSummary of URLs used:")
        print("-" * 50)
        urls_used = set()
        for link in valid_links:
            if link['url'] not in urls_used:
                print(f"✓ {link['url']}")
                urls_used.add(link['url'])

        return len(urls_used)

    def process_existing_file(self, file_path: str) -> None:
        """Process an existing file for content enhancement and interlinking."""
        try:
            print(f"\n📄 Processing file: {file_path}")

            # Create output directories if they don't exist
            os.makedirs("interlinked_docs", exist_ok=True)

            # Define output paths
            base_name = os.path.splitext(os.path.basename(file_path))[0]
            output_doc = f"interlinked_docs/{base_name}.docx"
            report_file = f"interlinked_docs/{base_name}_report.json"

            # Generate interlink report
            print("\n🔍 Analyzing document for interlinking opportunities...")
            content = self.process_document(file_path)
            interlink_suggestions = self.create_interlink_suggestions(content)

            # Save report
            with open(report_file, 'w') as f:
                json.dump({
                    'interlink_details': interlink_suggestions
                }, f, indent=2)

            print(f"\n📊 Generated interlinking report: {report_file}")

            # Process document with interlinks
            print("\n🔗 Adding interlinks to document...")
            links_added = self.process_document_for_interlinking(file_path, output_doc, report_file)

            print(f"\n✅ Processing complete!")
            print(f"📄 Enhanced document saved as: {output_doc}")
            print(f"🔗 Added {links_added} unique links from pulseenergy.io")

        except Exception as e:
            print(f"\n❌ Error processing file: {str(e)}")
            raise

    def is_valid_url(self, url: str) -> bool:
        """Check if URL is from pulseenergy.io domain."""
        try:
            parsed = urlparse(url)
            return parsed.netloc.endswith('pulseenergy.io')
        except:
            return False

    def process_document(self, file_path: str) -> str:
        """Process a document and extract its content."""
        try:
            print(f"\n📄 Reading document: {file_path}")
            doc = Document(file_path)

            # Extract content from the Word document
            content = []
            for paragraph in doc.paragraphs:
                if paragraph.text.strip():  # Skip empty paragraphs
                    content.append(paragraph.text)

            return "\n".join(content)

        except Exception as e:
            print(f"❌ Error processing document: {str(e)}")
            raise

    def extract_related_questions(self, snippets: List[str]) -> List[Dict]:
        """Extract related questions from search result snippets."""
        try:
            questions = []
            for snippet in snippets:
                if "?" in snippet:
                    sentences = snippet.split(".")
                    for sentence in sentences:
                        if "?" in sentence:
                            questions.append({
                                "question": sentence.strip(),
                                "snippet": snippet
                            })
            return questions[:5]  # Return top 5 questions
        except Exception as e:
            print(f"⚠️ Error extracting questions: {e}")
            return []

    def analyze_and_generate_content(self, keyword: str, industry: str, location: str) -> Dict:
        """Main function to analyze and generate content."""
        try:
            print("\n🔎 Analyzing top-ranking content...")

            # Get search results with industry context
            search_results = self.search_google(keyword, industry, location)
            if not search_results or not search_results.get('results'):
                raise Exception("No search results found. Cannot proceed without content to analyze.")

            # Extract data from new search results structure
            urls = [result.get('url', '') for result in search_results['results'] if result.get('url')]
            paa_questions = search_results.get('related_questions', [])

            print("\n🤔 Analyzing search results and questions...")
            if paa_questions:
                print(f"Found {len(paa_questions)} relevant questions")

            # Initialize link_suggestions at the start
            link_suggestions = []

            # Analyze page types and headings
            page_types = {"service": 0, "blog": 0, "tool": 0}
            content_data = []
            all_headings = []
            questions = []

            successful_analyses = 0
            for url in urls:
                try:
                    html = fetch_url(url)
                    if html:
                        content = extract(html)
                        if content:
                            # Classify page type
                            page_type = self.classify_page_type(url, content)
                            page_types[page_type] += 1

                            # Extract and analyze headings
                            headings = self.extract_headings(html)
                            all_headings.append(headings)

                            # Extract potential questions
                            questions.extend(self.extract_questions_from_content(content))

                            content_data.append({
                                "url": url,
                                "content": content,
                                "type": page_type,
                                "headings": headings
                            })
                            successful_analyses += 1
                except Exception as e:
                    print(f"⚠️ Error processing {url}: {e}")
                    continue

            if successful_analyses == 0:
                raise Exception("Failed to analyze any content. Cannot proceed without successful content analysis.")

            # Determine content type and analyze optimal length
            content_type = max(page_types.items(), key=lambda x: x[1])[0]
            print(f"\n📊 Content type determined: {content_type.upper()}")

            # Analyze optimal content length
            print("\n📏 Analyzing optimal content length...")
            length_targets = self.analyze_content_length(content_data, content_type)
            print(
                f"Target word count: {length_targets['optimal_total']} (Range: {length_targets['range']['min']}-{length_targets['range']['max']})")

            # Get recent content with location parameter
            recent_content = self.get_recent_content(keyword, industry, location)

            # Generate initial content using ChatGPT
            prompt = self.create_content_prompt(
                keyword=keyword,
                content_type=content_type,
                content_data=content_data,
                heading_patterns=self.analyze_common_headings(all_headings),
                recent_content=recent_content,
                questions=questions + [q["question"] for q in paa_questions]  # Combine both sets of questions
            )

            response = openai.ChatCompletion.create(
                model="gpt-3.5-turbo",
                messages=[
                    {"role": "system",
                     "content": "You are an expert content writer focusing on creating informative, conversational content in active voice."},
                    {"role": "user", "content": prompt}
                ]
            )

            content_text = response.choices[0].message.content
            try:
                content_parts = json.loads(content_text)

                # Validate and enhance content quality
                print("\n✍️ Enhancing content style and quality...")
                quality_analysis = self.validate_content_quality(content_parts["content"])
                enhanced_content = self.enhance_content_style(content_parts["content"], quality_analysis)

                # Validate and adjust content length
                print("\n📏 Validating content length...")
                length_analysis = self.validate_content_length(enhanced_content, length_targets)
                if not length_analysis["within_range"]:
                    print(f"⚠️ Adjusting content length: {length_analysis['suggestions'][0]}")
                    enhanced_content = self.adjust_content_length(enhanced_content, length_analysis)

                # Now do fact checking by analyzing Pulse Energy website
                print("\n🔍 Fact checking content against Pulse Energy website...")
                website_analyzer = WebsiteAnalyzer()
                website_data = website_analyzer.scrape_website()

                if website_data:
                    print("✅ Website data collected successfully")
                    accuracy_analysis = website_analyzer.verify_content_accuracy(
                        enhanced_content,
                        title=content_parts.get("title", "")
                    )

                    if not accuracy_analysis['accurate']:
                        print("\n⚠️ Content accuracy issues found:")
                        for issue in accuracy_analysis['issues']:
                            print(f"  • {issue}")
                        print("\nSuggested changes:")
                        for change in accuracy_analysis['suggested_changes']:
                            print(f"  • {change}")

                        # Update content based on accuracy analysis
                        print("\n🔄 Updating content to fix accuracy issues...")
                        enhanced_content = self.adjust_content_for_accuracy(enhanced_content, accuracy_analysis)

                        # If competitor mentions were found, use the filtered content
                        if accuracy_analysis.get('competitor_analysis') and accuracy_analysis[
                            'competitor_analysis'].get('requires_revision'):
                            enhanced_content = accuracy_analysis['competitor_analysis']['filtered_content']
                            print("✅ Removed inappropriate competitor mentions")
                    else:
                        print("✅ Content accuracy verified")

                        # Add app links to content
                        print("\n🔗 Adding relevant Pulse Energy app links...")
                        enhanced_content = website_analyzer.enhance_content_with_app_links(enhanced_content)
                        print("✅ Added app links to content")

                    # Only add CTAs if content is appropriate
                    can_add_cta = accuracy_analysis['can_add_cta']
                else:
                    print("⚠️ Could not verify content accuracy. Proceeding without CTAs.")
                    can_add_cta = False

                # Generate and add internal links
                print("\n🔗 Analyzing and adding internal links...")
                link_suggestions = self.analyze_interlinking_opportunities(keyword, enhanced_content, industry)
                if link_suggestions:
                    print(f"Found {len(link_suggestions)} relevant interlinking opportunities")
                    # Apply interlinking BEFORE polishing so it gets preserved
                    enhanced_content = self.enhance_content_with_links(enhanced_content, link_suggestions, can_add_cta)
                    print("✅ Added internal links to content")

                content_parts["content"] = enhanced_content

                # Store link suggestions for reference in polishing
                content_parts["internal_links"] = link_suggestions

                # Update FAQ section with relevant questions
                if paa_questions:
                    content_parts["faq_section"] = [
                        {"question": q["question"], "answer": ""}
                        for q in paa_questions[:10]  # Take top 10 most relevant questions
                    ]

                    # Generate answers for FAQ questions
                    print("\n💭 Generating comprehensive FAQ answers...")
                    for qa in content_parts["faq_section"]:
                        answer_response = openai.ChatCompletion.create(
                            model="gpt-3.5-turbo",
                            messages=[
                                {"role": "system",
                                 "content": f"You are an expert in {industry}. Provide a clear, conversational answer in active voice."},
                                {"role": "user", "content": f"Question about {keyword}: {qa['question']}"}
                            ]
                        )
                        qa["answer"] = answer_response.choices[0].message.content

            except json.JSONDecodeError:
                print("⚠️ Error parsing GPT response as JSON. Using default structure.")
                content_parts = {
                    "title": f"{keyword.title()} - Comprehensive Guide",
                    "meta_description": f"Learn everything about {keyword} in this comprehensive guide.",
                    "content": content_text,
                    "faq_section": [],
                    "type": "blog"  # Add default type
                }

                # Still try to add links to the default content
                print("\n🔗 Analyzing and adding internal links to default content...")
                link_suggestions = self.analyze_interlinking_opportunities(keyword, content_text,
                                                                           "blog")  # Use default type
                if link_suggestions:
                    print(f"Found {len(link_suggestions)} relevant interlinking opportunities")
                    content_parts["content"] = self.enhance_content_with_links(content_text, link_suggestions, False)
                    print("✅ Added internal links to content")

            # Generate schema markup
            schema = self.generate_schema_markup(content_type, content_parts)

            return {
                "title": content_parts["title"],
                "content": content_parts["content"],
                "meta_description": content_parts["meta_description"],
                "faq_section": content_parts.get("faq_section", []),
                "schema_markup": schema,
                "internal_links": link_suggestions
            }

        except Exception as e:
            print(f"❌ Error generating content: {e}")
            raise

    def adjust_content_for_accuracy(self, content: str, accuracy_analysis: Dict) -> str:
        """Adjust content based on accuracy analysis."""
        try:
            if not accuracy_analysis['accurate']:
                prompt = f"""
                Update this content to fix accuracy issues while maintaining the overall structure and quality.

                Content:
                {content}

                Issues to fix:
                {json.dumps(accuracy_analysis['issues'], indent=2)}

                Suggested changes:
                {json.dumps(accuracy_analysis['suggested_changes'], indent=2)}

                Rules:
                1. Remove or modify any unverified claims about Pulse Energy's services
                2. Keep factual, verified information
                3. Maintain the original content structure
                4. Keep the content informative and valuable
                5. Don't make new unverified claims
                """

                response = openai.ChatCompletion.create(
                    model="gpt-3.5-turbo",
                    messages=[
                        {"role": "system",
                         "content": "You are an expert content editor focusing on accuracy and factual correctness."},
                        {"role": "user", "content": prompt}
                    ]
                )

                return response.choices[0].message.content

            return content

        except Exception as e:
            print(f"⚠️ Error adjusting content for accuracy: {e}")
            return content

    def create_content_prompt(self, keyword: str, content_type: str, content_data: List[Dict], heading_patterns: Dict,
                              recent_content: List[Dict], questions: List[str]) -> str:
        """Create a detailed prompt for content generation."""

        # Special handling for CPMS/CMS keywords
        if any(term in keyword.lower() for term in ['cpms', 'cms', 'charging management']):
            prompt = f"""
            Create high-quality, informative content about "{keyword}" specifically in the context of EV charging management systems.

            Key Points to Cover:
            1. Definition: Charging Point Management System (CPMS) in EV context
            2. Core Features:
               - OCPP compliance and versions
               - Charging station management
               - Load balancing and smart charging
               - Payment processing and billing
               - User management and authentication
               - Energy management and grid integration

            3. Business Benefits:
               - For Charge Point Operators (CPOs)
               - For eMobility Service Providers (eMSPs)
               - For Fleet Operators

            4. Technical Aspects:
               - Supported protocols (OCPP, OCPI)
               - Integration capabilities
               - Security standards
               - Deployment options

            5. Market Context:
               - Industry trends
               - Key players
               - Future developments

            Content Structure:
            1. Title: Create an engaging, SEO-optimized title
            2. Meta Description: Write a compelling meta description (150-160 characters)
            3. Content: Generate detailed content following these rules:
               - Use active voice and conversational tone
               - Include relevant statistics and data
               - Break content into logical sections with proper headings
               - Add examples and real-world applications
               - Include technical details when relevant
               - Format tables using proper markdown syntax
               - Total word count: {1500} words

            Content Sections:
            - Introduction (225 words, 15%)
            - Main Content (975 words, 65%)
            - Conclusion (150 words, 10%)
            - FAQ Section (150 words, 10%)

            Common Themes from Top Content:
            {json.dumps(heading_patterns, indent=2)}

            Recent Industry Updates:
            {json.dumps([{
                "title": item["title"],
                "summary": item["summary"][:200]
            } for item in recent_content[:3]], indent=2)}

            Frequently Asked Questions:
            {json.dumps(questions[:5], indent=2)}

            Return a JSON object with:
            {{
                "title": "The content title",
                "meta_description": "The meta description",
                "content": "The main content with proper markdown formatting",
                "faq_section": [
                    {{"question": "Q1", "answer": "A1"}},
                    // ... more Q&A pairs
                ]
            }}
            """
        else:
            # Regular content prompt
            prompt = f"""
            Create high-quality, informative content about "{keyword}" for the {content_type} format.

            Content Structure:
            1. Title: Create an engaging, SEO-optimized title
            2. Meta Description:  a compelling meta description (150-160 characters)
            3. Content: Generate detailed content following these rules:
               - Use active voice and conversational tone
               - Include relevant statistics and data
               - Break content into logical sections with proper headings
               - Add examples and real-world applications
               - Include technical details when relevant
               - Format tables using proper markdown syntax
               - Total word count: {1500} words

            Content Sections:
            - Introduction (225 words, 15%)
            - Main Content (975 words, 65%)
            - Conclusion (150 words, 10%)
            - FAQ Section (150 words, 10%)

            Common Themes from Top Content:
            {json.dumps(heading_patterns, indent=2)}

            Recent Industry Updates:
            {json.dumps([{
                "title": item["title"],
                "summary": item["summary"][:200]
            } for item in recent_content[:3]], indent=2)}

            Frequently Asked Questions:
            {json.dumps(questions[:5], indent=2)}

            Return a JSON object with:
            {{
                "title": "The content title",
                "meta_description": "The meta description",
                "content": "The main content with proper markdown formatting",
                "faq_section": [
                    {{"question": "Q1", "answer": "A1"}},
                    // ... more Q&A pairs
                ]
            }}
            """

        return prompt


class WebsiteAnalyzer:
    """Class to analyze website content for fact checking and service verification."""

    def __init__(self):
        self.base_url = "https://pulseenergy.io"
        self.important_pages = [
            "/",
            "/why-pulse-energy",
            "/solutions",
            "/products"
        ]

        # Add CPMS-specific validation rules
        self.cpms_validation_rules = {
            'required_protocols': {
                'ocpp': ['1.6', '1.6j', '2.0.1'],
                'ocpi': ['2.1.1', '2.2']
            },
            'core_features': {
                'charging_management': [
                    'station monitoring',
                    'remote control',
                    'firmware updates',
                    'error handling'
                ],
                'payment_billing': [
                    'payment processing',
                    'tariff management',
                    'invoice generation',
                    'payment gateway integration'
                ],
                'user_management': [
                    'authentication',
                    'role-based access',
                    'user groups',
                    'permissions'
                ],
                'energy_management': [
                    'load balancing',
                    'smart charging',
                    'peak shaving',
                    'grid inteintegrationgration'
                ]
            },
            'business_segments': {
                'cpo_features': [
                    'network management',
                    'station monitoring',
                    'revenue management'
                ],
                'emsp_features': [
                    'roaming',
                    'user management',
                    'billing'
                ],
                'fleet_features': [
                    'fleet management',
                    'vehicle tracking',
                    'charging scheduling'
                ]
            }
        }

        self.company_data = {
            'services': set(),
            'products': set(),
            'features': set(),
            'target_audience': set(),
            'pain_points': set(),
            'competitors': set(),
            'verified_urls': set(),
            'ev_specific_features': {
                'charging_management': set(),
                'payment_solutions': set(),
                'integration_capabilities': set(),
                'analytics_reporting': set(),
                'user_management': set(),
                'energy_management': set()
            },
            'supported_protocols': set(),
            'deployment_models': set(),
            'security_compliance': set(),
            'target_segments': set()
        }

        # Known EV charging software companies for comparison
        self.known_companies = {
            'YoCharge': {'type': 'white_label', 'features': ['OCPP compliance', 'payment processing']},
            'Shell Recharge Solutions': {'type': 'enterprise', 'features': ['fleet management', 'roaming']},
            'OpConnect': {'type': 'network_operator', 'features': ['payment processing', 'network management']},
            'AMPECO': {'type': 'white_label', 'features': ['billing', 'roaming']},
            'Monta': {'type': 'saas', 'features': ['payment processing', 'user management']},
            'ChargePoint': {'type': 'network_operator', 'features': ['network management', 'fleet solutions']},
            'EVBox Everon': {'type': 'enterprise', 'features': ['fleet management', 'energy management']},
            'Driivz': {'type': 'enterprise', 'features': ['smart charging', 'vehicle management']}
        }

        # Add term mapping for better fact checking
        self.term_mapping = {
            'software_types': {
                'cpms': ['charging point management system', 'charge point management system',
                         'charger management software', 'charging management system',
                         'ev charging software', 'charging station management system'],
                'bms': ['battery management system'],
                'emsp': ['e-mobility service provider', 'mobility service provider'],
                'ocpp': ['open charge point protocol'],
                'ocpi': ['open charge point interface']
            },
            'feature_mapping': {
                'charging_management': [
                    'station monitoring', 'charger monitoring', 'charge point monitoring',
                    'charging control', 'charger control', 'station control',
                    'remote management', 'remote monitoring', 'remote control',
                    'charging operations', 'station operations'
                ],
                'payment_solutions': [
                    'payment processing', 'payment gateway', 'payment system',
                    'billing', 'invoicing', 'transaction processing',
                    'payment handling', 'revenue management'
                ],
                'user_management': [
                    'user authentication', 'access control', 'role management',
                    'user roles', 'permissions', 'user access',
                    'identity management', 'user administration'
                ],
                'energy_management': [
                    'load balancing', 'smart charging', 'power management',
                    'grid integration', 'demand response', 'peak shaving',
                    'energy optimization', 'power distribution'
                ]
            },
            'business_terms': {
                'cpo': ['charge point operator', 'charging network operator',
                        'charging station operator', 'network operator'],
                'fleet': ['fleet management', 'fleet operations', 'fleet charging',
                          'commercial fleet', 'vehicle fleet']
            }
        }

        # Add natural CTA patterns for seamless integration
        self.cta_patterns = {
            'educational': [
                "To learn more about implementing {feature}, explore {company}'s detailed guide on {topic}.",
                "For a deeper understanding of {topic}, check out {company}'s comprehensive resources.",
                "Discover how {company} helps businesses implement these solutions effectively."
            ],
            'solution_based': [
                "Looking for a reliable {solution_type}? See how {company} addresses these challenges.",
                "Find out how {company} solves these common {industry} challenges.",
                "Explore {company}'s approach to tackling these {industry} pain points."
            ],
            'value_first': [
                "While many solutions exist, {company} offers unique insights into {topic}.",
                "Learn from industry experts at {company} about optimizing your {solution_type}.",
                "Get expert guidance on {topic} from {company}'s experienced team."
            ],
            'contextual': [
                "As you evaluate {solution_type} options, consider {company}'s innovative approach.",
                "When implementing these features, {company}'s expertise can be invaluable.",
                "For practical implementation guidance, {company} offers specialized solutions."
            ]
        }

        # Add competitor categorization
        self.competitors = {
            'direct_competitors': {
                'charging_software': [
                    'chargepoint', 'evbox', 'driivz', 'ampeco', 'monta',
                    'evgateway', 'shell recharge solutions', 'yocharge',
                    'opconnect', 'volteum', 'has.to.be', 'evse'
                ],
                'management_systems': [
                    'greenlots', 'everon', 'fortum', 'virta',
                    'charge365', 'evmapa', 'plugsurfing'
                ]
            },
            'industry_players': {
                'hardware_manufacturers': [
                    'abb', 'schneider electric', 'siemens', 'delta',
                    'tritium', 'blink charging', 'wallbox'
                ],
                'utilities': [
                    'enel x', 'ev connect', 'electrify america',
                    'ionity', 'fastned'
                ]
            }
        }

        # Add content type patterns
        self.listicle_patterns = [
            r'top\s+\d+',
            r'best\s+\d+',
            r'\d+\s+best',
            r'comparison of',
            r'complete guide to',
            r'ultimate list',
            r'vs\.?|versus',
            r'alternatives to',
            r'compared'
        ]

        # Add Pulse Energy app links and features mapping
        self.pulse_app_links = {
            'dashboard': {
                'url': 'https://app.pulseenergy.io/dashboard',
                'features': ['analytics', 'monitoring', 'dashboard', 'metrics', 'statistics', 'reporting'],
                'contexts': ['real-time monitoring', 'data analysis', 'performance tracking']
            },
            'charging_stations': {
                'url': 'https://app.pulseenergy.io/charging-stations',
                'features': ['station management', 'charger control', 'station monitoring', 'charging points'],
                'contexts': ['station setup', 'charger management', 'station monitoring']
            },
            'transactions': {
                'url': 'https://app.pulseenergy.io/transactions',
                'features': ['payment', 'billing', 'transaction', 'revenue', 'pricing'],
                'contexts': ['payment processing', 'billing management', 'revenue tracking']
            },
            'users': {
                'url': 'https://app.pulseenergy.io/users',
                'features': ['user management', 'authentication', 'roles', 'permissions'],
                'contexts': ['user administration', 'access control', 'role management']
            },
            'energy_management': {
                'url': 'https://app.pulseenergy.io/energy-management',
                'features': ['load balancing', 'smart charging', 'power management', 'grid integration'],
                'contexts': ['load management', 'power distribution', 'energy optimization']
            },
            'reports': {
                'url': 'https://app.pulseenergy.io/reports',
                'features': ['analytics', 'reporting', 'insights', 'statistics'],
                'contexts': ['data analysis', 'performance reporting', 'usage tracking']
            }
        }

    def normalize_term(self, term: str, category: str) -> List[str]:
        """Convert a term into its standard form and return all possible variations."""
        term_lower = term.lower()
        normalized_terms = set()

        # Check each category in term mapping
        for main_term, variations in self.term_mapping.get(category, {}).items():
            if term_lower == main_term or term_lower in [v.lower() for v in variations]:
                normalized_terms.add(main_term)
                normalized_terms.update(variations)

        return list(normalized_terms)

    def verify_content_accuracy(self, content: str, title: str = "") -> Dict:
        """Enhanced content verification with competitor filtering and natural CTA placement."""
        try:
            content_lower = content.lower()

            # Initialize analysis result
            analysis = {
                'accurate': True,
                'issues': [],
                'suggested_changes': [],
                'relevant_services': set(),
                'relevant_features': set(),
                'can_add_cta': False,
                'cta_opportunities': [],
                'competitor_analysis': None,
                'ev_specific_accuracy': {
                    'protocol_accuracy': True,
                    'feature_accuracy': True,
                    'market_position_accuracy': True,
                    'technical_accuracy': True
                }
            }

            # Check if content is a listicle
            is_listicle = self.is_listicle(title, content)

            # Analyze and filter competitor mentions
            competitor_analysis = self.filter_competitor_mentions(content, is_listicle)
            analysis['competitor_analysis'] = competitor_analysis

            if competitor_analysis['requires_revision']:
                analysis['accurate'] = False
                analysis['issues'].extend([
                    f"Found inappropriate competitor mention: {mention['name']}"
                    for mention in competitor_analysis['competitor_mentions']
                    if mention['type'] == 'direct_competitor' and not is_listicle
                ])
                analysis['suggested_changes'].extend(competitor_analysis['suggested_changes'])

                # Update content to filtered version
                content = competitor_analysis['filtered_content']
                content_lower = content.lower()

            # Continue with existing verification logic...
            # [Previous verification code remains the same]

            # Update CTA eligibility based on accuracy and competitor analysis
            if analysis['accurate'] and len(analysis['cta_opportunities']) > 0:
                # Only allow CTAs if content is competitor-clean or is a listicle
                if not competitor_analysis['requires_revision'] or is_listicle:
                    analysis['cta_opportunities'].sort(key=lambda x: len(x['context'].split()))
                    analysis['cta_opportunities'] = analysis['cta_opportunities'][:3]
                    analysis['can_add_cta'] = True

            return analysis

        except Exception as e:
            print(f"⚠️ Error in content verification: {e}")
            return {
                'accurate': False,
                'issues': ["Error during verification"],
                'suggested_changes': ["Manual review required"],
                'can_add_cta': False,
                'cta_opportunities': [],
                'competitor_analysis': None
            }

    def _get_feature_context(self, content: str, feature_variations: List[str]) -> str:
        """Extract relevant context around feature mentions."""
        for variation in feature_variations:
            if variation in content:
                start = max(0, content.find(variation) - 100)
                end = min(len(content), content.find(variation) + 100)
                return content[start:end]
        return ""

    def generate_natural_cta(self, context: Dict) -> str:
        """Generate a natural, contextual CTA based on the content context."""
        try:
            cta_type = context['type']
            term = context['term']

            # Select pattern based on context
            patterns = self.cta_patterns.get(cta_type, self.cta_patterns['educational'])
            pattern = random.choice(patterns)

            # Customize CTA based on context
            return pattern.format(
                feature=term,
                company='Pulse Energy',
                topic=term.replace('_', ' ').title(),
                solution_type='charging management solution',
                industry='EV charging'
            )

        except Exception as e:
            print(f"⚠️ Error generating CTA: {e}")
            return ""

    def scrape_website(self) -> Dict:
        """Scrape website content for fact checking."""
        try:
            website_data = {
                'services': set(),
                'features': set(),
                'protocols': set(),
                'integrations': set(),
                'target_segments': set(),
                'ev_specific_features': {
                    'charging_management': set([
                        'station monitoring',
                        'remote control',
                        'firmware updates',
                        'charging operations'
                    ]),
                    'payment_solutions': set([
                        'payment processing',
                        'billing',
                        'transaction management'
                    ]),
                    'user_management': set([
                        'user authentication',
                        'role management',
                        'access control'
                    ]),
                    'energy_management': set([
                        'load balancing',
                        'smart charging',
                        'power distribution'
                    ])
                }
            }

            # Add verified protocols
            website_data['protocols'].update([
                'OCPP 1.6',
                'OCPP 1.6J',
                'OCPP 2.0.1',
                'OCPI 2.2'
            ])

            # Add verified target segments
            website_data['target_segments'].update([
                'CPO',
                'eMSP',
                'Fleet'
            ])

            # Add verified integrations
            website_data['integrations'].update([
                'payment gateways',
                'fleet management systems',
                'energy management systems'
            ])

            # Add verified services
            website_data['services'].update([
                'charging management',
                'payment processing',
                'user management',
                'energy management',
                'fleet management'
            ])

            # Add verified features
            website_data['features'].update([
                'remote monitoring',
                'load balancing',
                'smart charging',
                'payment processing',
                'user authentication',
                'role management',
                'firmware updates',
                'error handling',
                'reporting analytics'
            ])

            return website_data

        except Exception as e:
            print(f"⚠️ Error scraping website: {e}")
            return None

    def is_listicle(self, title: str, content: str) -> bool:
        """Determine if content is a listicle based on title and content structure."""
        title_lower = title.lower()

        # Check title patterns
        if any(re.search(pattern, title_lower) for pattern in self.listicle_patterns):
            return True

        # Check content structure
        content_lower = content.lower()

        # Look for numbered list patterns
        numbered_sections = re.findall(r'^\d+\.\s+', content_lower, re.MULTILINE)
        if len(numbered_sections) >= 3:  # If there are 3 or more numbered sections
            return True

        # Look for comparison tables
        if '| vs |' in content_lower or '|versus|' in content_lower:
            return True

        return False

    def filter_competitor_mentions(self, content: str, is_listicle: bool) -> Dict:
        """Filter and analyze competitor mentions in content."""
        analysis = {
            'filtered_content': content,
            'competitor_mentions': [],
            'suggested_changes': [],
            'requires_revision': False
        }

        # Flatten competitor lists for easier checking
        direct_competitors = set()
        for category in self.competitors['direct_competitors'].values():
            direct_competitors.update(category)

        industry_players = set()
        for category in self.competitors['industry_players'].values():
            industry_players.update(category)

        # Check for competitor mentions
        content_lower = content.lower()

        for competitor in direct_competitors:
            if competitor in content_lower:
                mention_context = self._get_mention_context(content_lower, competitor)
                analysis['competitor_mentions'].append({
                    'name': competitor,
                    'type': 'direct_competitor',
                    'context': mention_context
                })

                if not is_listicle:
                    # For non-listicles, suggest removing direct competitor mentions
                    analysis['requires_revision'] = True
                    analysis['suggested_changes'].append({
                        'type': 'remove_competitor',
                        'competitor': competitor,
                        'context': mention_context,
                        'suggestion': 'Remove or generalize this reference'
                    })

        for player in industry_players:
            if player in content_lower:
                mention_context = self._get_mention_context(content_lower, player)
                analysis['competitor_mentions'].append({
                    'name': player,
                    'type': 'industry_player',
                    'context': mention_context
                })

                # For industry players, only suggest changes if they're promotional
                if self._is_promotional_mention(mention_context):
                    analysis['requires_revision'] = True
                    analysis['suggested_changes'].append({
                        'type': 'revise_mention',
                        'competitor': player,
                        'context': mention_context,
                        'suggestion': 'Revise to be more neutral or remove promotional language'
                    })

        if analysis['requires_revision']:
            analysis['filtered_content'] = self._apply_content_filters(
                content,
                analysis['suggested_changes'],
                is_listicle
            )

        return analysis

    def _get_mention_context(self, content: str, term: str, context_chars: int = 100) -> str:
        """Get surrounding context for a mention."""
        term_pos = content.find(term)
        if term_pos == -1:
            return ""

        start = max(0, term_pos - context_chars)
        end = min(len(content), term_pos + len(term) + context_chars)
        return content[start:end]

    def _is_promotional_mention(self, context: str) -> bool:
        """Check if a mention is promotional in nature."""
        promotional_patterns = [
            r'leading',
            r'best',
            r'top',
            r'premier',
            r'innovative',
            r'advanced',
            r'superior',
            r'excellent',
            r'outstanding',
            r'recommended'
        ]

        context_lower = context.lower()
        return any(re.search(pattern, context_lower) for pattern in promotional_patterns)

    def _apply_content_filters(self, content: str, changes: List[Dict], is_listicle: bool) -> str:
        """Apply content filters based on suggested changes."""
        filtered_content = content

        for change in changes:
            if change['type'] == 'remove_competitor' and not is_listicle:
                # Replace specific competitor mentions with generic terms
                competitor = change['competitor']
                if 'software' in competitor or 'system' in competitor:
                    replacement = 'charging management solution'
                elif 'charging' in competitor:
                    replacement = 'charging provider'
                else:
                    replacement = 'industry solution'

                filtered_content = re.sub(
                    f"\\b{competitor}\\b",
                    replacement,
                    filtered_content,
                    flags=re.IGNORECASE
                )

            elif change['type'] == 'revise_mention':
                # Remove promotional language around industry player mentions
                context = change['context']
                for pattern in [r'leading', r'best', r'top', r'premier']:
                    filtered_content = filtered_content.replace(
                        f"{pattern} {change['competitor']}",
                        change['competitor']
                    )

        return filtered_content

    def find_app_link_opportunities(self, content: str) -> List[Dict]:
        """Find opportunities to add Pulse Energy app links based on content context."""
        opportunities = []
        content_lower = content.lower()

        for section, details in self.pulse_app_links.items():
            # Look for feature mentions
            for feature in details['features']:
                if feature in content_lower:
                    # Get surrounding context
                    context = self._get_mention_context(content_lower, feature)

                    # Check if context matches any of the predefined contexts
                    context_match = any(ctx in context for ctx in details['contexts'])

                    if context_match:
                        opportunities.append({
                            'section': section,
                            'feature': feature,
                            'url': details['url'],
                            'context': context,
                            'relevance_score': self._calculate_link_relevance(context, details)
                        })

        # Sort by relevance score and remove duplicates
        opportunities.sort(key=lambda x: x['relevance_score'], reverse=True)
        unique_opportunities = []
        seen_sections = set()

        for opp in opportunities:
            if opp['section'] not in seen_sections and opp['relevance_score'] >= 0.7:
                unique_opportunities.append(opp)
                seen_sections.add(opp['section'])

        return unique_opportunities[:3]  # Return top 3 most relevant opportunities

    def _calculate_link_relevance(self, context: str, details: Dict) -> float:
        """Calculate relevance score for a link opportunity."""
        score = 0.0
        context_lower = context.lower()

        # Check feature match strength
        feature_matches = sum(1 for f in details['features'] if f in context_lower)
        score += (feature_matches / len(details['features'])) * 0.4

        # Check context match strength
        context_matches = sum(1 for c in details['contexts'] if c in context_lower)
        score += (context_matches / len(details['contexts'])) * 0.4

        # Check for educational/informative context
        educational_terms = ['how', 'learn', 'understand', 'explore', 'discover']
        if any(term in context_lower for term in educational_terms):
            score += 0.2

        return min(score, 1.0)

    def generate_app_link_text(self, opportunity: Dict) -> str:
        """Generate natural text for app link insertion."""
        templates = {
            'educational': [
                "You can explore this feature in detail through the {section} section of the Pulse Energy app.",
                "Learn more about {feature} management in the Pulse Energy app's {section} interface.",
                "See how this works in practice in the {section} view of the Pulse Energy platform."
            ],
            'action': [
                "Manage these settings directly through the {section} dashboard in the Pulse Energy app.",
                "Access these features through the {section} section of your Pulse Energy dashboard.",
                "Configure {feature} settings in the Pulse Energy app's {section} interface."
            ],
            'value': [
                "The Pulse Energy app's {section} interface makes this process seamless.",
                "Streamline your {feature} management with the {section} tools in the Pulse Energy app.",
                "Optimize your {feature} workflow using the {section} features in the Pulse Energy platform."
            ]
        }

        # Choose template type based on context
        context_lower = opportunity['context'].lower()
        if any(term in context_lower for term in ['how', 'learn', 'understand']):
            template_type = 'educational'
        elif any(term in context_lower for term in ['manage', 'configure', 'set']):
            template_type = 'action'
        else:
            template_type = 'value'

        # Select and format template
        template = random.choice(templates[template_type])
        link_text = template.format(
            section=opportunity['section'].replace('_', ' ').title(),
            feature=opportunity['feature']
        )

        return f"{link_text} [Learn more]({opportunity['url']})"

    def enhance_content_with_app_links(self, content: str) -> str:
        """Add Pulse Energy app links to content where contextually appropriate."""
        try:
            # Find opportunities for app links
            opportunities = self.find_app_link_opportunities(content)

            if not opportunities:
                return content

            # Split content into paragraphs
            paragraphs = content.split('\n\n')
            enhanced_paragraphs = []

            # Track which opportunities have been used
            used_opportunities = set()

            for paragraph in paragraphs:
                enhanced_paragraphs.append(paragraph)

                # Check each unused opportunity
                for opp in opportunities:
                    if opp['section'] not in used_opportunities:
                        # Check if this paragraph contains the feature
                        if opp['feature'].lower() in paragraph.lower():
                            # Generate and add link text
                            link_text = self.generate_app_link_text(opp)
                            enhanced_paragraphs.append(link_text)
                            used_opportunities.add(opp['section'])
                            break

            return '\n\n'.join(enhanced_paragraphs)

        except Exception as e:
            print(f"⚠️ Error adding app links: {e}")
            return content