annotated_casl_app.py

import gradio as gr
import json
import os
import logging
import requests
import re
import time

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Anthropic API key
ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "")

if ANTHROPIC_API_KEY:
    logger.info("Claude API key found")
else:
    logger.warning("Claude API key not found - using demo mode")

def call_claude_api(prompt):
    """Call Claude API for annotation or analysis"""
    if not ANTHROPIC_API_KEY:
        return "❌ Claude API key not configured. Please set ANTHROPIC_API_KEY environment variable."
    
    try:
        headers = {
            "Content-Type": "application/json",
            "x-api-key": ANTHROPIC_API_KEY,
            "anthropic-version": "2023-06-01"
        }
        
        data = {
            "model": "claude-3-5-sonnet-20241022",
            "max_tokens": 4096,
            "messages": [
                {
                    "role": "user",
                    "content": prompt
                }
            ]
        }
        
        response = requests.post(
            "https://api.anthropic.com/v1/messages",
            headers=headers,
            json=data,
            timeout=90
        )
        
        if response.status_code == 200:
            response_json = response.json()
            return response_json['content'][0]['text']
        else:
            logger.error(f"Claude API error: {response.status_code} - {response.text}")
            return f"❌ Claude API Error: {response.status_code}"
            
    except Exception as e:
        logger.error(f"Error calling Claude API: {str(e)}")
        return f"❌ Error: {str(e)}"

def check_annotation_completeness(original_transcript, annotated_transcript):
    """Check if annotation is complete by verifying last 3 words are present"""
    import re
    
    # Clean and extract words from original transcript
    original_words = re.findall(r'\b\w+\b', original_transcript.strip())
    if len(original_words) < 3:
        return True, "Transcript too short to validate"
    
    # Get last 3 words from original
    last_three_words = original_words[-3:]
    
    # Clean annotated transcript (remove markers but keep words)
    cleaned_annotated = re.sub(r'\[.*?\]', '', annotated_transcript)
    annotated_words = re.findall(r'\b\w+\b', cleaned_annotated.strip())
    
    # Check if all last 3 words appear in the annotated transcript
    missing_words = []
    for word in last_three_words:
        if word.lower() not in [w.lower() for w in annotated_words]:
            missing_words.append(word)
    
    if missing_words:
        return False, f"Annotation appears incomplete. Missing words from end: {', '.join(missing_words)}"
    
    # Additional check: verify the last few words appear near the end
    if len(annotated_words) > 0:
        last_annotated_words = annotated_words[-10:]  # Check last 10 words
        last_original_in_annotated = sum(1 for word in last_three_words 
                                       if word.lower() in [w.lower() for w in last_annotated_words])
        
        if last_original_in_annotated < 2:  # At least 2 of the last 3 should be near the end
            return False, f"Annotation may be incomplete. Last words '{', '.join(last_three_words)}' not found near end of annotation"
    
    return True, "Annotation appears complete"

def annotate_transcript(transcript_content, age, gender, slp_notes):
    """First step: Annotate transcript with linguistic markers"""
    if not transcript_content or len(transcript_content.strip()) < 50:
        return "Error: Please provide a longer transcript for annotation."
    
    # Add SLP notes to the prompt if provided
    notes_section = ""
    if slp_notes and slp_notes.strip():
        notes_section = f"""
    
    SLP CLINICAL NOTES:
    {slp_notes.strip()}
    """
    
    annotation_prompt = f"""
    You are a speech-language pathologist preparing a transcript for detailed analysis. Your task is to ANNOTATE the ENTIRE transcript with linguistic markers at a WORD-BY-WORD level.

    Patient: {age}-year-old {gender}
    
    ORIGINAL TRANSCRIPT:
    {transcript_content}{notes_section}
    
    CRITICAL REQUIREMENT: You MUST annotate the COMPLETE transcript. Do NOT provide partial annotations or stop mid-sentence. Complete the ENTIRE transcript annotation in one response.

    DETAILED ANNOTATION INSTRUCTIONS:
    Annotate by adding markers in brackets IMMEDIATELY after each relevant word or phrase:

    FLUENCY MARKERS:
    - [FILLER] after: um[FILLER], uh[FILLER], like[FILLER], you know[FILLER], well[FILLER], so[FILLER]
    - [FALSE_START] after incomplete words: "I was go-[FALSE_START] going"
    - [REPETITION] after repeated words: "I I[REPETITION] went"
    - [REVISION] after self-corrections: "I went to the-[REVISION] I mean"
    - [PAUSE] for hesitations: "I was...[PAUSE] thinking"

    WORD RETRIEVAL MARKERS:
    - [CIRCUMLOCUTION] after roundabout descriptions: "that thing you write with[CIRCUMLOCUTION]"
    - [INCOMPLETE] after abandoned thoughts: "I was thinking about the...[INCOMPLETE]"
    - [GENERIC] after vague terms: thing[GENERIC], stuff[GENERIC], whatsit[GENERIC]
    - [WORD_SEARCH] after searching: "the... um...[WORD_SEARCH] car"

    GRAMMATICAL MARKERS:
    - [GRAM_ERROR] after mistakes: "I goed[GRAM_ERROR]", "He don't[GRAM_ERROR]"
    - [SYNTAX_ERROR] after word order problems: "Yesterday I to store went[SYNTAX_ERROR]"
    - [MORPH_ERROR] after morphological errors: "runned[MORPH_ERROR]", "childs[MORPH_ERROR]"
    - [RUN_ON] at end of run-on sentences

    VOCABULARY MARKERS:
    - [SIMPLE_VOCAB] after basic words: go[SIMPLE_VOCAB], big[SIMPLE_VOCAB], good[SIMPLE_VOCAB]
    - [COMPLEX_VOCAB] after sophisticated words: magnificent[COMPLEX_VOCAB], elaborate[COMPLEX_VOCAB]
    - [SEMANTIC_ERROR] after wrong word choices: "drove my bicycle[SEMANTIC_ERROR]"

    PRAGMATIC MARKERS:
    - [TOPIC_SHIFT] after topic changes: "Anyway, about cats[TOPIC_SHIFT]"
    - [TANGENT] after going off-topic: "Speaking of dogs, my vacation[TANGENT]"
    - [INAPPROPRIATE] after inappropriate content
    - [COHERENCE_BREAK] after illogical statements

    SENTENCE COMPLEXITY MARKERS:
    - [SIMPLE_SENT] after simple sentences: "I went home.[SIMPLE_SENT]"
    - [COMPLEX_SENT] after complex sentences: "When I got home, I made dinner.[COMPLEX_SENT]"
    - [COMPOUND_SENT] after compound sentences: "I went home, and made dinner.[COMPOUND_SENT]"
    - [FIGURATIVE] after metaphors/idioms: "raining cats and dogs[FIGURATIVE]"

    ADDITIONAL MARKERS:
    - [PRONOUN_REF] after unclear pronouns: "He told him that he[PRONOUN_REF] was wrong"
    - [MAZING] after confusing constructions
    - [PERSEVERATION] after repetitive patterns

    MANDATORY REQUIREMENTS:
    1. Do NOT stop until the entire transcript is complete
    2. Keep ALL original text intact
    3. Mark overlapping features when applicable
    4. Be consistent throughout
    5. Complete the annotation in ONE response - no partial outputs allowed

    PROVIDE THE COMPLETE ANNOTATED TRANSCRIPT - EVERY WORD MUST BE PROCESSED.
    """
    
    # Get initial annotation
    annotated_result = call_claude_api(annotation_prompt)
    
    # Check if annotation is complete
    is_complete, validation_message = check_annotation_completeness(transcript_content, annotated_result)
    
    if not is_complete:
        logger.warning(f"Annotation incomplete: {validation_message}")
        
        # Try once more with stronger emphasis on completion
        retry_prompt = f"""
        CRITICAL: The previous annotation was INCOMPLETE. You MUST complete the ENTIRE transcript.
        
        {validation_message}
        
        ORIGINAL TRANSCRIPT (COMPLETE THIS ENTIRELY):
        {transcript_content}{notes_section}
        
        MANDATORY REQUIREMENT: Annotate EVERY SINGLE WORD from start to finish. Do not stop until you reach the very last word of the transcript.
        
        {annotation_prompt.split('DETAILED ANNOTATION INSTRUCTIONS:')[1]}
        
        VERIFY: The last words of the original transcript are: {' '.join(transcript_content.strip().split()[-3:])}
        ENSURE these words appear at the END of your annotated transcript.
        """
        
        retry_result = call_claude_api(retry_prompt)
        
        # Check retry
        retry_complete, retry_message = check_annotation_completeness(transcript_content, retry_result)
        
        if retry_complete:
            logger.info("Retry successful - annotation now complete")
            return retry_result
        else:
            logger.error(f"Retry failed: {retry_message}")
            return f"⚠️ ANNOTATION INCOMPLETE: {retry_message}\n\nPartial annotation:\n{retry_result}"
    
    logger.info("Annotation completed successfully")
    return annotated_result

def analyze_annotated_transcript(annotated_transcript, age, gender, slp_notes):
    """Second step: Analyze the annotated transcript with comprehensive quantification"""
    if not annotated_transcript or len(annotated_transcript.strip()) < 50:
        return "Error: Please provide an annotated transcript for analysis."
    
    # Add SLP notes to the prompt if provided
    notes_section = ""
    if slp_notes and slp_notes.strip():
        notes_section = f"""
    
    SLP CLINICAL NOTES:
    {slp_notes.strip()}
    """
    
    analysis_prompt = f"""
    You are a speech-language pathologist conducting a COMPREHENSIVE analysis of a word-by-word annotated speech sample. Count EVERY marker precisely and provide detailed quantitative analysis.

    Patient: {age}-year-old {gender}
    
    ANNOTATED TRANSCRIPT:
    {annotated_transcript}{notes_section}
    
    ORIGINAL TRANSCRIPT (for reference and backup analysis):
    {annotated_transcript.replace('[FILLER]', '').replace('[FALSE_START]', '').replace('[REPETITION]', '').replace('[REVISION]', '').replace('[PAUSE]', '').replace('[CIRCUMLOCUTION]', '').replace('[INCOMPLETE]', '').replace('[GENERIC]', '').replace('[WORD_SEARCH]', '').replace('[GRAM_ERROR]', '').replace('[SYNTAX_ERROR]', '').replace('[MORPH_ERROR]', '').replace('[RUN_ON]', '').replace('[SIMPLE_VOCAB]', '').replace('[COMPLEX_VOCAB]', '').replace('[SEMANTIC_ERROR]', '').replace('[TOPIC_SHIFT]', '').replace('[TANGENT]', '').replace('[INAPPROPRIATE]', '').replace('[COHERENCE_BREAK]', '').replace('[SIMPLE_SENT]', '').replace('[COMPLEX_SENT]', '').replace('[COMPOUND_SENT]', '').replace('[FIGURATIVE]', '').replace('[PRONOUN_REF]', '').replace('[MAZING]', '').replace('[PERSEVERATION]', '')}
    
    ANALYSIS INSTRUCTIONS:
    Using the detailed linguistic markers in the annotated transcript, provide a comprehensive analysis with EXACT counts, percentages, and specific examples. If markers are missing or unclear, use the original transcript for backup analysis. Complete ALL sections below:

    COMPREHENSIVE SPEECH SAMPLE ANALYSIS:

    1. FLUENCY ANALYSIS (count each marker type):
    - Count [FILLER] markers: List each instance and calculate rate per 100 words
    - Count [FALSE_START] markers: List examples and analyze patterns
    - Count [REPETITION] markers: Categorize by type (word, phrase, sound)
    - Count [REVISION] markers: Analyze self-correction patterns
    - Count [PAUSE] markers: Assess hesitation frequency
    - Calculate total disfluency rate and severity level
    - Determine impact on communication effectiveness

    2. WORD RETRIEVAL ANALYSIS (precise counting):
    - Count [CIRCUMLOCUTION] markers: List each roundabout description
    - Count [INCOMPLETE] markers: Analyze abandoned thought patterns
    - Count [GENERIC] markers: Calculate specificity ratio
    - Count [WORD_SEARCH] markers: Identify retrieval difficulty areas
    - Count [WORD_FINDING] markers: Assess overall retrieval efficiency
    - Calculate word-finding accuracy percentage

    3. GRAMMATICAL ANALYSIS (detailed error counting):
    - Count [GRAM_ERROR] markers by subcategory:
      * Verb tense errors
      * Subject-verb agreement errors  
      * Pronoun errors
      * Article errors
    - Count [SYNTAX_ERROR] markers: Analyze word order problems
    - Count [MORPH_ERROR] markers: Categorize morphological mistakes
    - Count [RUN_ON] markers: Assess sentence boundary awareness
    - Calculate grammatical accuracy rate (correct vs. total attempts)

    4. VOCABULARY ANALYSIS (sophistication assessment):
    - Count [SIMPLE_VOCAB] markers: List basic vocabulary used
    - Count [COMPLEX_VOCAB] markers: List sophisticated vocabulary
    - Count [SEMANTIC_ERROR] markers: Analyze word choice accuracy
    - Calculate vocabulary sophistication ratio (complex/simple)
    - Assess semantic appropriateness and precision
    - Determine vocabulary diversity (type-token ratio)

    5. PRAGMATIC LANGUAGE ANALYSIS (coherence assessment):
    - Count [TOPIC_SHIFT] markers: Assess transition appropriateness
    - Count [TANGENT] markers: Analyze tangential speech patterns
    - Count [INAPPROPRIATE] markers: Evaluate contextual appropriateness
    - Count [COHERENCE_BREAK] markers: Assess logical flow
    - Count [PRONOUN_REF] markers: Analyze referential clarity
    - Evaluate overall discourse coherence and organization

    6. SENTENCE COMPLEXITY ANALYSIS (structural assessment):
    - Count [SIMPLE_SENT] markers: Calculate simple sentence percentage
    - Count [COMPLEX_SENT] markers: Analyze subordination use
    - Count [COMPOUND_SENT] markers: Assess coordination patterns
    - Count [FIGURATIVE] markers: Evaluate figurative language use
    - Count [MAZING] markers: Assess confusing constructions
    - Calculate syntactic complexity index

    7. QUANTITATIVE METRICS (comprehensive calculations):
    - Total word count and morpheme count
    - Mean Length of Utterance (MLU) in words and morphemes
    - Type-Token Ratio (TTR) for vocabulary diversity
    - Clauses per utterance ratio
    - Error rate per linguistic domain
    - Communication efficiency index

    8. ERROR PATTERN ANALYSIS:
    - Most frequent error types with exact counts
    - Error consistency vs. variability patterns
    - Developmental appropriateness of errors
    - Severity ranking of different error types
    - Compensatory strategies observed

    9. CLINICAL IMPLICATIONS:
    - Primary strengths: List with supporting evidence
    - Primary weaknesses: Rank by severity with counts
    - Intervention priorities: Based on error frequency and impact
    - Therapy targets: Specific, measurable goals
    - Prognosis indicators: Based on error patterns and consistency

    10. SUMMARY AND RECOMMENDATIONS:
    - Overall communication profile with percentile estimates
    - Priority treatment goals ranked by importance
    - Functional communication impact assessment
    - Recommended therapy approaches and frequency
    - Follow-up assessment timeline

    CRITICAL: Provide EXACT counts for every marker type, calculate precise percentages, and give specific examples from the transcript. Show your counting work clearly. Complete ALL 12 sections - use <CONTINUE> if needed.
    """
    
    return call_claude_api_with_continuation(analysis_prompt)

def calculate_linguistic_metrics(transcript_text):
    """Calculate comprehensive linguistic metrics from transcript"""
    import re
    import numpy as np
    
    if not transcript_text or not transcript_text.strip():
        return {}
    
    # Clean text and extract words
    cleaned_text = re.sub(r'\[.*?\]', '', transcript_text)  # Remove annotation markers
    sentences = re.split(r'[.!?]+', cleaned_text)
    sentences = [s.strip() for s in sentences if s.strip()]
    
    # Extract all words
    all_words = []
    for sentence in sentences:
        words = re.findall(r'\b\w+\b', sentence.lower())
        all_words.extend(words)
    
    if not all_words:
        return {}
    
    # Basic counts
    total_words = len(all_words)
    total_sentences = len(sentences)
    unique_words = len(set(all_words))
    
    # Type-Token Ratio
    ttr = unique_words / total_words if total_words > 0 else 0
    
    # Mean Length of Utterance (MLU)
    mlu_words = total_words / total_sentences if total_sentences > 0 else 0
    
    # Word frequency analysis
    word_freq = {}
    for word in all_words:
        word_freq[word] = word_freq.get(word, 0) + 1
    
    # Sort by frequency
    sorted_word_freq = dict(sorted(word_freq.items(), key=lambda x: x[1], reverse=True))
    
    # Sentence length statistics
    sentence_lengths = []
    for sentence in sentences:
        words_in_sentence = len(re.findall(r'\b\w+\b', sentence))
        sentence_lengths.append(words_in_sentence)
    
    avg_sentence_length = np.mean(sentence_lengths) if sentence_lengths else 0
    std_sentence_length = np.std(sentence_lengths) if sentence_lengths else 0
    
    # Vocabulary sophistication (words > 6 characters as proxy for complex vocabulary)
    complex_words = [word for word in all_words if len(word) > 6]
    vocabulary_sophistication = len(complex_words) / total_words if total_words > 0 else 0
    
    # Calculate morpheme count (approximate)
    morpheme_count = 0
    for word in all_words:
        # Basic morpheme counting (word + common suffixes)
        morpheme_count += 1
        if word.endswith(('s', 'ed', 'ing', 'er', 'est', 'ly')):
            morpheme_count += 1
        if word.endswith(('tion', 'sion', 'ness', 'ment', 'able', 'ible')):
            morpheme_count += 1
    
    mlu_morphemes = morpheme_count / total_sentences if total_sentences > 0 else 0
    
    return {
        'total_words': total_words,
        'total_sentences': total_sentences,
        'unique_words': unique_words,
        'type_token_ratio': round(ttr, 3),
        'mlu_words': round(mlu_words, 2),
        'mlu_morphemes': round(mlu_morphemes, 2),
        'avg_sentence_length': round(avg_sentence_length, 2),
        'sentence_length_std': round(std_sentence_length, 2),
        'vocabulary_sophistication': round(vocabulary_sophistication, 3),
        'word_frequency': dict(list(sorted_word_freq.items())[:20]),  # Top 20 most frequent
        'sentence_lengths': sentence_lengths,
        'complex_word_count': len(complex_words),
        'morpheme_count': morpheme_count,
        'tokenized_words': all_words,  # Add for lexical diversity analysis
        'cleaned_text': cleaned_text   # Add for lexical diversity analysis
    }

def calculate_advanced_lexical_diversity(transcript_text):
    """Calculate advanced lexical diversity measures using lexical-diversity library"""
    import re
    
    try:
        from lexical_diversity import lex_div as ld
        lexdiv_available = True
    except ImportError:
        lexdiv_available = False
        
    if not lexdiv_available:
        return {
            'library_available': False,
            'error': 'lexical-diversity library not installed. Install with: pip install lexical-diversity'
        }
    
    if not transcript_text or not transcript_text.strip():
        return {'library_available': True, 'error': 'No text provided'}
    
    # Clean text and prepare for lexical diversity analysis
    cleaned_text = re.sub(r'\[.*?\]', '', transcript_text)  # Remove annotation markers
    
    try:
        # Tokenize using lexical-diversity
        tokens = ld.tokenize(cleaned_text)
        
        if len(tokens) < 10:  # Need minimum tokens for meaningful analysis
            return {
                'library_available': True,
                'error': f'Insufficient tokens for analysis (need ≥10, got {len(tokens)})'
            }
        
        # Calculate various lexical diversity measures
        diversity_measures = {}
        
        # Basic TTR (included for comparison, but noted as unreliable)
        diversity_measures['simple_ttr'] = round(ld.ttr(tokens), 4)
        
        # Recommended measures
        try:
            diversity_measures['root_ttr'] = round(ld.root_ttr(tokens), 4)
        except:
            diversity_measures['root_ttr'] = None
            
        try:
            diversity_measures['log_ttr'] = round(ld.log_ttr(tokens), 4)
        except:
            diversity_measures['log_ttr'] = None
            
        try:
            diversity_measures['maas_ttr'] = round(ld.maas_ttr(tokens), 4)
        except:
            diversity_measures['maas_ttr'] = None
        
        # MSTTR (Mean Segmental TTR) - only if enough tokens
        if len(tokens) >= 50:
            try:
                diversity_measures['msttr_50'] = round(ld.msttr(tokens, window_length=50), 4)
            except:
                diversity_measures['msttr_50'] = None
        
        if len(tokens) >= 25:
            try:
                diversity_measures['msttr_25'] = round(ld.msttr(tokens, window_length=25), 4)
            except:
                diversity_measures['msttr_25'] = None
        
        # MATTR (Moving Average TTR) - only if enough tokens
        if len(tokens) >= 50:
            try:
                diversity_measures['mattr_50'] = round(ld.mattr(tokens, window_length=50), 4)
            except:
                diversity_measures['mattr_50'] = None
                
        if len(tokens) >= 25:
            try:
                diversity_measures['mattr_25'] = round(ld.mattr(tokens, window_length=25), 4)
            except:
                diversity_measures['mattr_25'] = None
        
        # HDD (Hypergeometric Distribution D)
        try:
            diversity_measures['hdd'] = round(ld.hdd(tokens), 4)
        except:
            diversity_measures['hdd'] = None
        
        # MTLD (Measure of Textual Lexical Diversity) - only if enough tokens
        if len(tokens) >= 50:
            try:
                diversity_measures['mtld'] = round(ld.mtld(tokens), 4)
            except:
                diversity_measures['mtld'] = None
                
            try:
                diversity_measures['mtld_ma_wrap'] = round(ld.mtld_ma_wrap(tokens), 4)
            except:
                diversity_measures['mtld_ma_wrap'] = None
                
            try:
                diversity_measures['mtld_ma_bid'] = round(ld.mtld_ma_bid(tokens), 4)
            except:
                diversity_measures['mtld_ma_bid'] = None
        
        return {
            'library_available': True,
            'token_count': len(tokens),
            'diversity_measures': diversity_measures,
            'tokens': tokens[:50]  # First 50 tokens for verification
        }
        
    except Exception as e:
        return {
            'library_available': True,
            'error': f'Error calculating lexical diversity: {str(e)}'
        }

def analyze_annotation_markers(annotated_transcript):
    """Analyze and count all annotation markers in the transcript with detailed word-level analysis"""
    import re
    
    if not annotated_transcript:
        return {}
    
    # Define all marker types
    marker_types = {
        'FILLER': r'\[FILLER\]',
        'FALSE_START': r'\[FALSE_START\]',
        'REPETITION': r'\[REPETITION\]',
        'REVISION': r'\[REVISION\]',
        'PAUSE': r'\[PAUSE\]',
        'CIRCUMLOCUTION': r'\[CIRCUMLOCUTION\]',
        'INCOMPLETE': r'\[INCOMPLETE\]',
        'GENERIC': r'\[GENERIC\]',
        'WORD_SEARCH': r'\[WORD_SEARCH\]',
        'GRAM_ERROR': r'\[GRAM_ERROR\]',
        'SYNTAX_ERROR': r'\[SYNTAX_ERROR\]',
        'MORPH_ERROR': r'\[MORPH_ERROR\]',
        'RUN_ON': r'\[RUN_ON\]',
        'SIMPLE_VOCAB': r'\[SIMPLE_VOCAB\]',
        'COMPLEX_VOCAB': r'\[COMPLEX_VOCAB\]',
        'SEMANTIC_ERROR': r'\[SEMANTIC_ERROR\]',
        'TOPIC_SHIFT': r'\[TOPIC_SHIFT\]',
        'TANGENT': r'\[TANGENT\]',
        'INAPPROPRIATE': r'\[INAPPROPRIATE\]',
        'COHERENCE_BREAK': r'\[COHERENCE_BREAK\]',
        'SIMPLE_SENT': r'\[SIMPLE_SENT\]',
        'COMPLEX_SENT': r'\[COMPLEX_SENT\]',
        'COMPOUND_SENT': r'\[COMPOUND_SENT\]',
        'FIGURATIVE': r'\[FIGURATIVE\]',
        'PRONOUN_REF': r'\[PRONOUN_REF\]',
        'MAZING': r'\[MAZING\]',
        'PERSEVERATION': r'\[PERSEVERATION\]'
    }
    
    # Count each marker type and extract the actual words
    marker_counts = {}
    marker_examples = {}
    marker_words = {}
    
    for marker_name, pattern in marker_types.items():
        matches = re.findall(pattern, annotated_transcript)
        marker_counts[marker_name] = len(matches)
        
        # Find examples with context and extract the actual words
        examples = []
        words = []
        
        # Find all instances of word[MARKER] pattern
        word_pattern = r'(\w+)' + pattern
        word_matches = re.finditer(word_pattern, annotated_transcript)
        
        for match in word_matches:
            word = match.group(1)
            words.append(word)
            
            # Get context around the match
            start = max(0, match.start() - 30)
            end = min(len(annotated_transcript), match.end() + 30)
            context = annotated_transcript[start:end].strip()
            examples.append(f'"{word}" in context: {context}')
        
        marker_examples[marker_name] = examples[:10]  # Keep first 10 examples
        marker_words[marker_name] = words
    
    # Calculate totals by category
    fluency_total = sum([marker_counts.get(m, 0) for m in ['FILLER', 'FALSE_START', 'REPETITION', 'REVISION', 'PAUSE']])
    grammar_total = sum([marker_counts.get(m, 0) for m in ['GRAM_ERROR', 'SYNTAX_ERROR', 'MORPH_ERROR', 'RUN_ON']])
    vocab_simple = marker_counts.get('SIMPLE_VOCAB', 0)
    vocab_complex = marker_counts.get('COMPLEX_VOCAB', 0)
    
    return {
        'marker_counts': marker_counts,
        'marker_examples': marker_examples,
        'marker_words': marker_words,
        'category_totals': {
            'fluency_issues': fluency_total,
            'grammar_errors': grammar_total,
            'simple_vocabulary': vocab_simple,
            'complex_vocabulary': vocab_complex,
            'vocab_sophistication_ratio': vocab_complex / (vocab_simple + vocab_complex) if (vocab_simple + vocab_complex) > 0 else 0
        }
    }

def generate_comprehensive_analysis_report(annotated_transcript, original_transcript):
    """Generate the most comprehensive analysis combining manual counts, lexical diversity, and linguistic metrics"""
    import re
    
    if not annotated_transcript:
        return "No annotated transcript provided."
    
    # Get all three types of analysis
    linguistic_metrics = calculate_linguistic_metrics(original_transcript)
    marker_analysis = analyze_annotation_markers(annotated_transcript)
    lexical_diversity = calculate_advanced_lexical_diversity(original_transcript)
    
    # Calculate rates per 100 words
    total_words = linguistic_metrics.get('total_words', 0)
    
    report_lines = []
    report_lines.append("=" * 100)
    report_lines.append("COMPREHENSIVE SPEECH ANALYSIS REPORT")
    report_lines.append("Combining Manual Counts + Advanced Lexical Diversity + Linguistic Metrics")
    report_lines.append("=" * 100)
    report_lines.append("")
    
    # SECTION 1: BASIC STATISTICS
    report_lines.append("1. BASIC STATISTICS:")
    report_lines.append(f"   • Total words: {total_words}")
    report_lines.append(f"   • Total sentences: {linguistic_metrics.get('total_sentences', 0)}")
    report_lines.append(f"   • Unique words: {linguistic_metrics.get('unique_words', 0)}")
    report_lines.append(f"   • MLU (words): {linguistic_metrics.get('mlu_words', 0):.2f}")
    report_lines.append(f"   • MLU (morphemes): {linguistic_metrics.get('mlu_morphemes', 0):.2f}")
    report_lines.append(f"   • Average sentence length: {linguistic_metrics.get('avg_sentence_length', 0):.2f}")
    report_lines.append("")
    
    # SECTION 2: ADVANCED LEXICAL DIVERSITY MEASURES
    report_lines.append("2. ADVANCED LEXICAL DIVERSITY MEASURES:")
    if lexical_diversity.get('library_available', False) and 'diversity_measures' in lexical_diversity:
        measures = lexical_diversity['diversity_measures']
        report_lines.append(f"   • Token count for analysis: {lexical_diversity.get('token_count', 0)}")
        report_lines.append("")
        report_lines.append("   RECOMMENDED MEASURES:")
        
        if measures.get('root_ttr') is not None:
            report_lines.append(f"   • Root TTR: {measures['root_ttr']:.4f}")
        if measures.get('log_ttr') is not None:
            report_lines.append(f"   • Log TTR: {measures['log_ttr']:.4f}")
        if measures.get('maas_ttr') is not None:
            report_lines.append(f"   • Maas TTR: {measures['maas_ttr']:.4f}")
        if measures.get('hdd') is not None:
            report_lines.append(f"   • HDD (Hypergeometric Distribution D): {measures['hdd']:.4f}")
        
        report_lines.append("")
        report_lines.append("   MOVING WINDOW MEASURES:")
        if measures.get('msttr_25') is not None:
            report_lines.append(f"   • MSTTR (25-word window): {measures['msttr_25']:.4f}")
        if measures.get('msttr_50') is not None:
            report_lines.append(f"   • MSTTR (50-word window): {measures['msttr_50']:.4f}")
        if measures.get('mattr_25') is not None:
            report_lines.append(f"   • MATTR (25-word window): {measures['mattr_25']:.4f}")
        if measures.get('mattr_50') is not None:
            report_lines.append(f"   • MATTR (50-word window): {measures['mattr_50']:.4f}")
        
        report_lines.append("")
        report_lines.append("   MTLD MEASURES:")
        if measures.get('mtld') is not None:
            report_lines.append(f"   • MTLD: {measures['mtld']:.4f}")
        if measures.get('mtld_ma_wrap') is not None:
            report_lines.append(f"   • MTLD (moving average, wrap): {measures['mtld_ma_wrap']:.4f}")
        if measures.get('mtld_ma_bid') is not None:
            report_lines.append(f"   • MTLD (moving average, bidirectional): {measures['mtld_ma_bid']:.4f}")
        
        report_lines.append("")
        report_lines.append("   COMPARISON MEASURE:")
        report_lines.append(f"   • Simple TTR (not recommended): {measures.get('simple_ttr', 0):.4f}")
        
    else:
        report_lines.append("   ⚠️ Advanced lexical diversity measures not available")
        if 'error' in lexical_diversity:
            report_lines.append(f"   Error: {lexical_diversity['error']}")
    
    report_lines.append("")
    
    # SECTION 3: MANUAL ANNOTATION COUNTS
    report_lines.append("3. MANUAL ANNOTATION COUNTS:")
    marker_counts = marker_analysis['marker_counts']
    marker_words = marker_analysis['marker_words']
    
    # Group markers by category for organized reporting
    categories = {
        'FLUENCY MARKERS': ['FILLER', 'FALSE_START', 'REPETITION', 'REVISION', 'PAUSE'],
        'WORD RETRIEVAL MARKERS': ['CIRCUMLOCUTION', 'INCOMPLETE', 'GENERIC', 'WORD_SEARCH'],
        'GRAMMAR MARKERS': ['GRAM_ERROR', 'SYNTAX_ERROR', 'MORPH_ERROR', 'RUN_ON'],
        'VOCABULARY MARKERS': ['SIMPLE_VOCAB', 'COMPLEX_VOCAB', 'SEMANTIC_ERROR'],
        'PRAGMATIC MARKERS': ['TOPIC_SHIFT', 'TANGENT', 'INAPPROPRIATE', 'COHERENCE_BREAK', 'PRONOUN_REF'],
        'SENTENCE COMPLEXITY MARKERS': ['SIMPLE_SENT', 'COMPLEX_SENT', 'COMPOUND_SENT', 'FIGURATIVE'],
        'OTHER MARKERS': ['MAZING', 'PERSEVERATION']
    }
    
    for category, markers in categories.items():
        category_total = sum(marker_counts.get(marker, 0) for marker in markers)
        if category_total > 0:
            report_lines.append(f"   {category}:")
            
            for marker in markers:
                count = marker_counts.get(marker, 0)
                if count > 0:
                    rate = (count / total_words * 100) if total_words > 0 else 0
                    words_list = marker_words.get(marker, [])
                    
                    report_lines.append(f"     • {marker}: {count} instances ({rate:.2f} per 100 words)")
                    
                    if words_list:
                        # Count frequency of each word
                        word_freq = {}
                        for word in words_list:
                            word_freq[word] = word_freq.get(word, 0) + 1
                        
                        # Sort by frequency
                        sorted_words = sorted(word_freq.items(), key=lambda x: x[1], reverse=True)
                        word_summary = []
                        for word, freq in sorted_words[:8]:  # Top 8 most frequent
                            if freq > 1:
                                word_summary.append(f'"{word}" ({freq}x)')
                            else:
                                word_summary.append(f'"{word}"')
                        
                        report_lines.append(f"       Words: {', '.join(word_summary)}")
            
            report_lines.append(f"     CATEGORY TOTAL: {category_total} instances")
            report_lines.append("")
    
    # SECTION 4: SUMMARY STATISTICS
    report_lines.append("4. SUMMARY STATISTICS:")
    category_totals = marker_analysis['category_totals']
    
    fluency_total = category_totals['fluency_issues']
    grammar_total = category_totals['grammar_errors']
    simple_vocab = category_totals['simple_vocabulary']
    complex_vocab = category_totals['complex_vocabulary']
    
    if total_words > 0:
        report_lines.append(f"   • Total fluency issues: {fluency_total} ({fluency_total/total_words*100:.2f} per 100 words)")
        report_lines.append(f"   • Total grammar errors: {grammar_total} ({grammar_total/total_words*100:.2f} per 100 words)")
        report_lines.append(f"   • Simple vocabulary: {simple_vocab} ({simple_vocab/total_words*100:.2f} per 100 words)")
        report_lines.append(f"   • Complex vocabulary: {complex_vocab} ({complex_vocab/total_words*100:.2f} per 100 words)")
        
        if simple_vocab + complex_vocab > 0:
            vocab_ratio = complex_vocab / (simple_vocab + complex_vocab)
            report_lines.append(f"   • Vocabulary sophistication ratio: {vocab_ratio:.3f}")
    
    # SECTION 5: WORD FREQUENCY ANALYSIS
    word_freq = linguistic_metrics.get('word_frequency', {})
    if word_freq:
        report_lines.append("")
        report_lines.append("5. MOST FREQUENT WORDS:")
        for i, (word, freq) in enumerate(list(word_freq.items())[:15], 1):
            percentage = (freq / total_words * 100) if total_words > 0 else 0
            report_lines.append(f"   {i:2d}. '{word}': {freq} times ({percentage:.1f}%)")
    
    report_lines.append("")
    report_lines.append("=" * 100)
    report_lines.append("END OF COMPREHENSIVE ANALYSIS REPORT")
    report_lines.append("=" * 100)
    
    return '\n'.join(report_lines)

def generate_manual_count_report(annotated_transcript):
    """Generate a basic manual count report (legacy function for compatibility)"""
    return generate_comprehensive_analysis_report(annotated_transcript, annotated_transcript)

def process_file(file):
    """Process uploaded transcript file"""
    if file is None:
        return "Please upload a file first."
    
    try:
        with open(file.name, 'r', encoding='utf-8', errors='ignore') as f:
            content = f.read()
        
        if not content.strip():
            return "File appears to be empty."
            
        return content
    except Exception as e:
        return f"Error reading file: {str(e)}"

def segment_response_by_sections(response_text):
    """Segment response by section titles and return a dictionary of sections"""
    required_sections = [
        "1. SPEECH FACTORS",
        "2. LANGUAGE SKILLS ASSESSMENT", 
        "3. COMPLEX SENTENCE ANALYSIS",
        "4. FIGURATIVE LANGUAGE ANALYSIS",
        "5. PRAGMATIC LANGUAGE ASSESSMENT",
        "6. VOCABULARY AND SEMANTIC ANALYSIS",
        "7. MORPHOLOGICAL AND PHONOLOGICAL ANALYSIS",
        "8. COGNITIVE-LINGUISTIC FACTORS",
        "9. FLUENCY AND RHYTHM ANALYSIS",
        "10. QUANTITATIVE METRICS",
        "11. CLINICAL IMPLICATIONS",
        "12. PROGNOSIS AND SUMMARY"
    ]
    
    sections = {}
    current_section = None
    current_content = []
    
    lines = response_text.split('\n')
    
    for line in lines:
        # Check if this line is a section header
        is_section_header = False
        for section in required_sections:
            if section in line:
                # Save previous section if exists
                if current_section and current_content:
                    sections[current_section] = '\n'.join(current_content).strip()
                
                # Start new section
                current_section = section
                current_content = []
                is_section_header = True
                break
        
        # If not a section header, add to current section content
        if not is_section_header and current_section:
            current_content.append(line)
    
    # Save the last section
    if current_section and current_content:
        sections[current_section] = '\n'.join(current_content).strip()
    
    return sections

def combine_sections_smartly(sections_dict):
    """Combine sections in the correct order without duplicates"""
    required_sections = [
        "1. SPEECH FACTORS",
        "2. LANGUAGE SKILLS ASSESSMENT", 
        "3. COMPLEX SENTENCE ANALYSIS",
        "4. FIGURATIVE LANGUAGE ANALYSIS",
        "5. PRAGMATIC LANGUAGE ASSESSMENT",
        "6. VOCABULARY AND SEMANTIC ANALYSIS",
        "7. MORPHOLOGICAL AND PHONOLOGICAL ANALYSIS",
        "8. COGNITIVE-LINGUISTIC FACTORS",
        "9. FLUENCY AND RHYTHM ANALYSIS",
        "10. QUANTITATIVE METRICS",
        "11. CLINICAL IMPLICATIONS",
        "12. PROGNOSIS AND SUMMARY"
    ]
    
    combined_parts = []
    combined_parts.append("COMPREHENSIVE SPEECH SAMPLE ANALYSIS")
    combined_parts.append("")
    
    for section in required_sections:
        if section in sections_dict:
            combined_parts.append(section)
            combined_parts.append("")
            combined_parts.append(sections_dict[section])
            combined_parts.append("")
    
    return '\n'.join(combined_parts)

def call_claude_api_with_continuation(prompt):
    """Call Claude API with smart continuation system - unlimited continuations until complete"""
    if not ANTHROPIC_API_KEY:
        return "❌ Claude API key not configured. Please set ANTHROPIC_API_KEY environment variable."
    
    # Define all required sections
    required_sections = [
        "1. SPEECH FACTORS",
        "2. LANGUAGE SKILLS ASSESSMENT", 
        "3. COMPLEX SENTENCE ANALYSIS",
        "4. FIGURATIVE LANGUAGE ANALYSIS",
        "5. PRAGMATIC LANGUAGE ASSESSMENT",
        "6. VOCABULARY AND SEMANTIC ANALYSIS",
        "7. MORPHOLOGICAL AND PHONOLOGICAL ANALYSIS",
        "8. COGNITIVE-LINGUISTIC FACTORS",
        "9. FLUENCY AND RHYTHM ANALYSIS",
        "10. QUANTITATIVE METRICS",
        "11. CLINICAL IMPLICATIONS",
        "12. PROGNOSIS AND SUMMARY"
    ]
    
    # Safety limits to prevent infinite loops
    MAX_CONTINUATIONS = 30  # Increased from 20 to 30 API calls
    MAX_TIME_MINUTES = 15   # Increased from 10 to 15 minutes total
    MIN_PROGRESS_PER_CALL = 0  # Changed from 1 to 0 to allow more flexibility
    
    try:
        all_sections = {}  # Store all sections found across all parts
        continuation_count = 0
        start_time = time.time()
        last_section_count = 0  # Track progress between calls
        
        # Add continuation instruction to original prompt
        initial_prompt = prompt + "\n\nCRITICAL INSTRUCTIONS: You MUST complete ALL 12 sections of the analysis. If your response is cut off or incomplete, end with <CONTINUE> to indicate more content is needed. Do not skip any sections. Use the checklist to ensure all sections are completed."
        
        while True:  # Unlimited continuations until complete
            if continuation_count == 0:
                current_prompt = initial_prompt
            else:
                # For continuations, provide context about what was already covered
                missing_sections = [s for s in required_sections if s not in all_sections]
                missing_text = "\n".join([f"- {section}" for section in missing_sections])
                
                current_prompt = prompt + f"\n\nCONTINUATION {continuation_count + 1}: The following sections are STILL MISSING and MUST be completed:\n\n{missing_text}\n\nCRITICAL: Provide ONLY these missing sections. Do not repeat any sections that are already complete. Focus exclusively on the missing sections listed above. Complete ALL missing sections in this response."
            
            headers = {
                "Content-Type": "application/json",
                "x-api-key": ANTHROPIC_API_KEY,
                "anthropic-version": "2023-06-01"
            }
            
            data = {
                "model": "claude-3-5-sonnet-20241022",
                "max_tokens": 4096,
                "messages": [
                    {
                        "role": "user",
                        "content": current_prompt
                    }
                ]
            }
            
            response = requests.post(
                "https://api.anthropic.com/v1/messages",
                headers=headers,
                json=data,
                timeout=90
            )
            
            if response.status_code == 200:
                response_json = response.json()
                response_text = response_json['content'][0]['text']
                
                # Log response for debugging
                print(f"\n=== PART {continuation_count + 1} RESPONSE ===")
                print(f"Length: {len(response_text)} characters")
                print(f"Contains CONTINUE: {'<CONTINUE>' in response_text}")
                print(f"First 200 chars: {response_text[:200]}...")
                print(f"Last 200 chars: {response_text[-200:]}...")
                print("=" * 50)
                
                # Segment this part and add new sections to our collection
                part_sections = segment_response_by_sections(response_text)
                for section, content in part_sections.items():
                    if section not in all_sections:  # Only add if not already present
                        all_sections[section] = content
                        print(f"Added section: {section}")
                    else:
                        print(f"Skipped duplicate section: {section}")
                
                # Check completion status
                completed_sections = len(all_sections)
                missing_sections = [s for s in required_sections if s not in all_sections]
                
                print(f"Completed sections: {completed_sections}/12")
                print(f"Missing sections: {missing_sections}")
                
                # Check if response indicates continuation is needed
                needs_continuation = "<CONTINUE>" in response_text
                
                print(f"Needs continuation: {needs_continuation}")
                print(f"Continuation count: {continuation_count}")
                
                # Safety checks to prevent infinite loops
                current_time = time.time()
                elapsed_minutes = (current_time - start_time) / 60
                current_section_count = len(all_sections)
                progress_made = current_section_count - last_section_count
                
                # Check if we're making progress
                if continuation_count > 0 and progress_made < MIN_PROGRESS_PER_CALL:
                    # Only stop if we've made multiple calls with no progress
                    if continuation_count > 3:  # Allow more attempts before giving up
                        logger.warning(f"No progress made in last call (added {progress_made} sections). Stopping to prevent infinite loop.")
                        break
                    else:
                        logger.info(f"No progress in call {continuation_count}, but continuing to allow more attempts...")
                
                # Check time limit
                if elapsed_minutes > MAX_TIME_MINUTES:
                    logger.warning(f"Time limit exceeded ({elapsed_minutes:.1f} minutes). Stopping to prevent excessive API usage.")
                    break
                
                # Check continuation limit
                if continuation_count >= MAX_CONTINUATIONS:
                    logger.warning(f"Continuation limit reached ({MAX_CONTINUATIONS} calls). Stopping to prevent excessive API usage.")
                    break
                
                # Continue if <CONTINUE> is present and safety checks pass
                if needs_continuation:
                    continuation_count += 1
                    last_section_count = current_section_count
                    logger.info(f"Continuing analysis (attempt {continuation_count}/{MAX_CONTINUATIONS}, {elapsed_minutes:.1f} minutes elapsed)")
                    continue
                else:
                    break
            else:
                logger.error(f"Claude API error: {response.status_code} - {response.text}")
                return f"❌ Claude API Error: {response.status_code}"
                
    except Exception as e:
        logger.error(f"Error calling Claude API: {str(e)}")
        return f"❌ Error: {str(e)}"
    
    # Combine all sections in the correct order
    final_response = combine_sections_smartly(all_sections)
    
    # Log final results
    print(f"\n=== FINAL SMART VALIDATION ===")
    print(f"Total sections found: {len(all_sections)}")
    print(f"All sections present: {len(all_sections) == 12}")
    print(f"Missing sections: {[s for s in required_sections if s not in all_sections]}")
    print(f"Total time: {(time.time() - start_time) / 60:.1f} minutes")
    print(f"Total API calls: {continuation_count + 1}")
    print("=" * 50)
    
    # Add completion indicator with safety info
    if continuation_count > 0:
        final_response += f"\n\n[Analysis completed in {continuation_count + 1} parts over {(time.time() - start_time) / 60:.1f} minutes]"
    
    # Add warning if incomplete due to safety limits
    if len(all_sections) < 12:
        missing_sections = [s for s in required_sections if s not in all_sections]
        final_response += f"\n\n⚠️ WARNING: Analysis incomplete due to safety limits. Missing sections: {', '.join(missing_sections)}"
        final_response += f"\n\n💡 TIP: Try running the analysis again, or use the 'Targeted Analysis' tab to focus on specific areas."
        final_response += f"\nThe 'Quick Questions' tab may also provide faster results for specific areas of interest."
    
    return final_response

def analyze_with_backup(annotated_transcript, original_transcript, age, gender, slp_notes):
    """Analyze annotated transcript with original as backup"""
    if not annotated_transcript or len(annotated_transcript.strip()) < 50:
        return "Error: Please provide an annotated transcript for analysis."
    
    # Add SLP notes to the prompt if provided
    notes_section = ""
    if slp_notes and slp_notes.strip():
        notes_section = f"""
    
    SLP CLINICAL NOTES:
    {slp_notes.strip()}
    """
    
    # Calculate quantitative metrics
    linguistic_metrics = calculate_linguistic_metrics(original_transcript)
    marker_analysis = analyze_annotation_markers(annotated_transcript)
    
    # Format metrics for inclusion in prompt
    metrics_text = f"""
    
    CALCULATED LINGUISTIC METRICS:
    - Total Words: {linguistic_metrics.get('total_words', 0)}
    - Total Sentences: {linguistic_metrics.get('total_sentences', 0)}
    - Unique Words: {linguistic_metrics.get('unique_words', 0)}
    - Type-Token Ratio: {linguistic_metrics.get('type_token_ratio', 0)}
    - MLU (Words): {linguistic_metrics.get('mlu_words', 0)}
    - MLU (Morphemes): {linguistic_metrics.get('mlu_morphemes', 0)}
    - Average Sentence Length: {linguistic_metrics.get('avg_sentence_length', 0)}
    - Vocabulary Sophistication: {linguistic_metrics.get('vocabulary_sophistication', 0)}
    
    ANNOTATION MARKER COUNTS:
    - Fluency Issues: {marker_analysis.get('category_totals', {}).get('fluency_issues', 0)}
    - Grammar Errors: {marker_analysis.get('category_totals', {}).get('grammar_errors', 0)}
    - Simple Vocabulary: {marker_analysis.get('category_totals', {}).get('simple_vocabulary', 0)}
    - Complex Vocabulary: {marker_analysis.get('category_totals', {}).get('complex_vocabulary', 0)}
    - Vocabulary Sophistication Ratio: {marker_analysis.get('category_totals', {}).get('vocab_sophistication_ratio', 0):.3f}
    """
    
    analysis_prompt = f"""
    You are a speech-language pathologist conducting a COMPREHENSIVE analysis of a word-by-word annotated speech sample. Use the provided quantitative metrics and count EVERY marker precisely.

    Patient: {age}-year-old {gender}
    
    ANNOTATED TRANSCRIPT:
    {annotated_transcript}{notes_section}
    
    ORIGINAL TRANSCRIPT (for reference and backup analysis):
    {original_transcript}
    
    {metrics_text}
    
    ANALYSIS INSTRUCTIONS:
    Using the detailed linguistic markers in the annotated transcript and the calculated metrics above, provide a comprehensive analysis with EXACT counts, percentages, and specific examples. Complete ALL 12 sections below:

    COMPREHENSIVE SPEECH SAMPLE ANALYSIS:

    1. SPEECH FACTORS (with EXACT counts and specific citations):

    A. Fluency Issues:
    - Count [FILLER] markers: List each instance and calculate rate per 100 words
    - Count [FALSE_START] markers: List examples and analyze patterns
    - Count [REPETITION] markers: Categorize by type (word, phrase, sound)
    - Count [REVISION] markers: Analyze self-correction patterns
    - Count [PAUSE] markers: Assess hesitation frequency
    - Calculate total disfluency rate and severity level

    B. Word Retrieval Issues:
    - Count [CIRCUMLOCUTION] markers: List each roundabout description
    - Count [INCOMPLETE] markers: Analyze abandoned thought patterns
    - Count [GENERIC] markers: Calculate specificity ratio
    - Count [WORD_SEARCH] markers: Identify retrieval difficulty areas

    C. Grammatical Errors:
    - Count [GRAM_ERROR] markers by subcategory (verb tense, subject-verb agreement, etc.)
    - Count [SYNTAX_ERROR] markers: Analyze word order problems
    - Count [MORPH_ERROR] markers: Categorize morphological mistakes
    - Count [RUN_ON] markers: Assess sentence boundary awareness

    2. LANGUAGE SKILLS ASSESSMENT (with specific evidence):

    A. Lexical/Semantic Skills:
    - Use calculated Type-Token Ratio: {linguistic_metrics.get('type_token_ratio', 0)}
    - Count [SIMPLE_VOCAB] vs [COMPLEX_VOCAB] markers
    - Assess vocabulary sophistication ratio: {marker_analysis.get('category_totals', {}).get('vocab_sophistication_ratio', 0):.3f}
    - Count [SEMANTIC_ERROR] markers and analyze patterns

    B. Syntactic Skills:
    - Count [SIMPLE_SENT], [COMPLEX_SENT], [COMPOUND_SENT] markers
    - Calculate sentence complexity ratios
    - Assess clause complexity and embedding

    C. Supralinguistic Skills:
    - Identify cause-effect relationships, inferences, non-literal language
    - Assess problem-solving language and metalinguistic awareness

    3. COMPLEX SENTENCE ANALYSIS (with exact counts):

    A. Coordinating Conjunctions:
    - Count and cite EVERY use of: and, but, or, so, yet, for, nor
    - Analyze patterns and age-appropriateness

    B. Subordinating Conjunctions:
    - Count and cite EVERY use of: because, although, while, since, if, when, where, that, which, who
    - Analyze clause complexity and embedding depth

    C. Sentence Structure Analysis:
    - Use calculated MLU: {linguistic_metrics.get('mlu_words', 0)} words, {linguistic_metrics.get('mlu_morphemes', 0)} morphemes
    - Calculate complexity ratios and assess developmental appropriateness

    4. FIGURATIVE LANGUAGE ANALYSIS (with exact counts):

    A. Similes and Metaphors:
    - Count [FIGURATIVE] markers for similes (using "like" or "as")
    - Count [FIGURATIVE] markers for metaphors (direct comparisons)

    B. Idioms and Non-literal Language:
    - Count and analyze idiomatic expressions
    - Assess comprehension and appropriate use

    5. PRAGMATIC LANGUAGE ASSESSMENT (with specific examples):

    A. Discourse Management:
    - Count [TOPIC_SHIFT] markers: Assess transition appropriateness
    - Count [TANGENT] markers: Analyze tangential speech patterns
    - Count [COHERENCE_BREAK] markers: Assess logical flow

    B. Referential Communication:
    - Count [PRONOUN_REF] markers: Analyze referential clarity
    - Assess communicative effectiveness

    6. VOCABULARY AND SEMANTIC ANALYSIS (with quantification):

    A. Vocabulary Diversity:
    - Total words: {linguistic_metrics.get('total_words', 0)}
    - Unique words: {linguistic_metrics.get('unique_words', 0)}
    - Type-Token Ratio: {linguistic_metrics.get('type_token_ratio', 0)}
    - Vocabulary sophistication: {linguistic_metrics.get('vocabulary_sophistication', 0)}

    B. Semantic Relationships:
    - Analyze word frequency patterns
    - Assess semantic precision and relationships

    7. MORPHOLOGICAL AND PHONOLOGICAL ANALYSIS (with counts):

    A. Morphological Markers:
    - Count [MORPH_ERROR] markers and categorize
    - Analyze morpheme use patterns
    - Assess morphological complexity

    B. Phonological Patterns:
    - Identify speech sound patterns from transcript
    - Assess syllable structure complexity

    8. COGNITIVE-LINGUISTIC FACTORS (with evidence):

    A. Working Memory:
    - Assess sentence length complexity using average: {linguistic_metrics.get('avg_sentence_length', 0)} words
    - Analyze information retention patterns

    B. Processing Efficiency:
    - Analyze linguistic complexity and word-finding patterns
    - Assess cognitive demands of language structures

    C. Executive Function:
    - Count self-correction patterns ([REVISION] markers)
    - Assess planning and organization in discourse

    9. FLUENCY AND RHYTHM ANALYSIS (with quantification):

    A. Disfluency Patterns:
    - Total fluency issues: {marker_analysis.get('category_totals', {}).get('fluency_issues', 0)}
    - Calculate disfluency rate per 100 words
    - Analyze impact on communication

    B. Language Flow:
    - Assess sentence length variability: std = {linguistic_metrics.get('sentence_length_std', 0)}
    - Analyze linguistic markers of hesitation

    10. QUANTITATIVE METRICS:
    - Total words: {linguistic_metrics.get('total_words', 0)}
    - Total sentences: {linguistic_metrics.get('total_sentences', 0)}
    - MLU (words): {linguistic_metrics.get('mlu_words', 0)}
    - MLU (morphemes): {linguistic_metrics.get('mlu_morphemes', 0)}
    - Type-Token Ratio: {linguistic_metrics.get('type_token_ratio', 0)}
    - Grammar error rate: Calculate from marker counts
    - Vocabulary sophistication ratio: {marker_analysis.get('category_totals', {}).get('vocab_sophistication_ratio', 0):.3f}

    11. CLINICAL IMPLICATIONS:
    - Primary strengths: List with supporting evidence from markers and metrics
    - Primary weaknesses: Rank by severity with exact counts
    - Intervention priorities: Based on error frequency and impact
    - Therapy targets: Specific, measurable goals

    12. PROGNOSIS AND SUMMARY:
    - Overall communication profile with percentile estimates
    - Developmental appropriateness assessment
    - Summary of key findings from quantitative analysis
    - Priority treatment goals and expected outcomes

    CRITICAL REQUIREMENTS:
    - Use the provided calculated metrics in your analysis
    - Provide EXACT counts for every marker type
    - Calculate precise percentages and show your work
    - Give specific examples from the transcript
    - If annotation is incomplete, supplement with analysis of the original transcript
    - Complete ALL 12 sections - use <CONTINUE> if needed
    """
    
    return call_claude_api_with_continuation(analysis_prompt)

def full_analysis_pipeline(transcript_content, age, gender, slp_notes, progress_callback=None):
    """Complete pipeline: annotate then analyze with progressive updates"""
    if not transcript_content or len(transcript_content.strip()) < 50:
        return "Error: Please provide a longer transcript for analysis.", ""
    
    # Step 1: Annotate transcript
    logger.info("Step 1: Annotating transcript with linguistic markers...")
    if progress_callback:
        progress_callback("🏷️ Step 1: Annotating transcript with linguistic markers...")
    
    annotated_transcript = annotate_transcript(transcript_content, age, gender, slp_notes)
    
    if annotated_transcript.startswith("❌"):
        return annotated_transcript, ""
    
    # Return annotated transcript immediately
    if progress_callback:
        progress_callback("✅ Step 1 Complete: Annotation finished! Starting analysis...")
    
    # Check if annotation was incomplete
    if annotated_transcript.startswith("⚠️ ANNOTATION INCOMPLETE"):
        logger.warning("Annotation incomplete, proceeding with analysis using original transcript as primary source")
        analysis_note = "⚠️ Note: Annotation was incomplete. Analysis primarily based on original transcript.\n\n"
    else:
        analysis_note = ""
    
    # Step 2: Analyze annotated transcript with original as backup
    logger.info("Step 2: Analyzing annotated transcript...")
    if progress_callback:
        progress_callback("📊 Step 2: Analyzing annotated transcript (this may take several minutes)...")
    
    analysis_result = analyze_with_backup(annotated_transcript, transcript_content, age, gender, slp_notes)
    
    if progress_callback:
        progress_callback("✅ Analysis Complete!")
    
    return annotated_transcript, analysis_note + analysis_result

def progressive_analysis_pipeline(transcript_content, age, gender, slp_notes):
    """Generator function for progressive analysis updates"""
    if not transcript_content or len(transcript_content.strip()) < 50:
        yield "Error: Please provide a longer transcript for analysis.", "", "❌ Error"
        return
    
    # Step 1: Annotate transcript
    logger.info("Step 1: Annotating transcript with linguistic markers...")
    yield "", "", "🏷️ Step 1: Annotating transcript with linguistic markers..."
    
    annotated_transcript = annotate_transcript(transcript_content, age, gender, slp_notes)
    
    if annotated_transcript.startswith("❌"):
        yield annotated_transcript, "", "❌ Annotation failed"
        return
    
    # Return annotated transcript immediately after completion
    yield annotated_transcript, "", "✅ Step 1 Complete! Starting analysis..."
    
    # Check if annotation was incomplete
    if annotated_transcript.startswith("⚠️ ANNOTATION INCOMPLETE"):
        logger.warning("Annotation incomplete, proceeding with analysis")
        analysis_note = "⚠️ Note: Annotation was incomplete. Analysis primarily based on original transcript.\n\n"
        yield annotated_transcript, "", "⚠️ Annotation incomplete, continuing with analysis..."
    else:
        analysis_note = ""
    
    # Step 2: Analyze annotated transcript
    logger.info("Step 2: Analyzing annotated transcript...")
    yield annotated_transcript, "", "📊 Step 2: Analyzing annotated transcript (this may take several minutes)..."
    
    analysis_result = analyze_with_backup(annotated_transcript, transcript_content, age, gender, slp_notes)
    
    # Final result
    yield annotated_transcript, analysis_note + analysis_result, "✅ Analysis Complete!"

# Example transcript data
example_transcript = """Well, um, I was thinking about, you know, the thing that happened yesterday. I was go- I mean I was going to the store and, uh, I seen this really big dog. Actually, it was more like a wolf or something. The dog, he was just standing there, and I thought to myself, "That's one magnificent creature." But then, um, I realized I forgot my wallet at home, so I had to turn around and go back. When I got home, my wife she says to me, "Where's the groceries?" And I'm like, "Well, honey, I had to come back because I forgot my thing." She wasn't too happy about that, let me tell you. Anyway, speaking of dogs, did I ever tell you about the time I went fishing? It was raining cats and dogs that day, and I caught three fishes. My brother, he don't like fishing much, but he came with me anyway. We was sitting there for hours, just waiting and waiting. The fish, they wasn't biting at all. But then, all of a sudden, I got a bite! I was so excited, I almost falled into the water. The fish was huge - well, maybe not huge, but pretty big for that lake. We cooked it up real good that night. My wife, she made some of that fancy stuff to go with it. What do you call it... that green thing... oh yeah, asparagus. She's always making these elaborate meals. Sometimes I think she tries too hard, you know? But I appreciate it. Life's been good to us, I guess. We been married for twenty-five years now. Time flies when you're having fun, as they say."""

example_annotated = """Well[FILLER], um[FILLER], I was thinking about, you[SIMPLE_VOCAB] know[FILLER], the thing[GENERIC] that happened yesterday[SIMPLE_VOCAB]. I was go-[FALSE_START] I mean I was going[SIMPLE_VOCAB] to the store[SIMPLE_VOCAB] and, uh[FILLER], I seen[GRAM_ERROR] this really big[SIMPLE_VOCAB] dog[SIMPLE_VOCAB].[SIMPLE_SENT] Actually, it was more like[FILLER] a wolf[SIMPLE_VOCAB] or something[GENERIC].[SIMPLE_SENT] The dog[SIMPLE_VOCAB], he[PRONOUN_REF] was just standing[SIMPLE_VOCAB] there, and I thought to myself, "That's one magnificent[COMPLEX_VOCAB] creature[COMPLEX_VOCAB]."[COMPLEX_SENT] But then, um[FILLER], I realized[COMPLEX_VOCAB] I forgot[SIMPLE_VOCAB] my wallet[SIMPLE_VOCAB] at home[SIMPLE_VOCAB], so I had to turn around and go[SIMPLE_VOCAB] back[SIMPLE_VOCAB].[COMPLEX_SENT] When I got home, my wife[SIMPLE_VOCAB] she[REPETITION] says[SIMPLE_VOCAB] to me, "Where's the groceries[SIMPLE_VOCAB]?"[COMPLEX_SENT] And I'm like[FILLER], "Well[FILLER], honey[SIMPLE_VOCAB], I had to come back because I forgot[SIMPLE_VOCAB] my thing[GENERIC]."[COMPLEX_SENT] She wasn't too happy[SIMPLE_VOCAB] about that, let me tell you.[SIMPLE_SENT] Anyway[TOPIC_SHIFT], speaking of dogs, did I ever tell you about the time I went fishing?[TANGENT][COMPLEX_SENT] It was raining cats and dogs[FIGURATIVE] that day, and I caught[SIMPLE_VOCAB] three fishes[MORPH_ERROR].[COMPOUND_SENT] My brother[SIMPLE_VOCAB], he[PRONOUN_REF] don't[GRAM_ERROR] like fishing[SIMPLE_VOCAB] much, but he came with me anyway[SIMPLE_VOCAB].[COMPLEX_SENT] We was[GRAM_ERROR] sitting[SIMPLE_VOCAB] there for hours[SIMPLE_VOCAB], just waiting[SIMPLE_VOCAB] and waiting[REPETITION].[SIMPLE_SENT] The fish[SIMPLE_VOCAB], they[PRONOUN_REF] wasn't[GRAM_ERROR] biting[SIMPLE_VOCAB] at all.[SIMPLE_SENT] But then, all of a sudden[SIMPLE_VOCAB], I got[SIMPLE_VOCAB] a bite[SIMPLE_VOCAB]![SIMPLE_SENT] I was so excited[SIMPLE_VOCAB], I almost falled[MORPH_ERROR] into the water[SIMPLE_VOCAB].[COMPLEX_SENT] The fish[SIMPLE_VOCAB] was huge[SIMPLE_VOCAB] - well[FILLER], maybe not huge[SIMPLE_VOCAB], but pretty big[SIMPLE_VOCAB] for that lake[SIMPLE_VOCAB].[REVISION][COMPLEX_SENT] We cooked[SIMPLE_VOCAB] it up real good[SIMPLE_VOCAB] that night[SIMPLE_VOCAB].[SIMPLE_SENT] My wife[SIMPLE_VOCAB], she[REPETITION] made some of that fancy[SIMPLE_VOCAB] stuff[GENERIC] to go[SIMPLE_VOCAB] with it.[SIMPLE_SENT] What do you call it... [WORD_SEARCH] that green[SIMPLE_VOCAB] thing[GENERIC]... [PAUSE] oh yeah, asparagus[COMPLEX_VOCAB].[CIRCUMLOCUTION] She's always making[SIMPLE_VOCAB] these elaborate[COMPLEX_VOCAB] meals[SIMPLE_VOCAB].[SIMPLE_SENT] Sometimes I think[SIMPLE_VOCAB] she tries[SIMPLE_VOCAB] too hard[SIMPLE_VOCAB], you know[FILLER]?[COMPLEX_SENT] But I appreciate[COMPLEX_VOCAB] it.[SIMPLE_SENT] Life's been good[SIMPLE_VOCAB] to us, I guess[SIMPLE_VOCAB].[SIMPLE_SENT] We been[GRAM_ERROR] married[SIMPLE_VOCAB] for twenty-five[COMPLEX_VOCAB] years[SIMPLE_VOCAB] now.[SIMPLE_SENT] Time flies when you're having fun[FIGURATIVE], as they say.[COMPLEX_SENT]"""

# Create Gradio interface
with gr.Blocks(title="Speech Analysis", theme=gr.themes.Soft()) as demo:
    gr.Markdown("""
    # 📋 Speech Analysis Tool with Annotations
    
    This tool performs a two-step comprehensive speech analysis:
    1. **Annotation**: Marks linguistic features in the transcript
    2. **Analysis**: Counts and analyzes the marked features for detailed assessment
    
    Upload a transcript or paste text below to begin the analysis.
    """)
    
    with gr.Tab("📝 Full Analysis Pipeline"):
        gr.Markdown("### Complete two-step analysis: annotation followed by comprehensive analysis")
        
        with gr.Row():
            with gr.Column(scale=2):
                transcript_input = gr.Textbox(
                    label="Speech Transcript",
                    placeholder="Paste the speech transcript here...",
                    lines=10,
                    max_lines=20
                )
                
                file_input = gr.File(
                    label="Or upload transcript file",
                    file_types=[".txt", ".doc", ".docx"]
                )
                
                with gr.Row():
                    age_input = gr.Textbox(
                        label="Age",
                        placeholder="e.g., 45",
                        value="45"
                    )
                    gender_input = gr.Dropdown(
                        label="Gender",
                        choices=["Male", "Female", "Other"],
                        value="Male"
                    )
                
                slp_notes_input = gr.Textbox(
                    label="SLP Clinical Notes (Optional)",
                    placeholder="Add any relevant clinical observations...",
                    lines=3
                )
                
                example_btn = gr.Button("📄 Load Example Transcript", variant="secondary", size="sm")
                
                # Single main analysis button
                ultimate_analysis_btn = gr.Button("🚀 Run Complete CASL Analysis", variant="primary", size="lg")
            
            with gr.Column(scale=3):
                status_display = gr.Markdown("Ready to analyze transcript")
                
                annotated_output = gr.Textbox(
                    label="Step 1: Annotated Transcript (✓ = Complete, ⚠️ = Incomplete)",
                    lines=15,
                    max_lines=25,
                    show_copy_button=True
                )
                
                analysis_output = gr.Textbox(
                    label="Step 2: Comprehensive Analysis",
                    lines=20,
                    max_lines=30,
                    show_copy_button=True
                )
    
    with gr.Tab("🏷️ Annotation Only"):
        gr.Markdown("### Step 1: Annotate transcript with linguistic markers")
        
        with gr.Row():
            with gr.Column():
                transcript_input_2 = gr.Textbox(
                    label="Speech Transcript",
                    placeholder="Paste the speech transcript here...",
                    lines=10
                )
                
                with gr.Row():
                    age_input_2 = gr.Textbox(label="Age", value="45")
                    gender_input_2 = gr.Dropdown(
                        label="Gender",
                        choices=["Male", "Female", "Other"],
                        value="Male"
                    )
                
                slp_notes_input_2 = gr.Textbox(
                    label="SLP Clinical Notes (Optional)",
                    lines=3
                )
                
                example_btn_2 = gr.Button("📄 Load Example Transcript", variant="secondary", size="sm")
                annotate_btn = gr.Button("🏷️ Annotate Transcript", variant="secondary")
            
            with gr.Column():
                annotation_output = gr.Textbox(
                    label="Annotated Transcript (✓ = Complete, ⚠️ = Incomplete)",
                    lines=20,
                    show_copy_button=True
                )
    
    with gr.Tab("📊 Analysis Only"):
        gr.Markdown("### Step 2: Analyze pre-annotated transcript")
        
        with gr.Row():
            with gr.Column():
                annotated_input = gr.Textbox(
                    label="Annotated Transcript",
                    placeholder="Paste annotated transcript with [MARKERS] here...",
                    lines=10
                )
                
                with gr.Row():
                    age_input_3 = gr.Textbox(label="Age", value="45")
                    gender_input_3 = gr.Dropdown(
                        label="Gender",
                        choices=["Male", "Female", "Other"],
                        value="Male"
                    )
                
                slp_notes_input_3 = gr.Textbox(
                    label="SLP Clinical Notes (Optional)",
                    lines=3
                )
                
                example_annotated_btn = gr.Button("📄 Load Example Annotated Transcript", variant="secondary", size="sm")
                analyze_only_btn = gr.Button("📊 Analyze Annotated Transcript", variant="secondary")
            
            with gr.Column():
                analysis_only_output = gr.Textbox(
                    label="Comprehensive Analysis",
                    lines=20,
                    show_copy_button=True
                )
    
    # Event handlers - now all components are defined
    example_btn.click(fn=lambda: example_transcript, outputs=[transcript_input])
    example_btn_2.click(fn=lambda: example_transcript, outputs=[transcript_input_2])
    example_annotated_btn.click(fn=lambda: example_annotated, outputs=[annotated_input])
    
    file_input.change(
        fn=process_file,
        inputs=[file_input],
        outputs=[transcript_input]
    )
    
    def run_annotation_step(transcript_content, age, gender, slp_notes):
        """Run just the annotation step and return immediately"""
        if not transcript_content or len(transcript_content.strip()) < 50:
            return "Error: Please provide a longer transcript for annotation.", "❌ Error"
        
        logger.info("Step 1: Annotating transcript with linguistic markers...")
        annotated_transcript = annotate_transcript(transcript_content, age, gender, slp_notes)
        
        if annotated_transcript.startswith("❌"):
            return annotated_transcript, "❌ Annotation failed"
        elif annotated_transcript.startswith("⚠️ ANNOTATION INCOMPLETE"):
            return annotated_transcript, "⚠️ Annotation incomplete but proceeding"
        else:
            return annotated_transcript, "✅ Annotation complete! Click 'Run Analysis' to continue."
    
    def run_analysis_step(annotated_transcript, original_transcript, age, gender, slp_notes):
        """Run the analysis step on the annotated transcript"""
        if not annotated_transcript or len(annotated_transcript.strip()) < 50:
            return "Error: Please provide an annotated transcript for analysis."
        
        logger.info("Step 2: Analyzing annotated transcript...")
        
        # Check if annotation was incomplete
        if annotated_transcript.startswith("⚠️ ANNOTATION INCOMPLETE"):
            analysis_note = "⚠️ Note: Annotation was incomplete. Analysis primarily based on original transcript.\n\n"
        else:
            analysis_note = ""
        
        analysis_result = analyze_with_backup(annotated_transcript, original_transcript, age, gender, slp_notes)
        return analysis_note + analysis_result
    
    def run_manual_count_only(annotated_transcript):
        """Generate only the manual count report without AI analysis"""
        if not annotated_transcript or len(annotated_transcript.strip()) < 50:
            return "Error: Please provide an annotated transcript for manual counting."
        
        return generate_manual_count_report(annotated_transcript)
    
    def run_verified_analysis(annotated_transcript, original_transcript, age, gender, slp_notes):
        """Run analysis with manual count verification"""
        if not annotated_transcript or len(annotated_transcript.strip()) < 50:
            return "Error: Please provide an annotated transcript for analysis."
        
        # Generate comprehensive analysis report first
        comprehensive_report = generate_comprehensive_analysis_report(annotated_transcript, original_transcript)
        
        # Get all the verified data
        marker_analysis = analyze_annotation_markers(annotated_transcript)
        linguistic_metrics = calculate_linguistic_metrics(original_transcript)
        lexical_diversity = calculate_advanced_lexical_diversity(original_transcript)
        
        # Create a comprehensive verified analysis prompt
        verified_prompt = f"""
        You are a speech-language pathologist conducting analysis based on COMPREHENSIVE VERIFIED DATA. 
        Do NOT recount anything - use ONLY the provided verified measurements below.

        Patient: {age}-year-old {gender}
        
        COMPREHENSIVE VERIFIED ANALYSIS DATA (DO NOT RECOUNT):
        {comprehensive_report}
        
        ANNOTATED TRANSCRIPT (for examples only, do not recount):
        {annotated_transcript}...
        
        INSTRUCTIONS: 
        Use ONLY the verified data provided above. Do NOT count or calculate anything yourself.
        
        Provide a comprehensive clinical interpretation organized into these sections:

        1. LEXICAL DIVERSITY INTERPRETATION:
        - Interpret the advanced lexical diversity measures (MTLD, HDD, MATTR, etc.)
        - Compare to age-appropriate norms
        - Clinical significance of diversity patterns

        2. FLUENCY PATTERN ANALYSIS:
        - Clinical interpretation of fluency marker counts and rates
        - Severity assessment based on verified counts
        - Impact on communication effectiveness

        3. GRAMMATICAL COMPETENCE ASSESSMENT:
        - Analysis of grammar error patterns from verified counts
        - Developmental appropriateness
        - Areas of strength vs. weakness

        4. VOCABULARY AND SEMANTIC ANALYSIS:
        - Interpretation of vocabulary sophistication measures
        - Word frequency pattern analysis
        - Semantic precision assessment

        5. PRAGMATIC LANGUAGE EVALUATION:
        - Discourse coherence based on verified markers
        - Social communication effectiveness
        - Conversational competence

        6. OVERALL COMMUNICATION PROFILE:
        - Integration of all verified measures
        - Strengths and areas of need
        - Functional communication impact

        7. CLINICAL RECOMMENDATIONS:
        - Specific intervention targets based on verified data
        - Therapy approaches and techniques
        - Progress monitoring suggestions
        - Prognosis and expected outcomes

        Focus on INTERPRETATION and CLINICAL SIGNIFICANCE, not counting. 
        All measurements are already verified and accurate.
        Cite specific examples from the transcript to support your interpretations.
        """
        
        ai_interpretation = call_claude_api(verified_prompt)
        
        return f"{comprehensive_report}\n\n{'='*100}\nCLINICAL INTERPRETATION BASED ON COMPREHENSIVE VERIFIED DATA\n{'='*100}\n\n{ai_interpretation}"
    
    def run_ultimate_analysis(annotated_transcript, original_transcript, age, gender, slp_notes):
        """The ultimate analysis: gather all statistical data, then do final 12-section clinical analysis"""
        if not annotated_transcript or len(annotated_transcript.strip()) < 50:
            return "Error: Please provide an annotated transcript for analysis."
        
        # STEP 1: Gather ALL statistical data
        linguistic_metrics = calculate_linguistic_metrics(original_transcript)
        marker_analysis = analyze_annotation_markers(annotated_transcript)
        lexical_diversity = calculate_advanced_lexical_diversity(original_transcript)
        
        # STEP 2: Get AI clinical insights (for interpretation, not counting)
        ai_clinical_insights = analyze_with_backup(annotated_transcript, original_transcript, age, gender, slp_notes)
        
        # STEP 3: Prepare all verified statistical values for final prompt
        stats_summary = f"""
        VERIFIED STATISTICAL VALUES (DO NOT RECOUNT - USE THESE EXACT NUMBERS):
        
        BASIC METRICS:
        • Total words: {linguistic_metrics.get('total_words', 0)}
        • Total sentences: {linguistic_metrics.get('total_sentences', 0)}
        • Unique words: {linguistic_metrics.get('unique_words', 0)}
        • MLU (words): {linguistic_metrics.get('mlu_words', 0):.2f}
        • MLU (morphemes): {linguistic_metrics.get('mlu_morphemes', 0):.2f}
        • Average sentence length: {linguistic_metrics.get('avg_sentence_length', 0):.2f}
        • Sentence length std: {linguistic_metrics.get('sentence_length_std', 0):.2f}
        
        LEXICAL DIVERSITY MEASURES (from lexical-diversity library):"""
        
        if lexical_diversity.get('library_available', False) and 'diversity_measures' in lexical_diversity:
            measures = lexical_diversity['diversity_measures']
            stats_summary += f"""
        • Simple TTR: {measures.get('simple_ttr', 'N/A')}
        • Root TTR: {measures.get('root_ttr', 'N/A')}
        • Log TTR: {measures.get('log_ttr', 'N/A')}
        • Maas TTR: {measures.get('maas_ttr', 'N/A')}
        • HDD: {measures.get('hdd', 'N/A')}
        • MSTTR (25-word): {measures.get('msttr_25', 'N/A')}
        • MSTTR (50-word): {measures.get('msttr_50', 'N/A')}
        • MATTR (25-word): {measures.get('mattr_25', 'N/A')}
        • MATTR (50-word): {measures.get('mattr_50', 'N/A')}
        • MTLD: {measures.get('mtld', 'N/A')}
        • MTLD (MA wrap): {measures.get('mtld_ma_wrap', 'N/A')}
        • MTLD (MA bidirectional): {measures.get('mtld_ma_bid', 'N/A')}"""
        else:
            stats_summary += "\n        • Lexical diversity measures not available"
        
        # Add manual annotation counts
        marker_counts = marker_analysis['marker_counts']
        category_totals = marker_analysis['category_totals']
        total_words = linguistic_metrics.get('total_words', 0)
        
        stats_summary += f"""
        
        MANUAL ANNOTATION COUNTS:
        • FILLER markers: {marker_counts.get('FILLER', 0)} ({marker_counts.get('FILLER', 0)/total_words*100:.2f} per 100 words)
        • FALSE_START markers: {marker_counts.get('FALSE_START', 0)}
        • REPETITION markers: {marker_counts.get('REPETITION', 0)}
        • REVISION markers: {marker_counts.get('REVISION', 0)}
        • PAUSE markers: {marker_counts.get('PAUSE', 0)}
        • GRAM_ERROR markers: {marker_counts.get('GRAM_ERROR', 0)}
        • SYNTAX_ERROR markers: {marker_counts.get('SYNTAX_ERROR', 0)}
        • MORPH_ERROR markers: {marker_counts.get('MORPH_ERROR', 0)}
        • SIMPLE_VOCAB markers: {marker_counts.get('SIMPLE_VOCAB', 0)}
        • COMPLEX_VOCAB markers: {marker_counts.get('COMPLEX_VOCAB', 0)}
        • SIMPLE_SENT markers: {marker_counts.get('SIMPLE_SENT', 0)}
        • COMPLEX_SENT markers: {marker_counts.get('COMPLEX_SENT', 0)}
        • COMPOUND_SENT markers: {marker_counts.get('COMPOUND_SENT', 0)}
        • FIGURATIVE markers: {marker_counts.get('FIGURATIVE', 0)}
        • PRONOUN_REF markers: {marker_counts.get('PRONOUN_REF', 0)}
        • TOPIC_SHIFT markers: {marker_counts.get('TOPIC_SHIFT', 0)}
        • TANGENT markers: {marker_counts.get('TANGENT', 0)}
        • CIRCUMLOCUTION markers: {marker_counts.get('CIRCUMLOCUTION', 0)}
        • GENERIC markers: {marker_counts.get('GENERIC', 0)}
        • WORD_SEARCH markers: {marker_counts.get('WORD_SEARCH', 0)}
        
        CATEGORY TOTALS:
        • Total fluency issues: {category_totals['fluency_issues']} ({category_totals['fluency_issues']/total_words*100:.2f} per 100 words)
        • Total grammar errors: {category_totals['grammar_errors']} ({category_totals['grammar_errors']/total_words*100:.2f} per 100 words)
        • Vocabulary sophistication ratio: {category_totals['vocab_sophistication_ratio']:.3f}
        """
        
        # STEP 4: Create the final comprehensive prompt
        final_prompt = f"""
        You are a speech-language pathologist conducting the FINAL COMPREHENSIVE 12-SECTION CASL ANALYSIS.

        Patient: {age}-year-old {gender}
        
        {stats_summary}
        
        CLINICAL INSIGHTS FROM AI ANALYSIS (for interpretation guidance):
        {ai_clinical_insights[:4000]}...
        
        ANNOTATED TRANSCRIPT (for specific examples):
        {annotated_transcript}
        
        CRITICAL INSTRUCTIONS:
        1. Use ONLY the verified statistical values provided above - DO NOT recount anything
        2. Use the clinical insights for interpretation guidance
        3. Use the annotated transcript for specific examples and quotes
        4. Complete ALL 12 sections of the comprehensive analysis
        
        COMPREHENSIVE SPEECH SAMPLE ANALYSIS:

        1. SPEECH FACTORS (with EXACT verified counts and specific citations):
        A. Fluency Issues: Use the verified counts above, cite specific examples from transcript
        B. Word Retrieval Issues: Use verified counts, analyze patterns with examples
        C. Grammatical Errors: Use verified error counts, categorize with examples

        2. LANGUAGE SKILLS ASSESSMENT (with verified evidence):
        A. Lexical/Semantic Skills: Use verified lexical diversity measures and vocabulary data
        B. Syntactic Skills: Use verified sentence complexity counts and MLU data
        C. Supralinguistic Skills: Clinical interpretation with transcript examples

        3. COMPLEX SENTENCE ANALYSIS (with verified counts):
        A. Coordinating Conjunctions: Count from transcript, use verified sentence data
        B. Subordinating Conjunctions: Count from transcript, analyze complexity
        C. Sentence Structure Analysis: Use verified MLU and sentence type data

        4. FIGURATIVE LANGUAGE ANALYSIS (with verified counts):
        A. Similes and Metaphors: Use verified figurative markers, cite examples
        B. Idioms and Non-literal Language: Analysis with specific examples

        5. PRAGMATIC LANGUAGE ASSESSMENT (with verified examples):
        A. Discourse Management: Use verified pragmatic marker counts
        B. Referential Communication: Use verified pronoun reference data

        6. VOCABULARY AND SEMANTIC ANALYSIS (with verified quantification):
        A. Vocabulary Diversity: Use ALL verified lexical diversity measures (MTLD, HDD, etc.)
        B. Semantic Relationships: Use verified word frequency and sophistication data

        7. MORPHOLOGICAL AND PHONOLOGICAL ANALYSIS (with verified counts):
        A. Morphological Markers: Use verified morphological data and MLU morphemes
        B. Phonological Patterns: Analysis from transcript evidence

        8. COGNITIVE-LINGUISTIC FACTORS (with verified evidence):
        A. Working Memory: Use verified sentence length and complexity data
        B. Processing Efficiency: Use verified fluency and error pattern data
        C. Executive Function: Use verified self-correction patterns

        9. FLUENCY AND RHYTHM ANALYSIS (with verified quantification):
        A. Disfluency Patterns: Use verified fluency counts and rates
        B. Language Flow: Use verified sentence variability data

        10. QUANTITATIVE METRICS (report ALL verified data):
        Report all the verified statistical values provided above

        11. CLINICAL IMPLICATIONS:
        Based on verified data, provide clinical interpretation and recommendations

        12. PROGNOSIS AND SUMMARY:
        Overall profile based on comprehensive verified data

        REQUIREMENTS:
        - Complete ALL 12 sections
        - Use ONLY verified statistical values (never recount)
        - Cite specific examples from annotated transcript
        - Provide clinical interpretation of the verified data
        - If response is cut off, end with <CONTINUE>
        """
        
        # STEP 5: Get the final comprehensive analysis
        final_result = call_claude_api_with_continuation(final_prompt)
        
        return final_result
    
    def run_full_pipeline(transcript_content, age, gender, slp_notes):
        """Run the complete pipeline but return annotation immediately"""
        if not transcript_content or len(transcript_content.strip()) < 50:
            return "Error: Please provide a longer transcript for analysis.", "", "❌ Error"
        
        # Step 1: Get annotation
        annotated_transcript, annotation_status = run_annotation_step(transcript_content, age, gender, slp_notes)
        
        if annotated_transcript.startswith("❌"):
            return annotated_transcript, "", annotation_status
        
        # Step 2: Run analysis
        analysis_result = run_analysis_step(annotated_transcript, transcript_content, age, gender, slp_notes)
        
        return annotated_transcript, analysis_result, "✅ Complete analysis finished!"
    
    def run_complete_casl_analysis(transcript_content, age, gender, slp_notes):
        """Run the complete CASL analysis pipeline with ultimate analysis"""
        if not transcript_content or len(transcript_content.strip()) < 50:
            return "Error: Please provide a longer transcript for analysis.", "", "❌ Error"
        
        # Step 1: Annotate transcript
        annotated_transcript, annotation_status = run_annotation_step(transcript_content, age, gender, slp_notes)
        
        if annotated_transcript.startswith("❌"):
            return annotated_transcript, "", annotation_status
        
        # Step 2: Run ultimate analysis
        ultimate_result = run_ultimate_analysis(annotated_transcript, transcript_content, age, gender, slp_notes)
        
        return annotated_transcript, ultimate_result, "✅ Complete CASL analysis finished!"
    
    # Single main event handler
    ultimate_analysis_btn.click(
        fn=run_complete_casl_analysis,
        inputs=[transcript_input, age_input, gender_input, slp_notes_input],
        outputs=[annotated_output, analysis_output, status_display]
    )
    
    annotate_btn.click(
        fn=annotate_transcript,
        inputs=[transcript_input_2, age_input_2, gender_input_2, slp_notes_input_2],
        outputs=[annotation_output]
    )
    
    def analyze_standalone(annotated_transcript, age, gender, slp_notes):
        """Analyze standalone annotated transcript"""
        # Extract original transcript by removing markers
        original_transcript = annotated_transcript
        for marker in ['[FILLER]', '[FALSE_START]', '[REPETITION]', '[REVISION]', '[PAUSE]', 
                      '[CIRCUMLOCUTION]', '[INCOMPLETE]', '[GENERIC]', '[WORD_SEARCH]', 
                      '[GRAM_ERROR]', '[SYNTAX_ERROR]', '[MORPH_ERROR]', '[RUN_ON]', 
                      '[SIMPLE_VOCAB]', '[COMPLEX_VOCAB]', '[SEMANTIC_ERROR]', 
                      '[TOPIC_SHIFT]', '[TANGENT]', '[INAPPROPRIATE]', '[COHERENCE_BREAK]', 
                      '[SIMPLE_SENT]', '[COMPLEX_SENT]', '[COMPOUND_SENT]', '[FIGURATIVE]', 
                      '[PRONOUN_REF]', '[MAZING]', '[PERSEVERATION]']:
            original_transcript = original_transcript.replace(marker, '')
        
        return analyze_with_backup(annotated_transcript, original_transcript, age, gender, slp_notes)
    
    analyze_only_btn.click(
        fn=analyze_standalone,
        inputs=[annotated_input, age_input_3, gender_input_3, slp_notes_input_3],
        outputs=[analysis_only_output]
    )

if __name__ == "__main__":
    demo.launch(
        server_name="0.0.0.0",
        server_port=7860,
        share=True,
        show_error=True
    )