def calculate_avg_tokens_and_keywords(jsonl_path, tokenizer):
    """Calculate average tokens and keyword frequencies in the first code element"""
    if not os.path.exists(jsonl_path):
        print(f"Warning: JSONL file not found: {jsonl_path}")
        return 0, 0, 0, 0, 0, 0, 0, 0, 0

    keywords = {"recheck", "rethink", "try again", "wait", "alternatively", "retry", "however"}
    total_tokens = 0
    total_keywords = 0
    total_correct_tokens = 0
    total_wrong_tokens = 0
    total_stop_tokens = 0
    clip_count = 0
    total_repeats = 0
    count = 0
    correct_count = 0
    wrong_count = 0
    stop_count = 0
    box_count = 0
    
    try:
        with open(jsonl_path, 'r', encoding='utf-8') as f:
            for line in f:
                data = json.loads(line)
                if 'code' in data and isinstance(data['code'], list) and len(data['code']) > 0:
                    code_text = data['code'][0].lower()
                    tokens = len(tokenizer.encode(code_text))
                    total_tokens += tokens
                    
                    # Count keywords
                    keyword_count = sum(code_text.count(keyword.lower()) for keyword in keywords)
                    total_keywords += keyword_count
                    # Check for \boxed occurrences
                    if extract_last_boxed(code_text) is not None:
                        box_count += 1
                    # Check finish reason
                    if data.get('finish_reason', [None])[0] == 'length':
                        clip_count += 1
                    elif data.get('finish_reason', [None])[0] == 'stop':
                        total_stop_tokens += tokens
                        stop_count += 1
                    
                    # Separate tokens for correct and wrong answers
                    is_correct = data.get('score', [False])[0] if isinstance(data.get('score', []), list) else False
                    if is_correct:
                        total_correct_tokens += tokens
                        correct_count += 1
                    else:
                        total_wrong_tokens += tokens
                        wrong_count += 1
                    try:
                        if is_repeat(code_text):
                    #    repeat_count += 1
                            total_repeats += 1
                    except Exception as e:
                        # print("test")
                        total_repeats += 1
                    count += 1
    except Exception as e:
        print(f"Error processing {jsonl_path}: {e}")
        return 0, 0, 0, 0, 0, 0, 0, 0, 0
        
    avg_correct_tokens = total_correct_tokens / correct_count if correct_count > 0 else 0
    avg_wrong_tokens = total_wrong_tokens / wrong_count if wrong_count > 0 else 0
    clip_ratio = clip_count / count if count > 0 else 0
    avg_stop_tokens = total_stop_tokens / stop_count if stop_count > 0 else 0
    box_ratio = box_count / count if count > 0 else 0  # Calculate the ratio of boxed occurrences
    repeat_ratio = total_repeats / count if count > 0 else 0  # Calculate the repeat ratio
    return (total_tokens / count if count > 0 else 0,
            total_keywords / count if count > 0 else 0,
            avg_correct_tokens,
            avg_wrong_tokens,
            clip_ratio,
            avg_stop_tokens,
            box_ratio,  # Return the boxed ratio
            stop_count / count if count > 0 else 0,
            repeat_ratio)
Calculate average tokens and keyword frequencies in the first code element
examples/simplelr_math_eval/sh/collect_results.py