def calculate_entity_size(entity_dict: Dict[str, Any]) -> Tuple[int, Dict[str, int]]:
    """Calculate the token size of entity and its fields."""
    total_size = 0
    field_sizes = {}

    for field_name, field_value in entity_dict.items():
        if isinstance(field_value, str):
            size = count_tokens(field_value)
        elif isinstance(field_value, (dict, list)):
            # For complex types, serialize to JSON to count tokens
            size = count_tokens(json.dumps(field_value))
        else:
            # For other types, convert to string
            size = count_tokens(str(field_value))

        field_sizes[field_name] = size
        total_size += size

    return total_size, field_sizes
Calculate the token size of entity and its fields.
backend/airweave/platform/transformers/entity_field_chunker.py