File size: 4,360 Bytes
a683148 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 |
# scale_data.py - Scale up MAP-NEO Mini training data
import subprocess
import sys
import time
from pathlib import Path
def scale_training_data():
print("🚀 MAP-NEO Mini Data Scaling")
print("=" * 50)
print("Target: 50,000 documents (10x current scale)")
print("Expected result: ~25,000 training sequences")
print("Estimated time: 45-60 minutes")
print("=" * 50)
# Check if we already have large dataset
large_data = Path("data/tokens/packed_1024_large.txt")
if large_data.exists():
print("✅ Large dataset already exists!")
print(f"Found: {large_data}")
return str(large_data)
# Check if small dataset exists (backup)
small_data = Path("data/tokens/packed_1024.txt")
if small_data.exists():
backup_path = Path("data/tokens/packed_1024_small_backup.txt")
print(f"📁 Backing up current dataset to: {backup_path}")
small_data.rename(backup_path)
# Process 50k documents
print("\n🔄 Starting data processing...")
print("This will download and process 50,000 English documents")
cmd = [
sys.executable, "data_prep.py",
"--num_docs", "50000",
"--seq_length", "1024"
]
start_time = time.time()
try:
result = subprocess.run(cmd, check=True, capture_output=False, text=True)
elapsed = time.time() - start_time
print(f"\n✅ Data scaling completed in {elapsed/60:.1f} minutes!")
# Rename for clarity
old_path = Path("data/tokens/packed_1024.txt")
new_path = Path("data/tokens/packed_1024_large.txt")
if old_path.exists():
old_path.rename(new_path)
print(f"📊 Large dataset saved as: {new_path}")
# Count sequences
with open(new_path, 'r') as f:
seq_count = sum(1 for _ in f)
print(f"📈 Total sequences: {seq_count:,}")
return str(new_path)
else:
print("❌ Expected output file not found")
return None
except subprocess.CalledProcessError as e:
print(f"❌ Error in data processing:")
print(f"Return code: {e.returncode}")
return None
except KeyboardInterrupt:
print("\n⏹️ Process interrupted by user")
return None
def update_training_config():
"""Update train_neo.py to use large dataset"""
print("\n🔧 Updating training configuration...")
train_file = Path("train_neo.py")
if not train_file.exists():
print("❌ train_neo.py not found")
return
# Read current file
content = train_file.read_text(encoding='utf-8')
# Update data path and training steps
old_data_path = 'data_path: str = "data/tokens/packed_1024.txt"'
new_data_path = 'data_path: str = "data/tokens/packed_1024_large.txt"'
old_max_steps = 'max_steps: int = 50000'
new_max_steps = 'max_steps: int = 100000'
if old_data_path in content:
content = content.replace(old_data_path, new_data_path)
print("✅ Updated data_path to use large dataset")
if old_max_steps in content:
content = content.replace(old_max_steps, new_max_steps)
print("✅ Updated max_steps to 100,000 for extended training")
# Write back
train_file.write_text(content, encoding='utf-8')
print("💾 Training configuration updated!")
def main():
print("MAP-NEO Mini Data Scaling Pipeline")
# Scale data
result = scale_training_data()
if result:
# Update config
update_training_config()
print("\n" + "="*60)
print("🎉 DATA SCALING COMPLETE!")
print("="*60)
print("Next steps:")
print("1. Your large dataset is ready for training")
print("2. Training config updated for 100k steps")
print("3. Run: python train_neo.py")
print("4. Expected training time: ~3-4 hours")
print("5. Expected quality: Much more coherent text!")
print("="*60)
else:
print("\n❌ Data scaling failed. Check the errors above.")
if __name__ == "__main__":
main()
|