Update README.md
Browse files
README.md
CHANGED
|
@@ -6,4 +6,23 @@ tags:
|
|
| 6 |
- sft
|
| 7 |
---
|
| 8 |
|
| 9 |
-
Experiment 1 SFT ALPACA INDO
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
- sft
|
| 7 |
---
|
| 8 |
|
| 9 |
+
Experiment 1 SFT ALPACA INDO
|
| 10 |
+
|
| 11 |
+
dataset: 9 millions token indo alpaca dataset
|
| 12 |
+
|
| 13 |
+
max_seq_length = 8192,
|
| 14 |
+
dataset_num_proc = 2,
|
| 15 |
+
packing = False,
|
| 16 |
+
args = TrainingArguments(
|
| 17 |
+
per_device_train_batch_size = 1,
|
| 18 |
+
gradient_accumulation_steps = 8,
|
| 19 |
+
warmup_steps = 5,
|
| 20 |
+
num_train_epochs = 1,
|
| 21 |
+
learning_rate = 5e-5,
|
| 22 |
+
fp16 = not is_bfloat16_supported(),
|
| 23 |
+
bf16 = is_bfloat16_supported(),
|
| 24 |
+
logging_steps = 1,
|
| 25 |
+
optim = "adamw_8bit",
|
| 26 |
+
weight_decay = 0.01,
|
| 27 |
+
lr_scheduler_type = "linear",
|
| 28 |
+
seed = 3407,
|