Upload folder using huggingface_hub

quantize.py

@@ -1,7 +1,8 @@
 from transformers import AutoProcessor, LlavaForConditionalGeneration
 
 from llmcompressor.modifiers.quantization import QuantizationModifier
-from llmcompressor.transformers import oneshot
+from llmcompressor import oneshot
+from llmcompressor.utils import dispatch_for_generation
 
 MODEL_ID = "llama-joycaption-beta-one-hf-llava"
 
@@ -17,17 +18,10 @@ processor = AutoProcessor.from_pretrained(MODEL_ID)
 recipe = QuantizationModifier(
     targets="Linear",
     scheme="FP8_DYNAMIC",
-    ignore=["re:.*lm_head", "re:multi_modal_projector.*", "re:vision_tower.*"],
+    ignore=["re:.*lm_head", "re:.*multi_modal_projector.*", "re:.*vision_tower.*"],
 )
 
 # Apply quantization and save to disk in compressed-tensors format.
 SAVE_DIR = MODEL_ID + "-FP8-Dynamic"
-oneshot(model=model, recipe=recipe, output_dir=SAVE_DIR)
+oneshot(model=model, recipe=recipe, output_dir=SAVE_DIR, save_compressed=True)
 processor.save_pretrained(SAVE_DIR)
-
-# Confirm generations of the quantized model look sane.
-print("========== SAMPLE GENERATION ==============")
-input_ids = processor(text="Hello my name is", return_tensors="pt").input_ids.to("cuda")
-output = model.generate(input_ids, max_new_tokens=20)
-print(processor.decode(output[0]))
-print("==========================================")