{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fae47fa6-6745-44c1-ae86-0fa08a1eccb7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch version :  2.4.1+cu124\n",
      "transformers version :  4.51.3\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import transformers\n",
    "from awq import AutoAWQForCausalLM\n",
    "from transformers import AutoTokenizer, AutoProcessor, AutoConfig\n",
    "\n",
    "print('torch version : ', torch.__version__)\n",
    "print('transformers version : ', transformers.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1c95abcd-4ea7-416d-9e8b-c9a96bc1c318",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bff4975696e04d38830aeb46375353ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "755be950-f1bb-4b3b-a098-991ec9a2dcf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = \"/workspace/models/Llama-4-Scout-17B-16E-Instruct-FP16\"\n",
    "quant_path = '/tmp/Llama-4-Scout-17B-16E-Instruct-AWQ'\n",
    "quant_config = {\n",
    "    \"zero_point\": True, \n",
    "    \"q_group_size\": 128, \n",
    "    \"w_bit\": 4, \n",
    "    \"version\": \"GEMM\", \n",
    "    \"modules_to_not_convert\": [\"feed_forward.router\"],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "274b6d5d-28ad-4696-bbc7-f545c0f2ba55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Llama4Config {\n",
       "  \"architectures\": [\n",
       "    \"Llama4ForConditionalGeneration\"\n",
       "  ],\n",
       "  \"boi_token_index\": 200080,\n",
       "  \"eoi_token_index\": 200081,\n",
       "  \"image_token_index\": 200092,\n",
       "  \"model_type\": \"llama4\",\n",
       "  \"text_config\": {\n",
       "    \"_attn_implementation_autoset\": true,\n",
       "    \"attention_bias\": false,\n",
       "    \"attention_chunk_size\": 8192,\n",
       "    \"attention_dropout\": 0.0,\n",
       "    \"attn_scale\": 0.1,\n",
       "    \"attn_temperature_tuning\": 4,\n",
       "    \"bos_token_id\": 200000,\n",
       "    \"cache_implementation\": \"hybrid\",\n",
       "    \"eos_token_id\": [\n",
       "      200001,\n",
       "      200007,\n",
       "      200008\n",
       "    ],\n",
       "    \"floor_scale\": 8192,\n",
       "    \"for_llm_compressor\": false,\n",
       "    \"head_dim\": 128,\n",
       "    \"hidden_act\": \"silu\",\n",
       "    \"hidden_size\": 5120,\n",
       "    \"initializer_range\": 0.02,\n",
       "    \"interleave_moe_layer_step\": 1,\n",
       "    \"intermediate_size\": 8192,\n",
       "    \"intermediate_size_mlp\": 16384,\n",
       "    \"max_position_embeddings\": 10485760,\n",
       "    \"model_type\": \"llama4_text\",\n",
       "    \"moe_layers\": [\n",
       "      0,\n",
       "      1,\n",
       "      2,\n",
       "      3,\n",
       "      4,\n",
       "      5,\n",
       "      6,\n",
       "      7,\n",
       "      8,\n",
       "      9,\n",
       "      10,\n",
       "      11,\n",
       "      12,\n",
       "      13,\n",
       "      14,\n",
       "      15,\n",
       "      16,\n",
       "      17,\n",
       "      18,\n",
       "      19,\n",
       "      20,\n",
       "      21,\n",
       "      22,\n",
       "      23,\n",
       "      24,\n",
       "      25,\n",
       "      26,\n",
       "      27,\n",
       "      28,\n",
       "      29,\n",
       "      30,\n",
       "      31,\n",
       "      32,\n",
       "      33,\n",
       "      34,\n",
       "      35,\n",
       "      36,\n",
       "      37,\n",
       "      38,\n",
       "      39,\n",
       "      40,\n",
       "      41,\n",
       "      42,\n",
       "      43,\n",
       "      44,\n",
       "      45,\n",
       "      46,\n",
       "      47\n",
       "    ],\n",
       "    \"no_rope_layers\": [\n",
       "      1,\n",
       "      1,\n",
       "      1,\n",
       "      0,\n",
       "      1,\n",
       "      1,\n",
       "      1,\n",
       "      0,\n",
       "      1,\n",
       "      1,\n",
       "      1,\n",
       "      0,\n",
       "      1,\n",
       "      1,\n",
       "      1,\n",
       "      0,\n",
       "      1,\n",
       "      1,\n",
       "      1,\n",
       "      0,\n",
       "      1,\n",
       "      1,\n",
       "      1,\n",
       "      0,\n",
       "      1,\n",
       "      1,\n",
       "      1,\n",
       "      0,\n",
       "      1,\n",
       "      1,\n",
       "      1,\n",
       "      0,\n",
       "      1,\n",
       "      1,\n",
       "      1,\n",
       "      0,\n",
       "      1,\n",
       "      1,\n",
       "      1,\n",
       "      0,\n",
       "      1,\n",
       "      1,\n",
       "      1,\n",
       "      0,\n",
       "      1,\n",
       "      1,\n",
       "      1,\n",
       "      0\n",
       "    ],\n",
       "    \"num_attention_heads\": 40,\n",
       "    \"num_experts_per_tok\": 1,\n",
       "    \"num_hidden_layers\": 48,\n",
       "    \"num_key_value_heads\": 8,\n",
       "    \"num_local_experts\": 16,\n",
       "    \"output_router_logits\": false,\n",
       "    \"pad_token_id\": 200018,\n",
       "    \"rms_norm_eps\": 1e-05,\n",
       "    \"rope_scaling\": {\n",
       "      \"factor\": 16.0,\n",
       "      \"high_freq_factor\": 1.0,\n",
       "      \"low_freq_factor\": 1.0,\n",
       "      \"original_max_position_embeddings\": 8192,\n",
       "      \"rope_type\": \"llama3\"\n",
       "    },\n",
       "    \"rope_theta\": 500000.0,\n",
       "    \"router_aux_loss_coef\": 0.001,\n",
       "    \"router_jitter_noise\": 0.0,\n",
       "    \"torch_dtype\": \"float16\",\n",
       "    \"use_cache\": true,\n",
       "    \"use_qk_norm\": true,\n",
       "    \"vocab_size\": 202048\n",
       "  },\n",
       "  \"tie_word_embeddings\": false,\n",
       "  \"torch_dtype\": \"float16\",\n",
       "  \"transformers_version\": \"4.51.3\",\n",
       "  \"vision_config\": {\n",
       "    \"_attn_implementation_autoset\": true,\n",
       "    \"attention_dropout\": 0.0,\n",
       "    \"hidden_act\": \"gelu\",\n",
       "    \"hidden_size\": 1408,\n",
       "    \"image_size\": 336,\n",
       "    \"initializer_range\": 0.02,\n",
       "    \"intermediate_size\": 5632,\n",
       "    \"model_type\": \"llama4_vision_model\",\n",
       "    \"multi_modal_projector_bias\": false,\n",
       "    \"norm_eps\": 1e-05,\n",
       "    \"num_attention_heads\": 16,\n",
       "    \"num_channels\": 3,\n",
       "    \"num_hidden_layers\": 34,\n",
       "    \"patch_size\": 14,\n",
       "    \"pixel_shuffle_ratio\": 0.5,\n",
       "    \"projector_dropout\": 0.0,\n",
       "    \"projector_input_dim\": 4096,\n",
       "    \"projector_output_dim\": 4096,\n",
       "    \"rope_theta\": 10000,\n",
       "    \"torch_dtype\": \"float16\",\n",
       "    \"vision_feature_layer\": -1,\n",
       "    \"vision_feature_select_strategy\": \"default\",\n",
       "    \"vision_output_dim\": 4096\n",
       "  }\n",
       "}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = AutoConfig.from_pretrained(model_path)\n",
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4becc671-d45f-4295-8bf3-9595a6a1236a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You have set `use_cache` to `False`, but cache_implementation is set to hybrid. cache_implementation will have no effect.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "484ad5106617400fac03c9622b460b71",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/50 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "696dd04fb9614bf5a1b0b0ffe7b948f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Replacing MoE Block...:   0%|          | 0/48 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 56min 56s, sys: 2h 13min 21s, total: 3h 10min 17s\n",
      "Wall time: 35min 53s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model = AutoAWQForCausalLM.from_pretrained(model_path, torch_dtype=torch.float16, device_map='cpu')\n",
    "model.model = model.to(torch.float16)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
    "processor = AutoProcessor.from_pretrained(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7cb92f32-9e5f-4262-b6db-f6819917c292",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tue Apr 29 14:13:24 2025       \n",
      "+-----------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 565.57.01              Driver Version: 565.57.01      CUDA Version: 12.7     |\n",
      "|-----------------------------------------+------------------------+----------------------+\n",
      "| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                        |               MIG M. |\n",
      "|=========================================+========================+======================|\n",
      "|   0  NVIDIA A100-SXM4-80GB          On  |   00000000:C2:00.0 Off |                    0 |\n",
      "| N/A   24C    P0             66W /  500W |       4MiB /  81920MiB |      0%      Default |\n",
      "|                                         |                        |             Disabled |\n",
      "+-----------------------------------------+------------------------+----------------------+\n",
      "                                                                                         \n",
      "+-----------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                              |\n",
      "|  GPU   GI   CI        PID   Type   Process name                              GPU Memory |\n",
      "|        ID   ID                                                               Usage      |\n",
      "|=========================================================================================|\n",
      "|  No running processes found                                                             |\n",
      "+-----------------------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3ca59a03-38d8-4fe3-905b-499aac0e1822",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df6ecf5c22cf409c939cac539db12fc6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/167 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Repo card metadata block was not found. Setting CardData to empty.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bccd6cc25eb04454a234f80fa1387def",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "val.jsonl.zst:   0%|          | 0.00/471M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c74329db5b74447a8fc3d4cfb8750df3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split:   0%|          | 0/214670 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "AWQ: 100%|██████████| 48/48 [2:31:04<00:00, 188.83s/it]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4h 28min 37s, sys: 2h 34min 23s, total: 7h 3min\n",
      "Wall time: 2h 31min 46s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model.quantize(tokenizer, quant_config=quant_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f5ccac8e-fd4c-4474-a0da-2305c57dbcbe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/tmp/Llama-4-Scout-17B-16E-Instruct-AWQ/processor_config.json']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.save_quantized(quant_path)\n",
    "tokenizer.save_pretrained(quant_path)\n",
    "processor.save_pretrained(quant_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "05b5d2c8-fbf7-4ac8-b854-6202c7d9fd15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb5f653e788a44768daa925ac661de0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/27.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "629e668e54934ce9b04a39ad377b359e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00003-of-00013.safetensors:   0%|          | 0.00/4.99G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2fc98d2a32cf4abcab88bb419405b622",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00004-of-00013.safetensors:   0%|          | 0.00/4.98G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d7c808182924fe2a56838f5b5eef99c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00002-of-00013.safetensors:   0%|          | 0.00/4.99G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d5e8cbbda74e45678911515bcc35220d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload 13 LFS files:   0%|          | 0/13 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc6360a5900f4e40afcef065bd932aa2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00001-of-00013.safetensors:   0%|          | 0.00/4.99G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0dcf0ccfcf04da4bbca6133700a5ebe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00005-of-00013.safetensors:   0%|          | 0.00/4.99G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39c497ce9d094cc3a85cb208c15bef67",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00006-of-00013.safetensors:   0%|          | 0.00/4.99G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88dae4a0b2bd4a31ad19c3a244e5b90d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00007-of-00013.safetensors:   0%|          | 0.00/4.98G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "330d08a5abc2446597822475b70b8b86",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00008-of-00013.safetensors:   0%|          | 0.00/4.99G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dcbde5192a794ccd95d7f22965f1afe6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00009-of-00013.safetensors:   0%|          | 0.00/4.99G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b209cbbccb0d4076980598c61872b55d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00010-of-00013.safetensors:   0%|          | 0.00/4.98G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb2daf0b6c3c4e77ac9bcb7de8b82dfa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00011-of-00013.safetensors:   0%|          | 0.00/4.99G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57f646c556154c8f83e70dfb6b02ea87",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00012-of-00013.safetensors:   0%|          | 0.00/3.87G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1f1ebb972a34ea0b0836e062a75b6ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model-00013-of-00013.safetensors:   0%|          | 0.00/2.07G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2def7f80f3694282a1eebd203e8fb59a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/27.9M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/kishizaki-sci/Llama-4-Scout-17B-16E-Instruct-AWQ-quant-example/commit/28b5efe956430c5a17962dad13b8c23d8c432797', commit_message='Upload processor', commit_description='', oid='28b5efe956430c5a17962dad13b8c23d8c432797', pr_url=None, repo_url=RepoUrl('https://huggingface.co/kishizaki-sci/Llama-4-Scout-17B-16E-Instruct-AWQ-quant-example', endpoint='https://huggingface.co', repo_type='model', repo_id='kishizaki-sci/Llama-4-Scout-17B-16E-Instruct-AWQ-quant-example'), pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.model.push_to_hub(\"kishizaki-sci/Llama-4-Scout-17B-16E-Instruct-AWQ-quant-example\")\n",
    "processor.push_to_hub(\"kishizaki-sci/Llama-4-Scout-17B-16E-Instruct-AWQ-quant-example\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1fb45be2-e8cf-4c35-bc83-83789609cb65",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "time.sleep(180)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3e826ab-1ac9-4e28-8021-ef254e72915e",
   "metadata": {},
   "outputs": [],
   "source": [
    "!runpodctl remove pod $RUNPOD_POD_ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c4a8fc0-208c-478d-978a-91c1f9466370",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}