Huggingface model convert gguf

config env

• mkdir model and cd model
• conda create -n gguf
• conda activate gguf
• pip install huggingface_hub

dl model from hf:get model_id,local_dir

• create file:download.py and save
  from huggingface_hub import snapshot_download
  model_id="Qwen/Qwen2-1.5B-Instruct"
  snapshot_download(repo_id=model_id, local_dir="Qwen2-1.5B-Instruct",local_dir_use_symlinks=False, revision="main")
• python download.py

config llama.cpp

• cd ..
• git clone https://github.com/ggerganov/llama.cpp.git
• cd llama.cpp
• pip install -r requirements.txt

run python llama.cpp/convert_hf_to_gguf.py -h

convert

• cd ..
  check path for directory of model
• #full f16 or f32
  python ./llama.cpp/convert_hf_to_gguf.py ./model/qwen2_1.5b_instruct  --outtype f32 --verbose --outfile ./Qwen2-1.5B-Instruct_f32.gguf
• #q2_k,q3_k_l/q3_k_m/q3_k_s,q4_0/q4_1/q4_k_m/q4_k_s,q5_0/q5_1/q5_k_m/q5_k_s,q6_k,q8_0,fp16/f32
  python ./llama.cpp/convert_hf_to_gguf.py ./model/qwen2_1.5b_instruct  --outtype q8_0 --verbose --outfile ./Qwen2-1.5B-Instruct_q8_0.gguf