AI 学习笔记

三种适配方式对比

我们可以从三个维度来理解模型适配：

什么时候需要微调？

• 领域适配：通用模型在医疗、法律、金融等专业领域表现不佳
• 风格迁移：需要模型以特定风格回答（如客服语气、学术写作）
• 任务特定：需要在特定格式的任务上提升表现（如代码生成、数据提取）
• 数据隐私：不能将敏感数据发送到API，需要本地部署
• 成本控制：API调用成本过高，微调后自部署更经济

全参数微调的资源需求

以一个7B参数的模型为例：

• 模型参数存储：7B × 4字节（FP32）= 28GB，或 7B × 2字节（FP16）= 14GB
• 优化器状态：Adam优化器需要额外存储动量和方差，约 2 × 模型参数大小
• 梯度：与模型参数同等大小，7B × 2字节 = 14GB
• 激活值：取决于batch size和序列长度，通常需要数GB到数十GB

结论：微调一个7B模型，至少需要 64GB 显存（FP16）或更多。这需要多张A100或H100 GPU。

LoRA的数学原理

对于预训练权重矩阵 $W_0 \in \mathbb{R}^{d \times k}$，LoRA将更新表示为：

$W = W_0 + \Delta W = W_0 + BA$

其中 $B \in \mathbb{R}^{d \times r}$，$A \in \mathbb{R}^{r \times k}$，$r \ll \min(d, k)$。

• $W_0$ 是冻结的预训练权重（不更新）
• $A$ 用高斯随机初始化，$B$ 初始化为零
• $r$ 是低秩的秩，通常取 4、8、16、64
• 可训练参数量：$r \times (d + k)$，远小于 $d \times k$

LoRA的优势

LoRA代码示例：使用PEFT库

from peft import LoraConfig, get_peft_model, TaskType
from transformers import AutoModelForCausalLM

# 加载预训练模型
model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-2-7b-hf",
    torch_dtype="auto",
    device_map="auto"
)

# 配置LoRA
lora_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,    # 因果语言模型
    r=8,                              # LoRA的秩
    lora_alpha=32,                    # 缩放因子，通常为2*r
    lora_dropout=0.1,                 # Dropout比例
    target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],  # 目标模块
    bias="none",                      # 不训练bias
)

# 应用LoRA
model = get_peft_model(model, lora_config)

# 查看可训练参数量
model.print_trainable_parameters()
# 输出: trainable params: 4,194,304 || all params: 6,742,609,920 || trainable%: 0.0622

QLoRA的三大技术创新

• 4-bit NormalFloat (NF4)：一种信息论最优的4-bit数据类型，专为正态分布权重设计。比普通4-bit量化保留更多信息。
• 双重量化（Double Quantization）：对量化常数本身也进行量化，进一步减少内存占用。每个参数额外节省约0.37bit。
• 分页优化器（Paged Optimizers）：使用NVIDIA统一内存自动处理显存溢出，将优化器状态分页到CPU内存。

QLoRA完整训练代码

import torch
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig,
    TrainingArguments,
)
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
from trl import SFTTrainer

# 1. 配置4-bit量化
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,                    # 启用4-bit加载
    bnb_4bit_quant_type="nf4",            # NF4量化
    bnb_4bit_compute_dtype=torch.bfloat16, # 计算精度
    bnb_4bit_use_double_quant=True,        # 双重量化
)

# 2. 加载量化后的模型
model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-2-7b-hf",
    quantization_config=bnb_config,
    device_map="auto",
    trust_remote_code=True,
)
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
tokenizer.pad_token = tokenizer.eos_token

# 3. 准备模型用于QLoRA训练
model = prepare_model_for_kbit_training(model)

# 4. 配置LoRA
lora_config = LoraConfig(
    r=16,                     # 秩
    lora_alpha=32,            # 缩放因子
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=[
        "q_proj", "k_proj", "v_proj", "o_proj",
        "gate_proj", "up_proj", "down_proj"
    ],
)

model = get_peft_model(model, lora_config)

# 5. 配置训练参数
training_args = TrainingArguments(
    output_dir="./qlora-output",
    num_train_epochs=3,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    learning_rate=2e-4,
    weight_decay=0.01,
    warmup_ratio=0.03,
    lr_scheduler_type="cosine",
    logging_steps=10,
    save_strategy="epoch",
    fp16=False,
    bf16=True,
    report_to="wandb",
    optim="paged_adamw_32bit",  # 分页优化器
    max_grad_norm=0.3,
)

print(f"可训练参数: {model.print_trainable_parameters()}")

PEFT支持的微调方法

完整微调流程：使用SFTTrainer

from datasets import load_dataset
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig,
    TrainingArguments,
)
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
from trl import SFTTrainer
import torch

# ========== 1. 加载数据集 ==========
dataset = load_dataset("json", data_files="train_data.jsonl", split="train")
print(f"训练样本数: {len(dataset)}")

# ========== 2. 配置量化 ==========
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
)

# ========== 3. 加载模型 ==========
model_name = "THUDM/chatglm3-6b"
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    quantization_config=bnb_config,
    device_map="auto",
    trust_remote_code=True,
)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
tokenizer.pad_token = tokenizer.eos_token

# ========== 4. 准备PEFT ==========
model = prepare_model_for_kbit_training(model)
lora_config = LoraConfig(
    r=8,
    lora_alpha=16,
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=["query_key_value"],  # ChatGLM特有层名
)
model = get_peft_model(model, lora_config)

# ========== 5. 数据格式化 ==========
def format_prompt(example):
    """将数据格式化为指令模板"""
    return f"""### 指令:
{example['instruction']}

### 输入:
{example['input']}

### 回答:
{example['output']}"""

# ========== 6. 训练配置 ==========
training_args = TrainingArguments(
    output_dir="./chatglm3-finetuned",
    num_train_epochs=3,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=8,
    learning_rate=2e-4,
    warmup_steps=100,
    logging_steps=10,
    save_steps=200,
    save_total_limit=3,
    bf16=True,
    optim="paged_adamw_32bit",
    report_to="tensorboard",
    max_grad_norm=0.3,
    lr_scheduler_type="cosine",
)

# ========== 7. 开始训练 ==========
trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=dataset,
    formatting_func=format_prompt,
    args=training_args,
    max_seq_length=512,
)

trainer.train()

# ========== 8. 保存模型 ==========
trainer.save_model("./chatglm3-finetuned/final")
tokenizer.save_pretrained("./chatglm3-finetuned/final")

常见数据集格式

# 格式1: 指令格式（Instruction-Response）
{"instruction": "翻译以下句子为英文", "input": "今天天气很好", "output": "The weather is nice today."}

# 格式2: 对话格式（Conversation）
{"conversations": [
    {"role": "user", "content": "什么是机器学习？"},
    {"role": "assistant", "content": "机器学习是人工智能的一个分支..."}
]}

# 格式3: Alpaca格式（带输入）
{"instruction": "总结以下文章", "input": "人工智能正在改变世界...", "output": "本文介绍了AI的发展趋势..."}

# 格式4: ShareGPT格式（多轮对话）
{"conversation_id": "1", "items": [
    {"role": "human", "content": "你好"},
    {"role": "gpt", "content": "你好！有什么可以帮助你的？"},
    {"role": "human", "content": "介绍一下LoRA"},
    {"role": "gpt", "content": "LoRA是一种参数高效微调方法..."}
]}

数据清洗与预处理

import json
import re

def clean_text(text):
    """文本清洗函数"""
    # 去除多余空白
    text = re.sub(r'\s+', ' ', text).strip()
    # 去除特殊控制字符
    text = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', text)
    return text

def validate_sample(sample):
    """验证数据样本质量"""
    # 检查必要字段
    if not all(k in sample for k in ["instruction", "output"]):
        return False
    # 检查文本长度
    if len(sample["instruction"]) < 5 or len(sample["output"]) < 10:
        return False
    # 检查是否有过多重复内容
    if sample["instruction"] == sample["output"]:
        return False
    return True

# 加载并清洗数据
cleaned_data = []
with open("raw_data.jsonl", "r", encoding="utf-8") as f:
    for line in f:
        sample = json.loads(line)
        sample["instruction"] = clean_text(sample["instruction"])
        sample["output"] = clean_text(sample["output"])
        if "input" in sample:
            sample["input"] = clean_text(sample["input"])
        if validate_sample(sample):
            cleaned_data.append(sample)

print(f"原始样本: {len(raw_data)}, 清洗后: {len(cleaned_data)}")

# 保存清洗后的数据
with open("cleaned_data.jsonl", "w", encoding="utf-8") as f:
    for item in cleaned_data:
        f.write(json.dumps(item, ensure_ascii=False) + "\n")

关键训练指标

配置TensorBoard/W&B监控

# 方式1: 使用TensorBoard
training_args = TrainingArguments(
    output_dir="./output",
    report_to="tensorboard",           # 报告到TensorBoard
    logging_dir="./logs",              # 日志目录
    logging_steps=10,                  # 每10步记录一次
)

# 启动TensorBoard:
# tensorboard --logdir ./logs

# 方式2: 使用Weights & Biases
import wandb
wandb.init(project="llm-finetune", name="chatglm3-qlora")

training_args = TrainingArguments(
    output_dir="./output",
    report_to="wandb",                 # 报告到W&B
    logging_steps=10,
)

# 方式3: 自定义回调函数
from transformers import TrainerCallback

class LossCallback(TrainerCallback):
    def on_log(self, args, state, control, logs=None, **kwargs):
        if logs and "loss" in logs:
            if state.global_step % 100 == 0:
                print(f"Step {state.global_step}: loss={logs['loss']:.4f}")
            # 自定义告警
            if logs["loss"] > 10:
                print("WARNING: Loss异常高，检查数据和学习率!")

    def on_evaluate(self, args, state, control, metrics=None, **kwargs):
        if metrics and "eval_loss" in metrics:
            if metrics["eval_loss"] > 2.0:
                print("WARNING: 验证loss偏高，可能过拟合!")

# 在训练器中添加回调
trainer = SFTTrainer(
    model=model,
    callbacks=[LossCallback()],
    # ... 其他参数
)

LoRA权重合并

from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

# 1. 加载基座模型（FP16精度）
base_model = AutoModelForCausalLM.from_pretrained(
    "THUDM/chatglm3-6b",
    torch_dtype=torch.float16,
    device_map="auto",
    trust_remote_code=True,
)

# 2. 加载LoRA权重
model = PeftModel.from_pretrained(
    base_model,
    "./chatglm3-finetuned/final",
    torch_dtype=torch.float16,
)

# 3. 合并权重
model = model.merge_and_unload()

# 4. 保存合并后的完整模型
model.save_pretrained("./chatglm3-merged")
tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm3-6b")
tokenizer.save_pretrained("./chatglm3-merged")

print("模型合并完成！保存到 ./chatglm3-merged")

GGUF格式转换（用于llama.cpp部署）

# 步骤1: 安装转换工具
# pip install llama-cpp-python
# git clone https://github.com/ggerganov/llama.cpp
# cd llama.cpp && make

# 步骤2: 将模型转换为GGUF格式
# python convert_hf_to_gguf.py ./chatglm3-merged --outfile chatglm3-finetuned.gguf

# 步骤3: 量化为不同精度（可选）
# Q5_K_M 推荐：质量与大小的平衡
# ./quantize ./chatglm3-finetuned.gguf chatglm3-finetuned-q5_k_m.gguf q5_k_m

# Q4_K_M 推荐：更小体积
# ./quantize ./chatglm3-finetuned.gguf chatglm3-finetuned-q4_k_m.gguf q4_k_m

# 步骤4: 使用llama.cpp推理
# ./main -m chatglm3-finetuned-q5_k_m.gguf -n 256 --repeat_penalty 1.1 -p "你的提示词"

合并脚本（可直接运行）

#!/usr/bin/env python3
"""LoRA权重合并与导出脚本"""
import argparse
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel

def merge_and_export(base_model_name, lora_path, output_dir, export_gguf=False):
    """合并LoRA权重并导出模型"""
    print(f"加载基座模型: {base_model_name}")
    base_model = AutoModelForCausalLM.from_pretrained(
        base_model_name,
        torch_dtype=torch.float16,
        device_map="auto",
        trust_remote_code=True,
    )
    tokenizer = AutoTokenizer.from_pretrained(base_model_name, trust_remote_code=True)

    print(f"加载LoRA权重: {lora_path}")
    model = PeftModel.from_pretrained(base_model, lora_path)

    print("合并权重...")
    model = model.merge_and_unload()

    print(f"保存模型到: {output_dir}")
    model.save_pretrained(output_dir, safe_serialization=True)
    tokenizer.save_pretrained(output_dir)

    if export_gguf:
        print("导出GGUF格式...")
        import subprocess
        subprocess.run([
            "python", "convert_hf_to_gguf.py",
            output_dir,
            "--outfile", f"{output_dir}/model.gguf",
        ], check=True)

    print("完成！")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--base-model", default="THUDM/chatglm3-6b")
    parser.add_argument("--lora-path", default="./chatglm3-finetuned/final")
    parser.add_argument("--output-dir", default="./chatglm3-merged")
    parser.add_argument("--export-gguf", action="store_true")
    args = parser.parse_args()

    merge_and_export(args.base_model, args.lora_path, args.output_dir, args.export_gguf)

实战项目架构

完整训练脚本：train.py

#!/usr/bin/env python3
"""
完整微调脚本：QLoRA微调ChatGLM-6B
环境要求: pip install torch transformers peft trl datasets bitsandbytes accelerate
"""
import json
import torch
from datasets import Dataset
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig,
    TrainingArguments,
)
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
from trl import SFTTrainer

# ========== 配置 ==========
MODEL_NAME = "THUDM/chatglm3-6b"
DATA_FILE = "train_data.jsonl"
OUTPUT_DIR = "./output/chatglm3-qlora"
MAX_SEQ_LENGTH = 512
NUM_EPOCHS = 3
BATCH_SIZE = 2
GRAD_ACCUM = 8
LEARNING_RATE = 2e-4
LORA_R = 16
LORA_ALPHA = 32

# ========== 1. 加载数据 ==========
print("📂 加载训练数据...")
with open(DATA_FILE, "r", encoding="utf-8") as f:
    raw_data = [json.loads(line) for line in f]

def format_sample(sample):
    """格式化为ChatGLM的对话格式"""
    return f"[Round 1]\n问：{sample['instruction']}\n答：{sample['output']}"

dataset = Dataset.from_list([{"text": format_sample(d)} for d in raw_data])
dataset = dataset.train_test_split(test_size=0.1)
print(f"训练集: {len(dataset['train'])}, 验证集: {len(dataset['test'])}")

# ========== 2. 量化配置 ==========
print("⚙️ 配置4-bit量化...")
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
)

# ========== 3. 加载模型 ==========
print("🤖 加载模型...")
model = AutoModelForCausalLM.from_pretrained(
    MODEL_NAME,
    quantization_config=bnb_config,
    device_map="auto",
    trust_remote_code=True,
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
model = prepare_model_for_kbit_training(model)

# ========== 4. LoRA配置 ==========
print("🔧 配置LoRA...")
lora_config = LoraConfig(
    r=LORA_R,
    lora_alpha=LORA_ALPHA,
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=["query_key_value"],
)
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()

# ========== 5. 训练参数 ==========
print("📋 配置训练参数...")
training_args = TrainingArguments(
    output_dir=OUTPUT_DIR,
    num_train_epochs=NUM_EPOCHS,
    per_device_train_batch_size=BATCH_SIZE,
    gradient_accumulation_steps=GRAD_ACCUM,
    learning_rate=LEARNING_RATE,
    weight_decay=0.01,
    warmup_ratio=0.03,
    lr_scheduler_type="cosine",
    logging_steps=10,
    save_steps=100,
    save_total_limit=3,
    bf16=True,
    optim="paged_adamw_32bit",
    max_grad_norm=0.3,
    report_to="tensorboard",
    evaluation_strategy="steps",
    eval_steps=100,
    load_best_model_at_end=True,
)

# ========== 6. 开始训练 ==========
print("🚀 开始训练...")
trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=dataset["train"],
    eval_dataset=dataset["test"],
    args=training_args,
    max_seq_length=MAX_SEQ_LENGTH,
)

trainer.train()

# ========== 7. 保存 ==========
print("💾 保存模型...")
trainer.save_model(f"{OUTPUT_DIR}/final")
tokenizer.save_pretrained(f"{OUTPUT_DIR}/final")
print("✅ 训练完成！")