1
# o1 推理时间与性能的关系
2
thinking_time_performance = {
3
    "低思考（<1K tokens）": {
4
        "AIME": "~30%",
5
        "适用": "简单问题、知识问答",
6
    },
7
    "中思考（1K-10K tokens）": {
8
        "AIME": "~60%",
9
        "适用": "中等难度推理",
10
    },
11
    "高思考（10K-100K tokens）": {
12
        "AIME": "~74%",
13
        "适用": "竞赛级数学、复杂编程",
14
    },
15
}

1
# RLVR 训练伪代码
2
def rlvr_training(model, problems, verifier, num_epochs=1000):
3
    optimizer = setup_optimizer(model)
4

5
    for epoch in range(num_epochs):
6
        batch = sample_problems(problems, batch_size=64)
7

8
        for problem in batch:
9
            # 1. 生成推理链（Chain-of-Thought）
10
            reasoning_chain = model.generate(
11
                prompt=problem.question,
12
                max_tokens=32000,  # 允许长推理链
13
                temperature=0.8,   # 适度随机
14
            )
15

16
            # 2. 提取最终答案
17
            answer = extract_answer(reasoning_chain)
18

19
            # 3. 验证答案正确性
20
            is_correct = verifier.verify(
21
                answer=answer,
22
                ground_truth=problem.answer,
23
                problem_type=problem.type,  # 数学/编程/逻辑
24
            )
25

26
            # 4. 计算奖励
27
            if is_correct:
28
                reward = 1.0
29
                # 额外奖励：更短的推理链获得略高奖励
30
                length_bonus = max(0, 1.0 - len(reasoning_chain) / 32000 * 0.1)
31
                reward += length_bonus
32
            else:
33
                reward = -0.1  # 轻微负奖励
34

35
            # 5. 策略梯度更新
36
            loss = -log_prob(model, reasoning_chain) * reward
37
            loss.backward()
38

39
        optimizer.step()
40
        optimizer.zero_grad()

1
# 不同类型的验证器
2
class MathVerifier:
3
    """数学问题验证器：比较最终答案"""
4
    def verify(self, answer, ground_truth):
5
        # 标准化数学表达式后比较
6
        normalized_answer = self.normalize(answer)
7
        normalized_truth = self.normalize(ground_truth)
8
        return math.isclose(normalized_answer, normalized_truth, rel_tol=1e-6)
9

10
class CodeVerifier:
11
    """编程问题验证器：运行测试用例"""
12
    def verify(self, code, test_cases):
13
        for test_input, expected_output in test_cases:
14
            actual_output = self.execute(code, test_input)
15
            if actual_output != expected_output:
16
                return False
17
        return True
18

19
class LogicVerifier:
20
    """逻辑推理验证器：检查证明步骤"""
21
    def verify(self, proof_steps, axioms):
22
        for step in proof_steps:
23
            if not self.is_valid_step(step, axioms):
24
                return False
25
        return True

1
# o1 的推理链生成与选择
2
def o1_reasoning(model, problem, num_chains=8, verifier=None):
3
    candidates = []
4

5
    for i in range(num_chains):
6
        # 生成推理链（温度较高，增加多样性）
7
        chain = model.generate(
8
            prompt=problem,
9
            max_tokens=32768,
10
            temperature=0.8,
11
        )
12

13
        answer = extract_final_answer(chain)
14

15
        # 验证答案
16
        if verifier:
17
            is_correct = verifier.verify(answer, problem.answer)
18
            candidates.append({
19
                "chain": chain,
20
                "answer": answer,
21
                "verified": is_correct,
22
                "length": len(chain),
23
            })
24

25
    # 选择策略
26
    # 1. 优先选择验证通过的
27
    verified = [c for c in candidates if c["verified"]]
28
    if verified:
29
        # 在验证通过的候选中选最短的（奥卡姆剃刀）
30
        best = min(verified, key=lambda x: x["length"])
31
    else:
32
        # 没有验证通过的，用多数投票
33
        answer_counts = Counter(c["answer"] for c in candidates)
34
        majority_answer = answer_counts.most_common(1)[0][0]
35
        best = [c for c in candidates if c["answer"] == majority_answer][0]
36

37
    return best["answer"], best["chain"]

1
让我分析这道数学题...
2

3
首先，我需要理解题目在问什么。给定 f(x) = x³ - 3x + 1，求在区间 [-2, 2] 上的极值。
4

5
步骤1：求导数 f'(x) = 3x² - 3
6
步骤2：令 f'(x) = 0，解得 x = ±1
7
步骤3：判断极值类型
8
  - f''(x) = 6x
9
  - f''(1) = 6 > 0 → x = 1 是极小值点
10
  - f''(-1) = -6 < 0 → x = -1 是极大值点
11

12
等等，让我再确认一下边界值...
13
f(-2) = -8 + 6 + 1 = -1
14
f(2) = 8 - 6 + 1 = 3
15
f(-1) = -1 + 3 + 1 = 3  → 极大值
16
f(1) = 1 - 3 + 1 = -1   → 极小值
17

18
所以极大值是 3（在 x = -1 和 x = 2 处都取到），极小值是 -1。

1
# o3 的 Extended Thinking 配置
2
def o3_inference(problem, thinking_budget="auto"):
3
    """o3 推理配置"""
4
    configs = {
5
        "low": {
6
            "thinking_tokens": 2000,
7
            "适合": "简单问题、快速回答",
8
        },
9
        "medium": {
10
            "thinking_tokens": 10000,
11
            "适合": "中等推理、日常编程",
12
        },
13
        "high": {
14
            "thinking_tokens": 100000,
15
            "适合": "竞赛数学、复杂调试",
16
        },
17
        "auto": {
18
            "thinking_tokens": "模型自主决定",
19
            "适合": "默认模式",
20
        },
21
    }
22

23
    config = configs[thinking_budget]
24
    return model.generate(
25
        problem,
26
        max_thinking_tokens=config["thinking_tokens"],
27
    )

1
# 推理成本对比
2
cost_comparison = {
3
    "GPT-4o（直接回答）": {
4
        "输入": "$2.50/1M tokens",
5
        "输出": "$10.00/1M tokens",
6
        "平均推理": "~500 tokens",
7
        "平均成本": "~$0.005/问题",
8
    },
9
    "o1（推理后回答）": {
10
        "输入": "$15.00/1M tokens",
11
        "推理": "$60.00/1M tokens",
12
        "平均推理": "~5000 tokens 思考 + ~500 tokens 输出",
13
        "平均成本": "~$0.35/问题",
14
    },
15
    "o3（深度推理）": {
16
        "推理": "按思考量计费",
17
        "高思考模式": "~$3.50/问题",
18
        "低思考模式": "~$0.15/问题",
19
    },
20
}