1
# GPT-3 的代码能力测试
2

3
# 输入
4
prompt = "def quicksort(arr):"
5

6
# GPT-3 输出（有限）：
7
# 只能生成简单的函数框架
8

9
# Codex（代码微调后）：
10
# 能生成完整、可运行的排序实现

1
# Codex 训练数据来源
2

3
training_data = {
4
    "source": "GitHub 公开代码仓库",
5
    "filter": "MIT / Apache / GPL 等开源许可",
6
    "language": "Python 为主",
7
    "size": "100B tokens",
8
    "quality_filter": "去除自动生成、低质量代码"
9
}

1
class Codex(nn.Module):
2
    """Codex 模型"""
3

4
    def __init__(self, vocab_size=50257, d_model=12288, n_layers=96):
5
        super().__init__()
6
        # 与 GPT-3 相同的架构
7
        self.transformer = GPT3Transformer(vocab_size, d_model, n_layers)
8

9
    def generate_code(self, prompt, max_tokens=100):
10
        """代码生成"""
11
        output = self.transformer.generate(
12
            input_ids=tokenize(prompt),
13
            max_new_tokens=max_tokens,
14
            temperature=0.0  # 代码通常需要确定性输出
15
        )
16
        return decode(output)

1
class HumanEvalDataset:
2
    """HumanEval 数据集"""
3

4
    def __init__(self):
5
        self.tasks = []
6

7
    def add_task(self, prompt, canonical_solution, test_cases):
8
        """添加一个编程任务"""
9
        self.tasks.append({
10
            "prompt": prompt,  # 自然语言描述
11
            "solution": canonical_solution,  # 参考解
12
            "tests": test_cases  # 功能测试
13
        })
14

15
    def evaluate(self, model_response):
16
        """验证功能正确性"""
17
        # 运行测试用例
18
        passed = run_tests(model_response, self.tasks[0]["tests"])
19
        return passed

1
# HumanEval 任务示例
2

3
task = {
4
    "prompt": '''
5
def has_close_elements(numbers: List[float], threshold: float) -> bool:
6
    """Check if any two numbers in a list are close to each other."""
7
''',
8
    "signature": "def has_close_elements(numbers: List[float], threshold: float) -> bool:",
9
    "test_cases": [
10
        "assert has_close_elements([1.0, 2.0, 3.0], 0.5) == False",
11
        "assert has_close_elements([1.0, 2.0, 3.0], 0.8) == True",
12
    ]
13
}
14

15
# Codex 需要生成完整的函数实现
16
# 通过所有测试用例才算正确

1
# 多次采样大幅提升成功率
2

3
def generate_with_sampling(model, prompt, n_samples=100):
4
    """生成多个样本并返回通过测试的"""
5
    samples = []
6

7
    for _ in range(n_samples):
8
        code = model.generate(prompt, temperature=0.4)
9
        samples.append(code)
10

11
    # 逐个测试
12
    for code in samples:
13
        if run_tests(code):
14
            return code  # 返回第一个通过的
15

16
    return None  # 所有都失败

1
# 1. 语法错误
2
code = "def foo(x]:"  # 括号不匹配
3

4
# 2. 逻辑错误
5
code = "return x / 2"  # 应该是 x % 2
6

7
# 3. 边界条件
8
code = "return arr[0]"  # arr 可能为空
9

10
# 4. 依赖缺失
11
code = "from typing import List"  # 可能没有导入

1
# 理解复杂的文档字符串
2

3
docstring = '''
4
Complete the function to solve the following problem:
5

6
You are given two strings `needle` and `haystack`.
7
Return the index of the first occurrence of `needle` in `haystack`,
8
or -1 if `needle` is not part of `haystack`.
9

10
Example:
11
>>> str_str("sadbutsad", "sad")
12
0
13
>>> str_str("leetcode", "leeto")
14
-1
15
'''
16

17
# Codex 需要：
18
# 1. 理解函数签名
19
# 2. 理解文档字符串描述
20
# 3. 实现正确的算法