1
# RAG: 检索增强生成
2
class RAG:
3
    def answer(self, question):
4
        docs = retriever.search(question)  # 检索
5
        context = concatenate(docs)
6
        answer = generator.generate(question, context)  # 生成
7
        return answer
8
    # 模型不控制检索过程
9

10
# WebGPT: 模型主动搜索
11
class WebGPT:
12
    def answer(self, question):
13
        while not done:
14
            action = model.decide_action()  # 模型决定动作
15
            observation = browser.execute(action)  # 执行
16
            model.update(observation)  # 观察结果
17
        return model.generate_answer()
18
    # 模型完全控制搜索过程

1
class WebBrowser:
2
    """WebGPT 的浏览器环境"""
3

4
    def __init__(self):
5
        self.current_page = None
6
        self.history = []
7

8
    def search(self, query):
9
        """执行搜索"""
10
        results = search_engine(query)
11
        self.history.append({
12
            "action": "search",
13
            "query": query,
14
            "results": results
15
        })
16
        return results
17

18
    def click(self, link_id):
19
        """点击链接"""
20
        new_page = navigate(link_id)
21
        self.current_page = new_page
22
        self.history.append({
23
            "action": "click",
24
            "link_id": link_id
25
        })
26
        return new_page.content
27

28
    def scroll(self, amount):
29
        """滚动页面"""
30
        content = self.current_page.scroll(amount)
31
        self.history.append({
32
            "action": "scroll",
33
            "amount": amount
34
        })
35
        return content
36

37
    def find_text(self, text):
38
        """查找文本"""
39
        positions = self.current_page.find(text)
40
        self.history.append({
41
            "action": "find",
42
            "text": text
43
        })
44
        return positions

1
class BehaviorClone:
2
    """行为克隆：用人类演示训练"""
3

4
    def __init__(self, model):
5
        self.model = model
6

7
    def train(self, demonstrations):
8
        """
9
        demonstrations: 人类使用浏览器的演示数据
10
        每个演示包含：(问题, 动作序列, 最终答案)
11
        """
12
        total_loss = 0
13

14
        for question, actions, answer in demonstrations:
15
            # 对每个动作计算损失
16
            context = question
17
            for action in actions:
18
                # 模型预测动作
19
                logits = self.model(context, action)
20
                loss = F.cross_entropy(logits, action)
21
                total_loss += loss
22

23
                # 更新上下文
24
                context += action
25

26
        # 反向传播
27
        total_loss.backward()

1
class RLHFTraining:
2
    """人类反馈强化学习"""
3

4
    def train_reward_model(self, trajectories):
5
        """
6
        训练奖励模型评估回答质量
7
        """
8
        for traj in trajectories:
9
            # traj: (问题, 浏览历史, 回答)
10
            human_rating = get_human_rating(traj)
11

12
            # 奖励模型学习
13
            reward = self.reward_model(traj)
14
            loss = F.mse_loss(reward, human_rating)
15
            loss.backward()
16

17
    def optimize_policy(self):
18
        """
19
        使用 PPO 优化策略
20
        """
21
        for _ in range(num_steps):
22
            # 采样新轨迹
23
            trajs = self.collect_trajectories()
24

25
            # 计算奖励
26
            rewards = [self.reward_model(t) for t in trajs]
27

28
            # PPO 更新
29
            self.policy.update(trajs, rewards)

1
# WebGPT 的关键：收集引用支持答案
2

3
class CitationCollector:
4
    """收集答案的引用来源"""
5

6
    def __init__(self, browser):
7
        self.browser = browser
8

9
    def answer_with_citations(self, question, browsing_history):
10
        """
11
        生成带引用的答案
12
        """
13
        answer = generate_answer(question)
14

15
        # 从浏览历史中提取引用
16
        citations = []
17
        for action in browsing_history:
18
            if action["action"] == "click":
19
                page_content = action["result"]
20
                if relevant_to_answer(page_content, answer):
21
                    citations.append({
22
                        "url": action["url"],
23
                        "text": extract_relevant_text(page_content)
24
                    })
25

26
        return {
27
            "answer": answer,
28
            "citations": citations
29
        }

1
# 人类评估：WebGPT vs Reddit 高赞回答
2

3
evaluation_results = {
4
    "WebGPT_preferred": 0.69,  # 69% 情况下人类更喜欢 WebGPT
5
    "Reddit_preferred": 0.31,  # 31% 情况下更喜欢 Reddit
6
    "tied": 0.0
7
}

1
# WebGPT 需要模型进行长程规划
2

3
class LongTermPlanning:
4
    """WebGPT 的规划挑战"""
5

6
    # 模型需要：
7
    # 1. 理解问题核心
8
    # 2. 设计搜索策略
9
    # 3. 评估信息价值
10
    # 4. 整合多源信息
11
    # 5. 生成连贯答案
12

13
    # 这比单次 RAG 检索复杂得多

1
# WebGPT 需要处理搜索失败
2

3
class ErrorRecovery:
4
    """搜索失败时的处理"""
5

6
    def handle_empty_results(self):
7
        """搜索无结果时"""
8
        # 尝试更通用的查询
9
        # 换用其他搜索引擎
10
        # 改变搜索策略
11

12
    def handle_irrelevant_page(self):
13
        """页面不相关时"""
14
        # 返回搜索结果
15
        # 选择其他链接
16

17
    def handle_trap(self):
18
        """陷入死循环时"""
19
        # 强制结束
20
        # 尝试其他方法