1
# 掩码语言模型（MLM）天然适合探测知识
2

3
# 将任务转换为完形填空
4
text = "Paris is the capital of [MASK]"
5
# BERT 输出: "France"
6

7
text = "The CEO of [MASK] is Satya Nadella"
8
# BERT 输出: "Microsoft"
9

10
text = "Einstein was born in [MASK]"
11
# BERT 输出: "Ulm"

1
class AutoPrompt:
2
    """AutoPrompt 核心"""
3

4
    def __init__(self, model, vocab, trigger_tokens):
5
        self.model = model
6
        self.vocab = vocab
7
        self.trigger_len = trigger_tokens
8

9
    def generate_prompt(self, template, masked_idx):
10
        """生成最优提示"""
11
        # 初始化触发词（使用 [MASK] 的表示）
12
        triggers = self.initialize_triggers(template)
13

14
        for step in range(max_steps):
15
            # 前向传播
16
            logits = self.model(input_ids=triggers)
17

18
            # 计算损失梯度
19
            loss = self.compute_loss(logits, target)
20
            loss.backward()
21

22
            # 梯度更新触发词
23
            with torch.no_grad():
24
                grad = triggers.grad
25
                # 选择梯度最大（损失减少最多）的 token
26
                best_tokens = self.top_k_tokens(grad, k=self.trigger_len)
27
                triggers = self.update_triggers(triggers, best_tokens)
28

29
        return triggers
30

31
    def top_k_tokens(self, grad, k):
32
        """选择梯度最大的 k 个 token"""
33
        # 梯度越大，移动到该 token 越能减少损失
34
        scores = -grad  # 负梯度，转为最大化
35
        return torch.topk(scores, k).indices

1
# 触发词：添加到输入中的可学习 token
2

3
# 原始问题
4
input_text = "The movie was amazing!"
5

6
# 添加 AutoPrompt 生成的触发词
7
prompt = "[T1] [T2] [T3] The movie was amazing! [SEP] It was [MASK]"
8

9
# [T1], [T2], [T3] 是自动学习的触发 token
10
# 可能学习到: "[positive] [really] [very]"

1
class AutomatedLabelSearch:
2
    """自动选择最合适的标签词"""
3

4
    def __init__(self, model, label_candidates):
5
        self.model = model
6
        self.labels = label_candidates
7

8
    def find_best_label(self, mask_position, true_label):
9
        """为 mask 位置找到最佳标签词"""
10
        best_label = None
11
        best_score = -float('inf')
12

13
        for label in self.labels:
14
            # 构建候选句子
15
            text = f"The movie was {label}!"
16
            input_ids = self.tokenize(text)
17

18
            # 计算似然
19
            prob = self.model(input_ids)[mask_position][label]
20

21
            if prob > best_score:
22
                best_score = prob
23
                best_label = label
24

25
        return best_label

1
# AutoPrompt 的通用框架
2

3
class PromptTemplate:
4
    """统一的提示模板"""
5

6
    # 情感分析
7
    sentiment_template = "[T1] {sentence} It was {mask}"
8

9
    # 自然语言推理
10
    nli_template = "{premise} [T1] {hypothesis} [MASK]"
11

12
    # 关系抽取
13
    re_template = "{subject} [T1] {relation} [MASK]"
14

15
# 自动化模板生成
16
prompt = AutoPrompt.generate(
17
    task="sentiment",
18
    model=bert_model,
19
    template=AutoPrompt.sentiment_template
20
)

1
# 关系抽取任务
2

3
# 原始输入
4
text = "[E1]Bill Gates[/E1] founded [E2]Microsoft[/E2]"
5

6
# AutoPrompt 生成的提示
7
prompt = "[T1] [T2] [T3] Bill Gates founded Microsoft [MASK]"
8

9
# AutoPrompt 结果：
10
# • 比监督模型更好的零样本性能
11
# • 自动发现有效的触发词

1
# 硬提示（Hard Prompt）- 离散 token
2
hard_prompt = "Translate to French: Hello, how are you?"
3

4
# 软提示（Soft Prompt）- 连续嵌入
5
soft_prompt = nn.Parameter(torch.randn(10, 768))  # 可学习
6

7
# AutoPrompt 结合两者：
8
# • 触发词是离散的（最终确定后）
9
# • 选择过程是可微的（梯度优化）