Merge pull request #143 from KenelmQLH/update_doc

[DOC] Update demo for knowledge prediction and paper segmentaion

Author: nnnyt

EduNLP/Pretrain/disenqnet_vec.py

@@ -161,7 +161,6 @@ def preprocess_dataset(pretrained_dir, disen_tokenizer, items, data_formation, t
     if not os.path.exists(concept_list_path):
         concepts = set()
         for data in items:
-            print(data)
             concept = data[data_formation["knowledge"]]
             for c in concept:
                 if c not in concepts:

EduNLP/Tokenizer/tokenizer.py

@@ -115,7 +115,7 @@ def _tokenize(self, item: Union[str, dict], key=lambda x: x, **kwargs):
 
 
 class PureTextTokenizer(Tokenizer):
-    def __init__(self, handle_figure_formula="skip", **kwargs):
+    def __init__(self, symbol="gmas", handle_figure_formula="skip", **kwargs):
         """
         Treat all elements in SIF item as prue text. Spectially, tokenize formulas as text.
 
@@ -184,13 +184,14 @@ def __init__(self, handle_figure_formula="skip", **kwargs):
             "text_params": text_params,
             "figure_params": kwargs.get("figure_params", None)
         }
+        self.symbol = symbol
 
     def __call__(self, items: Iterable, key=lambda x: x, **kwargs):
         for item in items:
             yield self._tokenize(item, key=key, **kwargs)
 
     def _tokenize(self, item: Union[str, dict], key=lambda x: x, **kwargs):
-        return tokenize(seg(key(item), symbol="gmas"), **self.tokenization_params, **kwargs).tokens
+        return tokenize(seg(key(item), symbol=self.symbol), **self.tokenization_params, **kwargs).tokens
 
 
 class AstFormulaTokenizer(Tokenizer):

examples/downstream/knowledge/konw_pred.ipynb

@@ -0,0 +1,215 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# 层级知识点预测"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import yaml\n",
+    "import tqdm\n",
+    "import torch\n",
+    "import numpy as np\n",
+    "from EduNLP.Pretrain import BertTokenizer\n",
+    "from EduNLP.ModelZoo.bert import BertForKnowledgePrediction\n",
+    "from EduNLP.Pretrain import finetune_bert_for_knowledge_prediction\n",
+    "from EduNLP.ModelZoo import load_items\n",
+    "\n",
+    "from utils import compute_perfs_per_layer, get_onehot_label_topk, metric, compute_perfs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_data = load_items(\"/path/to/data/train.jsonl\")\n",
+    "test_data = load_items(\"/path/to/data/test.jsonl\")\n",
+    "\n",
+    "pretrained_model_dir =\"/path/to/bert/checkpoint\"\n",
+    "checkpoint_dir = \"/path/to/knowledge_model/checkpoint\""
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 训练"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 以bert为例\n",
+    "data_params = {\n",
+    "    \"stem_key\": \"ques_content\",\n",
+    "    \"label_key\": \"know_list\"\n",
+    "}\n",
+    "train_params = {\n",
+    "    \"num_train_epochs\": 1,\n",
+    "    \"per_device_train_batch_size\": 2,\n",
+    "    \"per_device_eval_batch_size\": 2,\n",
+    "    \"no_cuda\": True,\n",
+    "}\n",
+    "model_params = {\n",
+    "    \"num_classes_list\": [10, 27, 963],\n",
+    "    \"num_total_classes\": 1000,\n",
+    "}\n",
+    " \n",
+    "\n",
+    "\"\"\"\n",
+    "数据格式：\n",
+    "{\n",
+    "    'ques_content': 'question...',\n",
+    "    'know_list': [lay_1_id, lay_2_id, lay_3_id]\n",
+    "}\n",
+    "\"\"\"\n",
+    "\n",
+    "# train without eval_items\n",
+    "finetune_bert_for_knowledge_prediction(\n",
+    "    train_data,\n",
+    "    checkpoint_dir,\n",
+    "    pretrained_model=pretrained_model_dir,\n",
+    "    train_params=train_params,\n",
+    "    data_params=data_params,\n",
+    "    model_params=model_params\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 加载模型和评估数据"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 针对多标签任务处理标签\n",
+    "class EvalDataset(torch.utils.data.Dataset):\n",
+    "    def __init__(self, data) -> None:\n",
+    "        self.data = data\n",
+    "        self.num_classes = model_params['num_classes_list']\n",
+    "        self.tokenizer = BertTokenizer.from_pretrained(pretrained_model_dir)\n",
+    "\n",
+    "    def __getitem__(self, idx):\n",
+    "        text, labels = self.data[idx][\"ques_content\"], self.data[idx][\"know_list\"]\n",
+    "        encodings = self.tokenizer(text, padding='max_length', truncation=True, return_tensors='pt')\n",
+    "        for k, v in encodings.items():\n",
+    "            encodings[k] = torch.squeeze(v, dim=0)\n",
+    "        one_hot_labels = [1. if idx in labels else 0. for idx in range(self.num_classes)]\n",
+    "        return encodings, torch.FloatTensor(one_hot_labels)\n",
+    "\n",
+    "    def __len__(self):\n",
+    "        return len(self.data)\n",
+    "\n",
+    "test_dataset  = EvalDataset(test_data)\n",
+    "eval_dataloader = EvalDataset(\n",
+    "    test_data,\n",
+    "    batch_size=1,\n",
+    "    shuffle=False,\n",
+    "    num_workers=4,\n",
+    ")\n",
+    "\n",
+    "model = BertForKnowledgePrediction.from_pretrained(checkpoint_dir)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 评估"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "device = \"cuda\" if not train_params[\"no_cuda\"] else \"cpu\"\n",
+    "\n",
+    "# 层级知识标签-配置信息\n",
+    "levels = len(model_params[\"num_classes_list\"])\n",
+    "classes_offset_list = [0, 10, 37]\n",
+    "classes_border_list = [[0, 9], [10, 36], [37, 1000]] # 层级id边界\n",
+    "hierarchy_dict = {} # child_know_id_to_parent_know_id\n",
+    "\n",
+    "# 评估top_k结果\n",
+    "top_k_list=[10, 20, 30]\n",
+    "\n",
+    "model.eval()\n",
+    "perfs_per_layer = [np.array([0 for _ in range(4)], dtype=np.int32) for _ in range(levels)]\n",
+    "total_perfs = np.array([0 for _ in range(4)], dtype=np.int32)\n",
+    "\n",
+    "k_total_perfs_list = [ np.array([0 for _ in range(4)], dtype=np.int32)for _ in range(len(top_k_list)) ]\n",
+    "for i, eval_batch in tqdm.tqdm(enumerate(eval_dataloader)):\n",
+    "    input_data, eval_batch_labels = eval_batch\n",
+    "    input_data = input_data.to(device)\n",
+    "    _, output_logits = model(**input_data)\n",
+    "\n",
+    "    local_perfs_per_layer, local_perfs = compute_perfs_per_layer(\n",
+    "        output_logits.cpu().detach().numpy(),\n",
+    "        eval_batch_labels.cpu().detach().numpy(),\n",
+    "        hierarchy_dict,\n",
+    "        classes_border_list,\n",
+    "        keep_consistency=True\n",
+    "    )\n",
+    "    perfs_per_layer = [perfs_per_layer[idx] + local_perfs_per_layer[idx] for idx in range(levels)]\n",
+    "    total_perfs += local_perfs\n",
+    "    \n",
+    "    # for recall@k\n",
+    "    for i_k, k in enumerate(top_k_list):\n",
+    "        pred_topk = get_onehot_label_topk(\n",
+    "            classes_border_list, classes_offset_list, scores_list=output_logits.cpu().detach().numpy(), top_num=k)\n",
+    "        flat_pred_topk = np.array([x[3] for x in pred_topk])\n",
+    "        k_total_perfs = compute_perfs(flat_pred_topk, eval_batch_labels.cpu().detach().numpy().tolist())\n",
+    "        k_total_perfs_list[i_k] += k_total_perfs\n",
+    "\n",
+    "# metric for overall\n",
+    "micro_precision, micro_recall, micro_f1, total_acc = metric(*total_perfs)\n",
+    "print(f\"Eval Results: Micro-Precision: {micro_precision:.4f}, \"\n",
+    "                + f\"Micro-Recall: {micro_recall:.4f}, Micro-F1: {micro_f1:.4f}, Acc: {total_acc:.4f}\")\n",
+    "\n",
+    "# metrics for per top_k\n",
+    "for i_k, k_total_perfs in enumerate(k_total_perfs_list):\n",
+    "    k = top_k_list[i_k]\n",
+    "    precision, recall, f1, acc = metric(*k_total_perfs)\n",
+    "    print(f\"TOPK={k}: Precision@{k}: {precision:.4f}, Recall@{k}: {recall:.4f}, F1@{k}: {f1:.4f}, Acc@{k}: {acc:.4f}\")\n",
+    "\n",
+    "# metrics for per layer\n",
+    "for layer_idx, perfs in enumerate(perfs_per_layer):\n",
+    "    precision, recall, f1, acc = metric(*perfs)\n",
+    "    print(f\"Layer {layer_idx + 1}: Precision: {precision:.4f}, Recall: {recall:.4f}, F1: {f1:.4f}, Acc: {acc:.4f}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

examples/downstream/knowledge/utils.py

@@ -0,0 +1,120 @@
+import numpy as np
+import torch
+import heapq
+from EduNLP.Pretrain import BertTokenizer
+
+
+def get_onehot_label_topk(classes_border_list, classes_offset_list, scores_list: np.ndarray, top_num=1):
+    """
+    Get the predicted labels based on the topK.
+
+    Args:
+        classes_border_list
+        classes_offset_list
+        scores_list: The all classes predicted scores provided by network
+        top_num: The max topK number (default: 5)
+    Returns:
+        predicted_onehot_labels: The predicted labels (onehot)
+    """
+    pred_onehot_labels = []
+    scores_list = np.ndarray.tolist(scores_list)
+    border, offset = classes_border_list, classes_offset_list
+    num_level = len(border)
+    for scores in scores_list:
+        onehot_labels_list = [0] * len(scores)
+        hlabels = {}
+        for level in range(num_level):
+            begin, end = border[level][0], border[level][1]
+            cur_scores = scores[begin: end + 1]
+            cur_offset = offset[level]
+            cur_onehot_labels_list = [0] * len(cur_scores)
+            # pred_onehot_scores[level].append(cur_scores)
+            max_num_index_list = list(map(cur_scores.index, heapq.nlargest(top_num, cur_scores)))
+            for i in max_num_index_list:
+                cur_onehot_labels_list[i] = 1
+                onehot_labels_list[i + cur_offset] = 1
+            hlabels[level] = cur_onehot_labels_list
+        # pred_onehot_scores[-1].append(scores)
+        hlabels[num_level] = onehot_labels_list
+        pred_onehot_labels.append(hlabels)
+    return pred_onehot_labels
+
+
+def compute_perfs(pred_labels: np.ndarray, true_labels: np.ndarray) -> tuple:
+    # TP: number of labels which is predicted as True and is actually True.
+    TP = np.sum(pred_labels * true_labels)
+    # FP: number of labels which is predicted as True and is actually False.
+    FP = np.sum(((pred_labels - true_labels) > 0).astype(np.int32))
+    # FN: number of labels which is predicted as False and is actually True.
+    FN = np.sum(((true_labels - pred_labels) > 0).astype(np.int32))
+    # FP: number of labels which is predicted as False and is actually False.
+    TN = np.sum(((pred_labels + true_labels) == 0).astype(np.int32))
+    return np.array([TP, FP, FN, TN], dtype=np.int32)
+
+
+def compute_perfs_per_layer(outputs: np.ndarray, true_labels: np.ndarray, hierarchy: dict, classes_border_list: list, keep_consistency: bool = True, threshold=0.5) -> tuple:
+    def _make_labels_consistent(input_labels: np.ndarray, hierarchy: dict):
+        input_labels = input_labels.astype(np.int32)
+        while len(hierarchy) > 0:
+            bottom_labels = set(hierarchy.keys()) - set(hierarchy.values())
+            for child in bottom_labels:
+                mask = (input_labels[:, child] == 1).astype(np.int32)
+                input_labels[:, hierarchy[child]] |= mask
+            for k in bottom_labels:
+                hierarchy.pop(k)
+        return input_labels
+
+    preds = []
+    for (start, end) in classes_border_list:
+        threshold_labels = (outputs[:, start:end + 1] >= threshold).astype(np.int32)
+        max_labels = (outputs[:, start:end + 1] == outputs[:, start:end + 1].max(axis=1)[:,None]).astype(np.int32)
+        preds.append(threshold_labels | max_labels)
+    pred_labels = np.concatenate(preds, axis=-1)
+    del preds
+    if keep_consistency:
+        pred_labels = _make_labels_consistent(pred_labels, hierarchy.copy())
+        true_labels = _make_labels_consistent(true_labels, hierarchy.copy())
+    # get perfs per layer
+    perfs_per_layer = []
+    for (start, end) in classes_border_list:
+        perfs_per_layer.append(compute_perfs(pred_labels[:, start:end + 1], true_labels[:, start:end + 1]))
+    total_perfs = compute_perfs(pred_labels, true_labels)
+    return perfs_per_layer, total_perfs
+
+
+def compute_topk_recall(topk_preds: list, true_labels: list) -> tuple:
+    rs = []
+    for pred, label in zip(topk_preds, true_labels):
+        _r = len(set(pred) & set(label)) / len(label)
+        rs.append(_r)
+    return np.mean(rs)
+
+
+def quantile(array: torch.Tensor, ratio: float):
+    """
+    get quantile of array
+    """
+    assert ratio >= 0 and ratio <= 1
+    assert len(array.shape) == 1
+    sorted_array = torch.sort(array, dim=-1, descending=True)[0]
+    index = min(int(len(array) * ratio + 0.5), len(array))
+    return sorted_array[index].item()
+
+
+def metric(TP, FP, FN, TN):
+    def _f1_score(precision, recall):
+        if precision + recall == 0:
+            return 0.
+        else:
+            return 2 * precision * recall / (precision + recall)
+    if TP + FP == 0:
+        precision = 0
+    else:
+        precision = TP / (TP + FP)
+    if TP + FN == 0:
+        recall = 0
+    else:
+        recall = TP / (TP + FN)
+    micro_f1 = _f1_score(precision, recall)
+    acc = (TP + TN) / (TP + FP + FN + TN)
+    return precision, recall, micro_f1, acc