결과물한눈에보기_korquad.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "a5792089-ea2b-434a-afe2-7aa50bb89c3e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# %pip install tabulate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "c3669d41-ee41-4961-82f8-fb778b12a541",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "7e9d99ca-1175-4d75-b9b2-04f119960ad2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "25a7e405-1368-46aa-ba51-0bc7a98962df",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "852c83ff-f39c-41a4-b6f9-05f22ad06e9b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# results/korean_origin_bench/20b/00_shot.json\n",
    "def _get_metric_name(v, display_all=True):\n",
    "    \n",
    "    metrics = ['f1', 'macro_f1', 'acc_norm', 'acc', 'exact_match']\n",
    "    for m in metrics:\n",
    "        if v.get(m):\n",
    "            return {\n",
    "                'metric': m,\n",
    "                'value': v[m],\n",
    "            }\n",
    "\n",
    "def get_df_klue(path, model_name=''):\n",
    "    data = []\n",
    "    for i in ['0', '5', '10', '50']:\n",
    "        shot = f'{path}/{i}_shot.json'\n",
    "        try:\n",
    "            data.append(\n",
    "                {\n",
    "                    **{\n",
    "                        f\"{k} (f1)\": v['f1']\n",
    "                        for k, v in json.load(open(shot))['results'].items()\n",
    "                    },\n",
    "                    **{\n",
    "                        f\"{k} (exact_match)\": v['exact_match']\n",
    "                    for k, v in json.load(open(shot))['results'].items()\n",
    "                    }\n",
    "                }\n",
    "            )\n",
    "        except FileNotFoundError:\n",
    "            pass\n",
    "    df = pd.DataFrame(data, index=[0, 5, 10, 50][:len(data)]).T\n",
    "    print(df.to_markdown())\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "5d226502-7541-427b-8b36-bec5489ca296",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['results/korquad/beomi/llama-2-ko-7b',\n",
       " 'results/korquad/beomi/llama-2-ko-7b-emb-dev',\n",
       " 'results/korquad/beomi/llama-2-koen-13b',\n",
       " 'results/korquad/beomi/open-llama-2-ko-7b']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "various_models = sorted(glob('results/korquad/*/*'))\n",
    "various_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "a93c00e7-941d-4149-9efe-bd22e66a039e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "results/korquad/beomi/llama-2-ko-7b\n",
      "|                       |       0 |       5 |\n",
      "|:----------------------|--------:|--------:|\n",
      "| korquad (f1)          | 66.6277 | 22.101  |\n",
      "| korquad (exact_match) | 57.4299 | 19.9861 |\n",
      "\n",
      "results/korquad/beomi/llama-2-ko-7b-emb-dev\n",
      "|                       |       0 |\n",
      "|:----------------------|--------:|\n",
      "| korquad (f1)          | 68.7333 |\n",
      "| korquad (exact_match) | 61.2574 |\n",
      "\n",
      "results/korquad/beomi/llama-2-koen-13b\n",
      "\n",
      "\n",
      "results/korquad/beomi/open-llama-2-ko-7b\n",
      "|                       |       0 |       5 |\n",
      "|:----------------------|--------:|--------:|\n",
      "| korquad (f1)          | 66.0021 | 22.3634 |\n",
      "| korquad (exact_match) | 58.1053 | 20.4711 |\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model in various_models:\n",
    "    print(model)\n",
    "    get_df_klue(model)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "071411b4-1497-48d0-8440-2abda75dd499",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# results/all/beomi/llama-2-ko-7b-emb-dev\n",
    "# |                                  |        0 |        5 |       10 |       50 |\n",
    "# |:---------------------------------|---------:|---------:|---------:|---------:|\n",
    "# | kobest_boolq (macro_f1)          | 0.337452 | 0.570559 | 0.585608 | 0.567486 |\n",
    "# | kobest_copa (macro_f1)           | 0.665517 | 0.69096  | 0.686659 | 0.698672 |\n",
    "# | kobest_hellaswag (macro_f1)      | 0.383174 | 0.379488 | 0.380637 | 0.383733 |\n",
    "# | kobest_sentineg (macro_f1)       | 0.367966 | 0.91937  | 0.96473  | 0.947103 |\n",
    "# | kohatespeech (macro_f1)          | 0.300627 | 0.362646 | 0.364923 | 0.287482 |\n",
    "# | kohatespeech_apeach (macro_f1)   | 0.337667 | 0.486543 | 0.452716 | 0.335808 |\n",
    "# | kohatespeech_gen_bias (macro_f1) | 0.124535 | 0.489203 | 0.461098 | 0.461714 |\n",
    "# | korunsmile (f1)                  | 0.35822  | 0.367208 | 0.381937 | 0.376735 |\n",
    "# | nsmc (acc)                       | 0.50714  | 0.80386  | 0.83964  | 0.86138  |\n",
    "# | pawsx_ko (acc)                   | 0.539    | 0.4915   | 0.5225   | 0.521    |\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "821b7c25-8f9e-4598-be7c-66df019331ec",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3048c2e-d84e-4f00-ba8b-ef5e7ccb8b70",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}