collect_all_benchmarks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import pandas as pd\n",
    "import ace_tools_open as tools;\n",
    "\n",
    "# Path to the \"results\" folder\n",
    "RESULTS_DIR = \"TSLibrary/results\"\n",
    "\n",
    "# Regular expression to extract accuracy\n",
    "accuracy_pattern = re.compile(r\"accuracy:([\\d\\.]+)\")\n",
    "\n",
    "# List to store extracted data\n",
    "data = []\n",
    "\n",
    "# Iterate over folders in the \"results\" directory\n",
    "for folder in os.listdir(RESULTS_DIR):\n",
    "    folder_path = os.path.join(RESULTS_DIR, folder)\n",
    "    if os.path.isdir(folder_path):\n",
    "        result_file = os.path.join(folder_path, \"result_classification.txt\")\n",
    "        \n",
    "        # Extract dataset and model name from folder name\n",
    "        parts = folder.split(\"_\")\n",
    "        if len(parts) < 3:\n",
    "            continue  # Skip if folder name format is incorrect\n",
    "        dataset_name = parts[1]\n",
    "        model_name = parts[2]\n",
    "        \n",
    "        # Check if result file exists\n",
    "        if os.path.isfile(result_file):\n",
    "            with open(result_file, \"r\") as f:\n",
    "                content = f.read()\n",
    "                \n",
    "                # Search for accuracy\n",
    "                match = accuracy_pattern.search(content)\n",
    "                if match:\n",
    "                    accuracy = round(100*float(match.group(1)), 2)\n",
    "                    data.append((dataset_name, model_name, accuracy))\n",
    "\n",
    "\n",
    "df_tmp = pd.DataFrame(data, columns=[\"Dataset\", \"Model\", \"Accuracy\"])\n",
    "df_attention = df_tmp.pivot(index=\"Dataset\", columns=\"Model\", values=\"Accuracy\").reset_index()\n",
    "df_attention.columns.name = None\n",
    "df_non_attention = pd.read_csv(\"out/classification_results.csv\")\n",
    "\n",
    "df = pd.merge(df_non_attention, df_attention, on=\"Dataset\", how=\"outer\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display table\n",
    "datasets = [\n",
    "    \"PDSetting1\",\n",
    "    \"PDSetting2\",\n",
    "    \"PronunciationAudio\",\n",
    "    \"ECGFiveDays\",\n",
    "    \"FreezerSmallTrain\",\n",
    "    \"HouseTwenty\",\n",
    "    \"InsectEPGRegularTrain\",\n",
    "    \"ItalyPowerDemand\",\n",
    "    \"Lightning7\",\n",
    "    \"MoteStrain\",\n",
    "    \"PowerCons\",\n",
    "    \"SonyAIBORobotSurface2\",\n",
    "    \"UWaveGestureLibraryAll\",\n",
    "]\n",
    "\n",
    "\n",
    "models = [\n",
    "    \"DTW\", # (2011)\n",
    "    \"TSF\", # (2013)\n",
    "    \"BOSS\", # (2015)\n",
    "    \"BOSS-E\", # (2015)\n",
    "    \"RISE\", # (2016)\n",
    "    \"Shapelet\", # (2017)\n",
    "    \"catch22\", # (2019)\n",
    "    \"SVC\", # (2019)\n",
    "    \"LSTM-FCN\", # (2019)\n",
    "    \"Teaser\", # (2020)\n",
    "    \"Rocket\", # (2020)\n",
    "    \"Hive-Cote 2\", # (2021)\n",
    "    \"Informer\", # (2021)\n",
    "    \"Autoformer\", # (2021)\n",
    "    \"FEDformer\", # (2022)\n",
    "    \"ETSformer\", # (2022)\n",
    "    \"LightTS\", # (2022)\n",
    "    \"PatchTST\", # (2023)\n",
    "    \"Crossformer\", # (2023)\n",
    "    \"DLinear\", # (2023)\n",
    "    \"TimesNet\", # (2023)\n",
    "    \"iTransformer\", # (2024)\n",
    "    \"Mamba\", # (2024)\n",
    "    \"Motion Code\", # (2024)\n",
    "]\n",
    "df = df[['Dataset'] + models]\n",
    "df = df.set_index(\"Dataset\").loc[datasets].reset_index() \n",
    "df[\"Dataset\"] = pd.Categorical(df[\"Dataset\"], categories=datasets, ordered=True)\n",
    "df = df.sort_values(\"Dataset\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "# Highlight dataframe with best and second-best accuracy\n",
    "def highlight_best_html_rounded(data):\n",
    "    styled_data = data.copy()\n",
    "\n",
    "    for index, row in data.iterrows():\n",
    "        # Convert row values, keeping \"Error\" as NaN to ignore in ranking\n",
    "        row_values = row[1:].replace(\"Error\", np.nan).astype(float).round(2)\n",
    "\n",
    "        # Get unique sorted values (descending order), ignoring NaN\n",
    "        sorted_values = np.sort(row_values.dropna().unique())[::-1]\n",
    "\n",
    "        # Identify max and second-best values, avoiding NaN influence\n",
    "        max_val = sorted_values[0] if len(sorted_values) > 0 else None\n",
    "        second_best_val = sorted_values[1] if len(sorted_values) > 1 else None\n",
    "        #third_best_val = sorted_values[2] if len(sorted_values) > 2 else None\n",
    "\n",
    "        for col in row.index[1:]:\n",
    "            val = row[col]\n",
    "            if val == \"Error\":\n",
    "                continue  # Skip highlighting for \"Error\" values\n",
    "\n",
    "            val = round(float(val), 2) \n",
    "\n",
    "            if val == max_val:\n",
    "                styled_data.at[index, col] = f'<span style=\"background-color: red; color: white;\">{val}</span>'\n",
    "            elif val == second_best_val:\n",
    "                styled_data.at[index, col] = f'<span style=\"background-color: blue; color: white;\">{val}</span>'\n",
    "            #elif val == third_best_val:\n",
    "            #    styled_data.at[index, col] = f'<span style=\"background-color: green; color: white;\">{val}</span>'\n",
    "            else:\n",
    "                styled_data.at[index, col] = f\"{val}\"  # Keep values rounded\n",
    "\n",
    "    return styled_data\n",
    "\n",
    "# Apply the improved highlighting function with rounding\n",
    "highlighted_df = highlight_best_html_rounded(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification Results\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table id=\"itables_ad849315_adec_4f94_956f_22defa2cb494\" class=\"display nowrap\" data-quarto-disable-processing=\"true\" style=\"table-layout:auto;width:auto;margin:auto;caption-side:bottom\">\n",
       "<thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      \n",
       "      <th>Dataset</th>\n",
       "      <th>DTW</th>\n",
       "      <th>TSF</th>\n",
       "      <th>BOSS</th>\n",
       "      <th>BOSS-E</th>\n",
       "      <th>RISE</th>\n",
       "      <th>Shapelet</th>\n",
       "      <th>catch22</th>\n",
       "      <th>SVC</th>\n",
       "      <th>LSTM-FCN</th>\n",
       "      <th>Teaser</th>\n",
       "      <th>Rocket</th>\n",
       "      <th>Hive-Cote 2</th>\n",
       "      <th>Informer</th>\n",
       "      <th>Autoformer</th>\n",
       "      <th>FEDformer</th>\n",
       "      <th>ETSformer</th>\n",
       "      <th>LightTS</th>\n",
       "      <th>PatchTST</th>\n",
       "      <th>Crossformer</th>\n",
       "      <th>DLinear</th>\n",
       "      <th>TimesNet</th>\n",
       "      <th>iTransformer</th>\n",
       "      <th>Mamba</th>\n",
       "      <th>Motion Code</th>\n",
       "    </tr>\n",
       "  </thead><tbody><tr>\n",
       "<td style=\"vertical-align:middle; text-align:left\">\n",
       "<a href=https://mwouts.github.io/itables/><svg class=\"main-svg\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"\n",
       "width=\"64\" viewBox=\"0 0 500 400\" style=\"font-family: 'Droid Sans', sans-serif;\">\n",
       "    <g style=\"fill:#d9d7fc\">\n",
       "        <path d=\"M100,400H500V357H100Z\" />\n",
       "        <path d=\"M100,300H400V257H100Z\" />\n",
       "        <path d=\"M0,200H400V157H0Z\" />\n",
       "        <path d=\"M100,100H500V57H100Z\" />\n",
       "        <path d=\"M100,350H500V307H100Z\" />\n",
       "        <path d=\"M100,250H400V207H100Z\" />\n",
       "        <path d=\"M0,150H400V107H0Z\" />\n",
       "        <path d=\"M100,50H500V7H100Z\" />\n",
       "    </g>\n",
       "    <g style=\"fill:#1a1366;stroke:#1a1366;\">\n",
       "   <rect x=\"100\" y=\"7\" width=\"400\" height=\"43\">\n",
       "    <animate\n",
       "      attributeName=\"width\"\n",
       "      values=\"0;400;0\"\n",
       "      dur=\"5s\"\n",
       "      repeatCount=\"indefinite\" />\n",
       "      <animate\n",
       "      attributeName=\"x\"\n",
       "      values=\"100;100;500\"\n",
       "      dur=\"5s\"\n",
       "      repeatCount=\"indefinite\" />\n",
       "  </rect>\n",
       "        <rect x=\"0\" y=\"107\" width=\"400\" height=\"43\">\n",
       "    <animate\n",
       "      attributeName=\"width\"\n",
       "      values=\"0;400;0\"\n",
       "      dur=\"3.5s\"\n",
       "      repeatCount=\"indefinite\" />\n",
       "    <animate\n",
       "      attributeName=\"x\"\n",
       "      values=\"0;0;400\"\n",
       "      dur=\"3.5s\"\n",
       "      repeatCount=\"indefinite\" />\n",
       "  </rect>\n",
       "        <rect x=\"100\" y=\"207\" width=\"300\" height=\"43\">\n",
       "    <animate\n",
       "      attributeName=\"width\"\n",
       "      values=\"0;300;0\"\n",
       "      dur=\"3s\"\n",
       "      repeatCount=\"indefinite\" />\n",
       "    <animate\n",
       "      attributeName=\"x\"\n",
       "      values=\"100;100;400\"\n",
       "      dur=\"3s\"\n",
       "      repeatCount=\"indefinite\" />\n",
       "  </rect>\n",
       "        <rect x=\"100\" y=\"307\" width=\"400\" height=\"43\">\n",
       "    <animate\n",
       "      attributeName=\"width\"\n",
       "      values=\"0;400;0\"\n",
       "      dur=\"4s\"\n",
       "      repeatCount=\"indefinite\" />\n",
       "      <animate\n",
       "      attributeName=\"x\"\n",
       "      values=\"100;100;500\"\n",
       "      dur=\"4s\"\n",
       "      repeatCount=\"indefinite\" />\n",
       "  </rect>\n",
       "        <g style=\"fill:transparent;stroke-width:8; stroke-linejoin:round\" rx=\"5\">\n",
       "            <g transform=\"translate(45 50) rotate(-45)\">\n",
       "                <circle r=\"33\" cx=\"0\" cy=\"0\" />\n",
       "                <rect x=\"-8\" y=\"32\" width=\"16\" height=\"30\" />\n",
       "            </g>\n",
       "\n",
       "            <g transform=\"translate(450 152)\">\n",
       "                <polyline points=\"-15,-20 -35,-20 -35,40 25,40 25,20\" />\n",
       "                <rect x=\"-15\" y=\"-40\" width=\"60\" height=\"60\" />\n",
       "            </g>\n",
       "\n",
       "            <g transform=\"translate(50 352)\">\n",
       "                <polygon points=\"-35,-5 0,-40 35,-5\" />\n",
       "                <polygon points=\"-35,10 0,45 35,10\" />\n",
       "            </g>\n",
       "\n",
       "            <g transform=\"translate(75 250)\">\n",
       "                <polyline points=\"-30,30 -60,0 -30,-30\" />\n",
       "                <polyline points=\"0,30 -30,0 0,-30\" />\n",
       "            </g>\n",
       "\n",
       "            <g transform=\"translate(425 250) rotate(180)\">\n",
       "                <polyline points=\"-30,30 -60,0 -30,-30\" />\n",
       "                <polyline points=\"0,30 -30,0 0,-30\" />\n",
       "            </g>\n",
       "        </g>\n",
       "    </g>\n",
       "</svg>\n",
       "</a>\n",
       "Loading ITables v2.2.5 from the internet...\n",
       "(need <a href=https://mwouts.github.io/itables/troubleshooting.html>help</a>?)</td>\n",
       "</tr></tbody>\n",
       "</table>\n",
       "<link href=\"https://www.unpkg.com/dt_for_itables@2.0.13/dt_bundle.css\" rel=\"stylesheet\">\n",
       "<script type=\"module\">\n",
       "    import {DataTable, jQuery as $} from 'https://www.unpkg.com/dt_for_itables@2.0.13/dt_bundle.js';\n",
       "\n",
       "    document.querySelectorAll(\"#itables_ad849315_adec_4f94_956f_22defa2cb494:not(.dataTable)\").forEach(table => {\n",
       "        if (!(table instanceof HTMLTableElement))\n",
       "            return;\n",
       "\n",
       "        // Define the table data\n",
       "        const data = [[\"PDSetting1\", \"63.35\", \"63.98\", \"61.8\", \"65.53\", \"<span style=\\\"background-color: blue; color: white;\\\">70.81</span>\", \"52.8\", \"68.94\", \"63.96\", \"43.48\", \"59.94\", \"61.49\", \"59.63\", \"67.7\", \"56.21\", \"66.46\", \"59.63\", \"66.15\", \"58.07\", \"70.19\", \"66.77\", \"68.63\", \"69.88\", \"68.94\", \"<span style=\\\"background-color: red; color: white;\\\">71.12</span>\"], [\"PDSetting2\", \"43.12\", \"51.98\", \"45.92\", \"36.83\", \"53.61\", \"44.99\", \"51.52\", \"48.02\", \"24.01\", \"37.53\", \"51.52\", \"50.82\", \"<span style=\\\"background-color: blue; color: white;\\\">53.85</span>\", \"35.2\", \"47.55\", \"39.63\", \"47.55\", \"36.83\", \"<span style=\\\"background-color: blue; color: white;\\\">53.85</span>\", \"51.28\", \"51.75\", \"50.35\", \"47.55\", \"<span style=\\\"background-color: red; color: white;\\\">54.31</span>\"], [\"PronunciationAudio\", \"50.0\", \"<span style=\\\"background-color: red; color: white;\\\">87.5</span>\", \"68.75\", \"62.5\", \"62.5\", \"68.75\", \"50.0\", \"62.5\", \"56.25\", \"Error\", \"75.0\", \"75.0\", \"68.75\", \"68.75\", \"75.0\", \"<span style=\\\"background-color: red; color: white;\\\">87.5</span>\", \"68.75\", \"75.0\", \"<span style=\\\"background-color: blue; color: white;\\\">81.25</span>\", \"<span style=\\\"background-color: red; color: white;\\\">87.5</span>\", \"75.0\", \"<span style=\\\"background-color: blue; color: white;\\\">81.25</span>\", \"75.0\", \"<span style=\\\"background-color: red; color: white;\\\">87.5</span>\"], [\"ECGFiveDays\", \"54.47\", \"58.07\", \"50.06\", \"58.42\", \"59.35\", \"52.61\", \"52.85\", \"49.71\", \"53.54\", \"Error\", \"56.79\", \"55.75\", \"53.66\", \"57.72\", \"58.19\", \"59.93\", \"55.87\", \"56.45\", \"59.0\", \"<span style=\\\"background-color: blue; color: white;\\\">60.28</span>\", \"55.17\", \"56.91\", \"54.59\", \"<span style=\\\"background-color: red; color: white;\\\">66.55</span>\"], [\"FreezerSmallTrain\", \"52.42\", \"54.28\", \"50.0\", \"50.95\", \"53.79\", \"50.0\", \"53.58\", \"50.0\", \"50.0\", \"50.11\", \"52.67\", \"58.18\", \"58.84\", \"51.79\", \"50.98\", \"61.09\", \"61.33\", \"56.21\", \"60.63\", \"<span style=\\\"background-color: blue; color: white;\\\">65.23</span>\", \"56.21\", \"63.68\", \"57.33\", \"<span style=\\\"background-color: red; color: white;\\\">70.25</span>\"], [\"HouseTwenty\", \"57.98\", \"57.14\", \"52.1\", \"57.98\", \"42.02\", \"57.14\", \"45.38\", \"45.38\", \"57.98\", \"53.78\", \"58.82\", \"59.66\", \"62.18\", \"58.82\", \"57.98\", \"<span style=\\\"background-color: blue; color: white;\\\">69.75</span>\", \"63.87\", \"60.5\", \"54.62\", \"59.66\", \"63.03\", \"63.03\", \"60.5\", \"<span style=\\\"background-color: red; color: white;\\\">70.59</span>\"], [\"InsectEPGRegularTrain\", \"<span style=\\\"background-color: red; color: white;\\\">100.0</span>\", \"<span style=\\\"background-color: red; color: white;\\\">100.0</span>\", \"<span style=\\\"background-color: blue; color: white;\\\">99.2</span>\", \"91.97\", \"83.13\", \"44.58\", \"95.98\", \"85.94\", \"<span style=\\\"background-color: red; color: white;\\\">100.0</span>\", \"<span style=\\\"background-color: red; color: white;\\\">100.0</span>\", \"46.18\", \"<span style=\\\"background-color: red; color: white;\\\">100.0</span>\", \"<span style=\\\"background-color: red; color: white;\\\">100.0</span>\", \"47.39\", \"46.18\", \"77.91\", \"91.57\", \"49.8\", \"<span style=\\\"background-color: red; color: white;\\\">100.0</span>\", \"83.53\", \"<span style=\\\"background-color: red; color: white;\\\">100.0</span>\", \"85.54\", \"<span style=\\\"background-color: red; color: white;\\\">100.0</span>\", \"<span style=\\\"background-color: red; color: white;\\\">100.0</span>\"], [\"ItalyPowerDemand\", \"57.05\", \"68.71\", \"52.77\", \"53.26\", \"65.79\", \"62.49\", \"55.88\", \"49.85\", \"61.61\", \"63.17\", \"70.36\", \"72.98\", \"74.73\", \"73.66\", \"<span style=\\\"background-color: red; color: white;\\\">76.97</span>\", \"74.44\", \"71.62\", \"71.14\", \"76.68\", \"<span style=\\\"background-color: blue; color: white;\\\">76.87</span>\", \"73.66\", \"75.02\", \"75.9\", \"72.5\"], [\"Lightning7\", \"21.92\", \"28.77\", \"12.33\", \"28.77\", \"26.03\", \"20.55\", \"24.66\", \"26.03\", \"17.81\", \"21.92\", \"27.4\", \"<span style=\\\"background-color: blue; color: white;\\\">32.88</span>\", \"27.4\", \"26.03\", \"28.77\", \"26.03\", \"30.14\", \"27.4\", \"28.77\", \"<span style=\\\"background-color: red; color: white;\\\">39.73</span>\", \"28.77\", \"27.4\", \"26.03\", \"31.51\"], [\"MoteStrain\", \"56.47\", \"61.1\", \"53.83\", \"53.51\", \"61.5\", \"47.76\", \"57.19\", \"50.64\", \"56.55\", \"Error\", \"<span style=\\\"background-color: blue; color: white;\\\">68.85</span>\", \"56.95\", \"61.98\", \"51.76\", \"52.24\", \"64.3\", \"57.43\", \"58.55\", \"59.19\", \"66.85\", \"60.3\", \"63.02\", \"60.86\", \"<span style=\\\"background-color: red; color: white;\\\">72.68</span>\"], [\"PowerCons\", \"78.33\", \"92.22\", \"65.56\", \"77.22\", \"85.56\", \"74.44\", \"80.0\", \"77.78\", \"68.33\", \"51.11\", \"87.22\", \"90.0\", \"90.56\", \"55.56\", \"62.22\", \"84.44\", \"93.33\", \"86.67\", \"<span style=\\\"background-color: blue; color: white;\\\">94.44</span>\", \"92.78\", \"93.89\", \"92.78\", \"<span style=\\\"background-color: red; color: white;\\\">95.0</span>\", \"92.78\"], [\"SonyAIBORobotSurface2\", \"63.27\", \"67.68\", \"48.06\", \"61.91\", \"69.78\", \"69.36\", \"64.43\", \"61.7\", \"63.27\", \"68.84\", \"74.71\", \"<span style=\\\"background-color: red; color: white;\\\">78.49</span>\", \"73.56\", \"75.55\", \"76.18\", \"75.24\", \"74.4\", \"74.82\", \"<span style=\\\"background-color: blue; color: white;\\\">76.6</span>\", \"73.56\", \"71.67\", \"75.87\", \"76.18\", \"75.97\"], [\"UWaveGestureLibraryAll\", \"78.25\", \"83.67\", \"12.23\", \"74.87\", \"79.79\", \"49.5\", \"47.38\", \"Error\", \"12.67\", \"26.52\", \"83.45\", \"78.5\", \"81.6\", \"16.42\", \"37.02\", \"84.45\", \"85.23\", \"84.95\", \"<span style=\\\"background-color: red; color: white;\\\">87.05</span>\", \"85.65\", \"<span style=\\\"background-color: blue; color: white;\\\">86.07</span>\", \"83.36\", \"82.05\", \"80.18\"]];\n",
       "\n",
       "        // Define the dt_args\n",
       "        let dt_args = {\"layout\": {\"topStart\": \"pageLength\", \"topEnd\": \"search\", \"bottomStart\": \"info\", \"bottomEnd\": \"paging\"}, \"order\": [], \"warn_on_selected_rows_not_rendered\": true};\n",
       "        dt_args[\"data\"] = data;\n",
       "\n",
       "        \n",
       "        new DataTable(table, dt_args);\n",
       "    });\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tools.display_dataframe_to_user(name=\"Classification Results\", dataframe=highlighted_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "html_filename = \"out/all_classification_benchmark_results.html\"\n",
    "highlighted_df.to_html(html_filename, escape=False, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "basics",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}