notebook/Analyze results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Video Face Manipulation Detection Through Ensemble of CNNs\n",
    "Image and Sound Processing Lab - Politecnico di Milano\n",
    "- Nicolò Bonettini\n",
    "- Edoardo Daniele Cannas\n",
    "- Sara Mandelli\n",
    "- Luca Bondi\n",
    "- Paolo Bestagini"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn.metrics as M\n",
    "from scipy.special import expit\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "results_root = Path('results/')\n",
    "results_model_folder = list(results_root.glob('net-*'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def compute_metrics(df_res:pd.DataFrame,train_tag:str) -> dict:\n",
    "    numreal = sum(df_res['label']==False)\n",
    "    numfake = sum(df_res['label']==True\n",
    ")\n",
    "    \n",
    "    netname = train_tag.split('net-')[1].split('_')[0]\n",
    "    traindb = train_tag.split('traindb-')[1].split('_')[0]\n",
    "    \n",
    "    loss = M.log_loss(df_res['label'],expit(df_res['score']))\n",
    "    acc = M.accuracy_score(df_res['label'],df_res['score']>0)\n",
    "    accbal = M.balanced_accuracy_score(df_res['label'],df_res['score']>0)\n",
    "    rocauc = M.roc_auc_score(df_res['label'],df_res['score'])\n",
    "    \n",
    "    res_dict = {'traintag':train_tag,\n",
    "                'net':netname,\n",
    "                'traindb': traindb,\n",
    "                'testdb':testdb,'testsplit':testsplit,\n",
    "                'numreal':numreal,'numfake':numfake,\n",
    "                'loss':loss,\n",
    "                'acc':acc,'accbal':accbal,\n",
    "                'rocauc':rocauc} \n",
    "    return res_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "results_frame_list = []\n",
    "results_video_list = []\n",
    "\n",
    "for model_folder in tqdm(results_model_folder):\n",
    "    train_model_tag = model_folder.name\n",
    "    model_results = model_folder.glob('*.pkl')\n",
    "    for model_path in model_results:\n",
    "        testdb,testsplit = model_path.with_suffix('').name.rsplit('_',1)\n",
    "        \n",
    "        df_frames = pd.read_pickle(model_path)\n",
    "        results_frame_list.append(compute_metrics(df_frames,train_model_tag))\n",
    "        \n",
    "        df_videos = df_frames[['video','label','score']].groupby('video').mean()\n",
    "        df_videos['label'] = df_videos['label'].astype(np.bool)\n",
    "        results_video_list.append(compute_metrics(df_videos,train_model_tag))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "df_res_frames = pd.DataFrame(results_frame_list)\n",
    "df_res_frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "df_res_video = pd.DataFrame(results_video_list)\n",
    "df_res_video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "df_res_frames.to_csv(results_root.joinpath('frames.csv'),index=False)\n",
    "df_res_video.to_csv(results_root.joinpath('videos.csv'),index=False)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}