Initial_Data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import dask.dataframe as dd\n",
    "from dask.distributed import Client\n",
    "#Importing plot lib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import matplotlib.dates as mdates\n",
    "sns.set(rc={'figure.figsize':(20, 10)})\n",
    "import statsmodels as sm\n",
    "from statsmodels.tsa.stattools import adfuller\n",
    "from numpy import log\n",
    "from statsmodels.graphics.tsaplots import plot_acf, plot_pacf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating distributed client\n",
    "client = Client()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dask DataFrame Structure:\n",
      "                 2016-06-01 00:05:01   39.364 211.62.96.220 42.219.158.212  55107  64188     UDP  .A....      0    0.1     19   3958 background\n",
      "npartitions=3889                                                                                                                               \n",
      "                              object  float64        object         object  int64  int64  object  object  int64  int64  int64  int64     object\n",
      "                                 ...      ...           ...            ...    ...    ...     ...     ...    ...    ...    ...    ...        ...\n",
      "...                              ...      ...           ...            ...    ...    ...     ...     ...    ...    ...    ...    ...        ...\n",
      "                                 ...      ...           ...            ...    ...    ...     ...     ...    ...    ...    ...    ...        ...\n",
      "                                 ...      ...           ...            ...    ...    ...     ...     ...    ...    ...    ...    ...        ...\n",
      "Dask Name: from-delayed, 11667 tasks\n"
     ]
    }
   ],
   "source": [
    "df = dd.read_csv('june*')\n",
    "print (df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dask DataFrame Structure:\n",
      "                 2016-06-20 00:06:55    2.856 42.219.159.90 83.107.168.102  36351    445     TCP  ....S.      0    0.1      2     96 background\n",
      "npartitions=1325                                                                                                                               \n",
      "                              object  float64        object         object  int64  int64  object  object  int64  int64  int64  int64     object\n",
      "                                 ...      ...           ...            ...    ...    ...     ...     ...    ...    ...    ...    ...        ...\n",
      "...                              ...      ...           ...            ...    ...    ...     ...     ...    ...    ...    ...    ...        ...\n",
      "                                 ...      ...           ...            ...    ...    ...     ...     ...    ...    ...    ...    ...        ...\n",
      "                                 ...      ...           ...            ...    ...    ...     ...     ...    ...    ...    ...    ...        ...\n",
      "Dask Name: from-delayed, 3975 tasks\n"
     ]
    }
   ],
   "source": [
    "test_df = dd.read_csv('test*')\n",
    "print (test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns=['te','td','sa','da','sp','dp','pr','flg','fwd','stos','pkt','byt','label']\n",
    "test_df.columns=['te','td','sa','da','sp','dp','pr','flg','fwd','stos','pkt','byt','label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>te</th>\n",
       "      <th>td</th>\n",
       "      <th>sa</th>\n",
       "      <th>da</th>\n",
       "      <th>sp</th>\n",
       "      <th>dp</th>\n",
       "      <th>pr</th>\n",
       "      <th>flg</th>\n",
       "      <th>fwd</th>\n",
       "      <th>stos</th>\n",
       "      <th>pkt</th>\n",
       "      <th>byt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2016-06-01 00:05:03</td>\n",
       "      <td>39.828</td>\n",
       "      <td>42.219.158.226</td>\n",
       "      <td>71.247.111.184</td>\n",
       "      <td>80</td>\n",
       "      <td>52475</td>\n",
       "      <td>TCP</td>\n",
       "      <td>.AP.S.</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>57</td>\n",
       "      <td>79635</td>\n",
       "      <td>background</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-01 00:05:04</td>\n",
       "      <td>36.128</td>\n",
       "      <td>42.219.153.155</td>\n",
       "      <td>223.80.226.127</td>\n",
       "      <td>443</td>\n",
       "      <td>54691</td>\n",
       "      <td>TCP</td>\n",
       "      <td>.AP.S.</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2791</td>\n",
       "      <td>background</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2016-06-01 00:05:04</td>\n",
       "      <td>36.204</td>\n",
       "      <td>223.80.226.127</td>\n",
       "      <td>42.219.153.155</td>\n",
       "      <td>54691</td>\n",
       "      <td>443</td>\n",
       "      <td>TCP</td>\n",
       "      <td>.AP.S.</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>3896</td>\n",
       "      <td>background</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2016-06-01 00:05:04</td>\n",
       "      <td>42.452</td>\n",
       "      <td>42.219.153.7</td>\n",
       "      <td>42.187.82.40</td>\n",
       "      <td>53</td>\n",
       "      <td>53</td>\n",
       "      <td>UDP</td>\n",
       "      <td>.A....</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>175</td>\n",
       "      <td>background</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2016-06-01 00:05:06</td>\n",
       "      <td>36.968</td>\n",
       "      <td>42.219.153.89</td>\n",
       "      <td>143.72.8.137</td>\n",
       "      <td>63532</td>\n",
       "      <td>53</td>\n",
       "      <td>UDP</td>\n",
       "      <td>.A....</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>233</td>\n",
       "      <td>background</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>2016-06-01 00:05:06</td>\n",
       "      <td>36.992</td>\n",
       "      <td>143.72.8.137</td>\n",
       "      <td>42.219.153.89</td>\n",
       "      <td>53</td>\n",
       "      <td>63532</td>\n",
       "      <td>UDP</td>\n",
       "      <td>.A....</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>281</td>\n",
       "      <td>background</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2016-06-01 00:05:06</td>\n",
       "      <td>38.172</td>\n",
       "      <td>240.177.183.143</td>\n",
       "      <td>42.219.155.59</td>\n",
       "      <td>443</td>\n",
       "      <td>62402</td>\n",
       "      <td>TCP</td>\n",
       "      <td>.AP...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>82</td>\n",
       "      <td>background</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2016-06-01 00:05:06</td>\n",
       "      <td>40.648</td>\n",
       "      <td>42.219.155.59</td>\n",
       "      <td>52.76.101.190</td>\n",
       "      <td>42776</td>\n",
       "      <td>22163</td>\n",
       "      <td>UDP</td>\n",
       "      <td>.A....</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>275</td>\n",
       "      <td>background</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2016-06-01 00:05:07</td>\n",
       "      <td>176.320</td>\n",
       "      <td>42.219.153.89</td>\n",
       "      <td>92.225.32.126</td>\n",
       "      <td>61846</td>\n",
       "      <td>443</td>\n",
       "      <td>TCP</td>\n",
       "      <td>.AP...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2220</td>\n",
       "      <td>background</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2016-06-01 00:05:07</td>\n",
       "      <td>176.532</td>\n",
       "      <td>92.225.32.126</td>\n",
       "      <td>42.219.153.89</td>\n",
       "      <td>443</td>\n",
       "      <td>61846</td>\n",
       "      <td>TCP</td>\n",
       "      <td>.AP...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>2070</td>\n",
       "      <td>background</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    te       td               sa              da     sp  \\\n",
       "0  2016-06-01 00:05:03   39.828   42.219.158.226  71.247.111.184     80   \n",
       "1  2016-06-01 00:05:04   36.128   42.219.153.155  223.80.226.127    443   \n",
       "2  2016-06-01 00:05:04   36.204   223.80.226.127  42.219.153.155  54691   \n",
       "3  2016-06-01 00:05:04   42.452     42.219.153.7    42.187.82.40     53   \n",
       "4  2016-06-01 00:05:06   36.968    42.219.153.89    143.72.8.137  63532   \n",
       "5  2016-06-01 00:05:06   36.992     143.72.8.137   42.219.153.89     53   \n",
       "6  2016-06-01 00:05:06   38.172  240.177.183.143   42.219.155.59    443   \n",
       "7  2016-06-01 00:05:06   40.648    42.219.155.59   52.76.101.190  42776   \n",
       "8  2016-06-01 00:05:07  176.320    42.219.153.89   92.225.32.126  61846   \n",
       "9  2016-06-01 00:05:07  176.532    92.225.32.126   42.219.153.89    443   \n",
       "\n",
       "      dp   pr     flg  fwd  stos  pkt    byt       label  \n",
       "0  52475  TCP  .AP.S.    0     0   57  79635  background  \n",
       "1  54691  TCP  .AP.S.    0     0    9   2791  background  \n",
       "2    443  TCP  .AP.S.    0     0   13   3896  background  \n",
       "3     53  UDP  .A....    0     0    2    175  background  \n",
       "4     53  UDP  .A....    0     0    3    233  background  \n",
       "5  63532  UDP  .A....    0     0    3    281  background  \n",
       "6  62402  TCP  .AP...    0     0    2     82  background  \n",
       "7  22163  UDP  .A....    0     0    4    275  background  \n",
       "8    443  TCP  .AP...    0     0    5   2220  background  \n",
       "9  61846  TCP  .AP...    0     0   10   2070  background  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Keeping a copy\n",
    "data = df\n",
    "test_data = test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Changing column data type for te\n",
    "data['te'] = dd.to_datetime(data['te'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "distributed.nanny - WARNING - Restarting worker\n",
      "distributed.nanny - WARNING - Restarting worker\n",
      "distributed.nanny - WARNING - Restarting worker\n"
     ]
    }
   ],
   "source": [
    "data['ts'] = data['te'] - pd.to_timedelta(data['td'], unit='ms')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data['ts'] = test_data['te'] - pd.to_timedelta(data.td, unit='ms')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to mark traffic has anomalous data when windowing\n",
    "normal_traffic_type = ['background', 'blacklist']\n",
    "def isAnomolus(x):\n",
    "    data = ~x.isin(normal_traffic_type)\n",
    "    if data.any():\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to mark traffic has anomalous data when windowing\n",
    "normal_traffic_type = ['background', 'blacklist']\n",
    "def isAnomolus_new(x):\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "isAnomolus_agg = dd.Aggregation(\n",
    "    name = 'isAnomolus',\n",
    "    chunk = lambda x: x.apply(lambda a: (~a.isin(normal_traffic_type)).any()),\n",
    "    agg = lambda y: y.apply(lambda z: 1 if z.any() == True else 0)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data windowed for 5 min by adding pkt and byt, whreas label and true if any anomalous traffic exists in time window\n",
    "data_win = data.groupby(pd.Grouper(key='te', freq='5T')).agg({\n",
    "                                        \"pkt\":  \"sum\",\n",
    "                                        \"byt\":  \"sum\",\n",
    "                                        \"label\": isAnomolus_agg\n",
    "                                    }).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "data_win"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data windowed for 5 min by adding pkt and byt, whreas label and true if any anomalous traffic exists in time window\n",
    "data_win_ts = data.groupby(pd.Grouper(key='ts', freq='5T')).agg({\n",
    "                                        \"pkt\":  \"sum\",\n",
    "                                        \"byt\":  \"sum\",\n",
    "                                        \"label\": isAnomolus_agg\n",
    "                                    }).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data windowed for 5 min by adding pkt and byt, whreas label and true if any anomalous traffic exists in time window\n",
    "test_data_win = test_data.groupby(pd.Grouper(key='te', freq='5T')).agg({\n",
    "                                        \"pkt\":  \"sum\",\n",
    "                                        \"byt\":  \"sum\",\n",
    "                                        \"label\": isAnomolus_agg\n",
    "                                    }).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_data_win"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data windowed for 5 min by adding pkt and byt, whreas label and true if any anomalous traffic exists in time window\n",
    "test_data_win_ts = test_data.groupby(pd.Grouper(key='ts', freq='5T')).agg({\n",
    "                                        \"pkt\":  \"sum\",\n",
    "                                        \"byt\":  \"sum\",\n",
    "                                        \"label\": isAnomolus_agg\n",
    "                                    }).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_win_anomaly = data_win[data_win['label'] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_win_anomaly"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Taking only normal traffic(including blacklisted) \n",
    "The idea is to forecats the normal traffic, and compare with anomalous traffic for any deviation. Hence getting normal traffic windowed for each 5min which will be used for forecasting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter normal traffic\n",
    "data_normal_traffic = data[data['label'].isin(normal_traffic_type)] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_normal_traffic = test_data[test_data['label'].isin(normal_traffic_type)] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Window normal traffic for 5min window\n",
    "data_normal_traffic_win = data_normal_traffic.groupby(pd.Grouper(key='te', freq='5T')).agg({\n",
    "                                        \"pkt\":  \"sum\",\n",
    "                                        \"byt\":  \"sum\"\n",
    "                                    }).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Window normal traffic for 5min window\n",
    "test_data_normal_traffic_win = test_data_normal_traffic.groupby(pd.Grouper(key='te', freq='5T')).agg({\n",
    "                                        \"pkt\":  \"sum\",\n",
    "                                        \"byt\":  \"sum\"\n",
    "                                    }).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Window normal traffic for 5min window\n",
    "data_normal_traffic_win_ts = data_normal_traffic.groupby(pd.Grouper(key='ts', freq='5T')).agg({\n",
    "                                        \"pkt\":  \"sum\",\n",
    "                                        \"byt\":  \"sum\"\n",
    "                                    }).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Window normal traffic for 5min window\n",
    "test_data_normal_traffic_win_ts = test_data_normal_traffic.groupby(pd.Grouper(key='ts', freq='5T')).agg({\n",
    "                                        \"pkt\":  \"sum\",\n",
    "                                        \"byt\":  \"sum\"\n",
    "                                    }).compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_win.to_csv('data_win.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_normal_traffic_win.to_csv('data_normal_traffic_win.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_win.to_csv('test_data_win.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_normal_traffic_win.to_csv('test_data_normal_traffic_win.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_win_ts.to_csv('data_win_ts.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_normal_traffic_win_ts.to_csv('data_normal_traffic_win_ts.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_win_ts.to_csv('test_data_win_ts.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_normal_traffic_win_ts.to_csv('test_data_normal_traffic_win_ts.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "plt.plot(data_normal_traffic_win.index, data_normal_traffic_win.byt)\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_data = pd.merge(data_win, data_normal_traffic_win, on=['te'], how='inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_data['byt_diff'] = merged_data.byt_x - merged_data.byt_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.rc('font', size=12)\n",
    "fig, ax = plt.subplots(figsize=(20, 6))\n",
    "myFmt = mdates.DateFormatter('%Y-%M-%D')\n",
    "ax.xaxis.set_major_formatter(myFmt)\n",
    "ax.plot(merged_data.byt_diff, label='Difference')\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### EDA for windowed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(data_win.label)\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(merged_data.label, merged_data.byt_diff)\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_data[(merged_data.byt_diff == 0) & (merged_data.label == 1)].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_data[merged_data.byt_diff !=0].byt_diff.nsmallest(40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['anomaly'] = data['label'].apply(lambda x: 0 if x in(normal_traffic_type) else 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "protocols = ['TCP', 'UDP', 'ICMP', 'GRE', 'ESP', 'RSVP', 'IPv6', 'IPIP', '255','nan']\n",
    "data['protocol'] = data['pr'].apply(lambda x: protocols.index(x) if x in protocols else -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data['pr'].unique().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data['flg'].unique().compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}