co-attention.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'cv2'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-90e1a8b01c66>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'cv2'"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import numpy\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator,array_to_img,img_to_array, load_img\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from keras.models import Sequential\n",
    "from keras.models import Model\n",
    "from keras.layers import Conv2D,MaxPooling2D,Activation,Dropout,Flatten,Dense,BatchNormalization\n",
    "from keras.applications.vgg19 import VGG19, preprocess_input\n",
    "import keras.backend as K\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras import Input, Model\n",
    "import PIL\n",
    "from PIL import Image,ImageOps\n",
    "from PIL import ImageDraw \n",
    "import shutil\n",
    "import os\n",
    "from os import listdir\n",
    "from PIL import Image\n",
    "import xml.etree.ElementTree as ET\n",
    "from tqdm import tqdm\n",
    "import seaborn as sns\n",
    "import string\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "import pickle\n",
    "import re\n",
    "from numpy import savez_compressed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tag Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train=pd.read_csv('traindata.csv',nrows=20)\n",
    "test=pd.read_csv('testdata.csv',nrows=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>image1_id</th>\n",
       "      <th>image2_id</th>\n",
       "      <th>report</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>CXR3080_IM-1440</td>\n",
       "      <td>CXR3080_IM-1440-1001</td>\n",
       "      <td>CXR3080_IM-1440-1002</td>\n",
       "      <td>startseq the cavity and the left upper lobe ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>CXR2721_IM-1183</td>\n",
       "      <td>CXR2721_IM-1183-1001</td>\n",
       "      <td>CXR2721_IM-1183-2001</td>\n",
       "      <td>startseq heart size at the upper limits of nor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>CXR2160_IM-0778</td>\n",
       "      <td>CXR2160_IM-0778-1001</td>\n",
       "      <td>CXR2160_IM-0778-2001</td>\n",
       "      <td>startseq the cardiomediastinal silhouette is w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>CXR3586_IM-1764</td>\n",
       "      <td>CXR3586_IM-1764-1001</td>\n",
       "      <td>CXR3586_IM-1764-2001</td>\n",
       "      <td>startseq the heart is mildly enlarged the medi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>CXR3139_IM-1476</td>\n",
       "      <td>CXR3139_IM-1476-1001</td>\n",
       "      <td>CXR3139_IM-1476-2001</td>\n",
       "      <td>startseq no focal areas of consolidation no pl...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0       patient_id             image1_id             image2_id  \\\n",
       "0           0  CXR3080_IM-1440  CXR3080_IM-1440-1001  CXR3080_IM-1440-1002   \n",
       "1           1  CXR2721_IM-1183  CXR2721_IM-1183-1001  CXR2721_IM-1183-2001   \n",
       "2           2  CXR2160_IM-0778  CXR2160_IM-0778-1001  CXR2160_IM-0778-2001   \n",
       "3           3  CXR3586_IM-1764  CXR3586_IM-1764-1001  CXR3586_IM-1764-2001   \n",
       "4           4  CXR3139_IM-1476  CXR3139_IM-1476-1001  CXR3139_IM-1476-2001   \n",
       "\n",
       "                                              report  \n",
       "0  startseq the cavity and the left upper lobe ha...  \n",
       "1  startseq heart size at the upper limits of nor...  \n",
       "2  startseq the cardiomediastinal silhouette is w...  \n",
       "3  startseq the heart is mildly enlarged the medi...  \n",
       "4  startseq no focal areas of consolidation no pl...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.drop('Unnamed: 0',axis='columns',inplace=True)\n",
    "test.drop('Unnamed: 0',axis='columns',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>image1_id</th>\n",
       "      <th>image2_id</th>\n",
       "      <th>report</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CXR3080_IM-1440</td>\n",
       "      <td>CXR3080_IM-1440-1001</td>\n",
       "      <td>CXR3080_IM-1440-1002</td>\n",
       "      <td>startseq the cavity and the left upper lobe ha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CXR2721_IM-1183</td>\n",
       "      <td>CXR2721_IM-1183-1001</td>\n",
       "      <td>CXR2721_IM-1183-2001</td>\n",
       "      <td>startseq heart size at the upper limits of nor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CXR2160_IM-0778</td>\n",
       "      <td>CXR2160_IM-0778-1001</td>\n",
       "      <td>CXR2160_IM-0778-2001</td>\n",
       "      <td>startseq the cardiomediastinal silhouette is w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CXR3586_IM-1764</td>\n",
       "      <td>CXR3586_IM-1764-1001</td>\n",
       "      <td>CXR3586_IM-1764-2001</td>\n",
       "      <td>startseq the heart is mildly enlarged the medi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CXR3139_IM-1476</td>\n",
       "      <td>CXR3139_IM-1476-1001</td>\n",
       "      <td>CXR3139_IM-1476-2001</td>\n",
       "      <td>startseq no focal areas of consolidation no pl...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        patient_id             image1_id             image2_id  \\\n",
       "0  CXR3080_IM-1440  CXR3080_IM-1440-1001  CXR3080_IM-1440-1002   \n",
       "1  CXR2721_IM-1183  CXR2721_IM-1183-1001  CXR2721_IM-1183-2001   \n",
       "2  CXR2160_IM-0778  CXR2160_IM-0778-1001  CXR2160_IM-0778-2001   \n",
       "3  CXR3586_IM-1764  CXR3586_IM-1764-1001  CXR3586_IM-1764-2001   \n",
       "4  CXR3139_IM-1476  CXR3139_IM-1476-1001  CXR3139_IM-1476-2001   \n",
       "\n",
       "                                              report  \n",
       "0  startseq the cavity and the left upper lobe ha...  \n",
       "1  startseq heart size at the upper limits of nor...  \n",
       "2  startseq the cardiomediastinal silhouette is w...  \n",
       "3  startseq the heart is mildly enlarged the medi...  \n",
       "4  startseq no focal areas of consolidation no pl...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>image1_id</th>\n",
       "      <th>image2_id</th>\n",
       "      <th>report</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CXR1306_IM-0200</td>\n",
       "      <td>CXR1306_IM-0200-2001</td>\n",
       "      <td>CXR1306_IM-0200-3001</td>\n",
       "      <td>startseq the  examination consists of frontal ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CXR1786_IM-0512</td>\n",
       "      <td>CXR1786_IM-0512-1001</td>\n",
       "      <td>CXR1786_IM-0512-2001</td>\n",
       "      <td>startseq the lungs are clear bilaterally speci...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CXR2072_IM-0706</td>\n",
       "      <td>CXR2072_IM-0706-1001</td>\n",
       "      <td>CXR2072_IM-0706-2001</td>\n",
       "      <td>startseq stable nonenlarged cardiomediastinal ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CXR467_IM-2095</td>\n",
       "      <td>CXR467_IM-2095-2001</td>\n",
       "      <td>CXR467_IM-2095-2001</td>\n",
       "      <td>startseq lungs are clear bilaterally cardiac a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CXR3087_IM-1444</td>\n",
       "      <td>CXR3087_IM-1444-1001</td>\n",
       "      <td>CXR3087_IM-1444-2001</td>\n",
       "      <td>startseq the heart is enlarged the mediastinal...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        patient_id             image1_id             image2_id  \\\n",
       "0  CXR1306_IM-0200  CXR1306_IM-0200-2001  CXR1306_IM-0200-3001   \n",
       "1  CXR1786_IM-0512  CXR1786_IM-0512-1001  CXR1786_IM-0512-2001   \n",
       "2  CXR2072_IM-0706  CXR2072_IM-0706-1001  CXR2072_IM-0706-2001   \n",
       "3   CXR467_IM-2095   CXR467_IM-2095-2001   CXR467_IM-2095-2001   \n",
       "4  CXR3087_IM-1444  CXR3087_IM-1444-1001  CXR3087_IM-1444-2001   \n",
       "\n",
       "                                              report  \n",
       "0  startseq the  examination consists of frontal ...  \n",
       "1  startseq the lungs are clear bilaterally speci...  \n",
       "2  startseq stable nonenlarged cardiomediastinal ...  \n",
       "3  startseq lungs are clear bilaterally cardiac a...  \n",
       "4  startseq the heart is enlarged the mediastinal...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train_features_list_new.pkl', 'rb') as f:\n",
    "    train_feat_list = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('test_features_list_new.pkl', 'rb') as f:\n",
    "    test_feat_list = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_feat_list_new = []\n",
    "for i in range(0,20):\n",
    "    train_feat_list_new.append(train_feat_list[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_feat_list_new = []\n",
    "for i in range(0,10):\n",
    "    test_feat_list_new.append(test_feat_list[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 10)"
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_feat_list_new), len(test_feat_list_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tags = pd.read_csv(\"tags.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tags</th>\n",
       "      <th>patient_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>normal</td>\n",
       "      <td>CXR3698_IM-1846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>normal</td>\n",
       "      <td>CXR370_IM-1848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>normal</td>\n",
       "      <td>CXR3679_IM-1831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Stents coronary vessels</td>\n",
       "      <td>CXR3689_IM-1840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>normal</td>\n",
       "      <td>CXR3664_IM-1822</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      tags       patient_id\n",
       "0                   normal  CXR3698_IM-1846\n",
       "1                   normal   CXR370_IM-1848\n",
       "2                   normal  CXR3679_IM-1831\n",
       "3  Stents coronary vessels  CXR3689_IM-1840\n",
       "4                   normal  CXR3664_IM-1822"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tags.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new_2 = pd.DataFrame(train.merge(df_tags, on='patient_id', how='left'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>image1_id</th>\n",
       "      <th>image2_id</th>\n",
       "      <th>report</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>CXR2721_IM-1183</td>\n",
       "      <td>CXR2407_IM-0954-1001</td>\n",
       "      <td>CXR2721_IM-1183-2001</td>\n",
       "      <td>startseq the cardiomediastinal silhouette and ...</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             patient_id             image1_id             image2_id  \\\n",
       "count                20                    20                    20   \n",
       "unique               20                    20                    20   \n",
       "top     CXR2721_IM-1183  CXR2407_IM-0954-1001  CXR2721_IM-1183-2001   \n",
       "freq                  1                     1                     1   \n",
       "\n",
       "                                                   report    tags  \n",
       "count                                                  20      20  \n",
       "unique                                                 20      13  \n",
       "top     startseq the cardiomediastinal silhouette and ...  normal  \n",
       "freq                                                    1       8  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new_2.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>image1_id</th>\n",
       "      <th>image2_id</th>\n",
       "      <th>report</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CXR3080_IM-1440</td>\n",
       "      <td>CXR3080_IM-1440-1001</td>\n",
       "      <td>CXR3080_IM-1440-1002</td>\n",
       "      <td>startseq the cavity and the left upper lobe ha...</td>\n",
       "      <td>Cavitation lung upper lobe right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CXR2721_IM-1183</td>\n",
       "      <td>CXR2721_IM-1183-1001</td>\n",
       "      <td>CXR2721_IM-1183-2001</td>\n",
       "      <td>startseq heart size at the upper limits of nor...</td>\n",
       "      <td>Deformity thoracic vertebrae multiple mild</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CXR2160_IM-0778</td>\n",
       "      <td>CXR2160_IM-0778-1001</td>\n",
       "      <td>CXR2160_IM-0778-2001</td>\n",
       "      <td>startseq the cardiomediastinal silhouette is w...</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CXR3586_IM-1764</td>\n",
       "      <td>CXR3586_IM-1764-1001</td>\n",
       "      <td>CXR3586_IM-1764-2001</td>\n",
       "      <td>startseq the heart is mildly enlarged the medi...</td>\n",
       "      <td>Cardiomegaly mild</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CXR3139_IM-1476</td>\n",
       "      <td>CXR3139_IM-1476-1001</td>\n",
       "      <td>CXR3139_IM-1476-2001</td>\n",
       "      <td>startseq no focal areas of consolidation no pl...</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        patient_id             image1_id             image2_id  \\\n",
       "0  CXR3080_IM-1440  CXR3080_IM-1440-1001  CXR3080_IM-1440-1002   \n",
       "1  CXR2721_IM-1183  CXR2721_IM-1183-1001  CXR2721_IM-1183-2001   \n",
       "2  CXR2160_IM-0778  CXR2160_IM-0778-1001  CXR2160_IM-0778-2001   \n",
       "3  CXR3586_IM-1764  CXR3586_IM-1764-1001  CXR3586_IM-1764-2001   \n",
       "4  CXR3139_IM-1476  CXR3139_IM-1476-1001  CXR3139_IM-1476-2001   \n",
       "\n",
       "                                              report  \\\n",
       "0  startseq the cavity and the left upper lobe ha...   \n",
       "1  startseq heart size at the upper limits of nor...   \n",
       "2  startseq the cardiomediastinal silhouette is w...   \n",
       "3  startseq the heart is mildly enlarged the medi...   \n",
       "4  startseq no focal areas of consolidation no pl...   \n",
       "\n",
       "                                         tags  \n",
       "0            Cavitation lung upper lobe right  \n",
       "1  Deformity thoracic vertebrae multiple mild  \n",
       "2                                      normal  \n",
       "3                           Cardiomegaly mild  \n",
       "4                                      normal  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new_2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "patient_id    0\n",
       "image1_id     0\n",
       "image2_id     0\n",
       "report        0\n",
       "tags          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new_2.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>image1_id</th>\n",
       "      <th>image2_id</th>\n",
       "      <th>report</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CXR3080_IM-1440</td>\n",
       "      <td>CXR3080_IM-1440-1001</td>\n",
       "      <td>CXR3080_IM-1440-1002</td>\n",
       "      <td>startseq the cavity and the left upper lobe ha...</td>\n",
       "      <td>Cavitation lung upper lobe right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CXR2721_IM-1183</td>\n",
       "      <td>CXR2721_IM-1183-1001</td>\n",
       "      <td>CXR2721_IM-1183-2001</td>\n",
       "      <td>startseq heart size at the upper limits of nor...</td>\n",
       "      <td>Deformity thoracic vertebrae multiple mild</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CXR2160_IM-0778</td>\n",
       "      <td>CXR2160_IM-0778-1001</td>\n",
       "      <td>CXR2160_IM-0778-2001</td>\n",
       "      <td>startseq the cardiomediastinal silhouette is w...</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CXR3586_IM-1764</td>\n",
       "      <td>CXR3586_IM-1764-1001</td>\n",
       "      <td>CXR3586_IM-1764-2001</td>\n",
       "      <td>startseq the heart is mildly enlarged the medi...</td>\n",
       "      <td>Cardiomegaly mild</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CXR3139_IM-1476</td>\n",
       "      <td>CXR3139_IM-1476-1001</td>\n",
       "      <td>CXR3139_IM-1476-2001</td>\n",
       "      <td>startseq no focal areas of consolidation no pl...</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        patient_id             image1_id             image2_id  \\\n",
       "0  CXR3080_IM-1440  CXR3080_IM-1440-1001  CXR3080_IM-1440-1002   \n",
       "1  CXR2721_IM-1183  CXR2721_IM-1183-1001  CXR2721_IM-1183-2001   \n",
       "2  CXR2160_IM-0778  CXR2160_IM-0778-1001  CXR2160_IM-0778-2001   \n",
       "3  CXR3586_IM-1764  CXR3586_IM-1764-1001  CXR3586_IM-1764-2001   \n",
       "4  CXR3139_IM-1476  CXR3139_IM-1476-1001  CXR3139_IM-1476-2001   \n",
       "\n",
       "                                              report  \\\n",
       "0  startseq the cavity and the left upper lobe ha...   \n",
       "1  startseq heart size at the upper limits of nor...   \n",
       "2  startseq the cardiomediastinal silhouette is w...   \n",
       "3  startseq the heart is mildly enlarged the medi...   \n",
       "4  startseq no focal areas of consolidation no pl...   \n",
       "\n",
       "                                         tags  \n",
       "0            Cavitation lung upper lobe right  \n",
       "1  Deformity thoracic vertebrae multiple mild  \n",
       "2                                      normal  \n",
       "3                           Cardiomegaly mild  \n",
       "4                                      normal  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new_2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new_2['tags'].replace('\\d+', '', regex=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patient_id</th>\n",
       "      <th>image1_id</th>\n",
       "      <th>image2_id</th>\n",
       "      <th>report</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CXR3080_IM-1440</td>\n",
       "      <td>CXR3080_IM-1440-1001</td>\n",
       "      <td>CXR3080_IM-1440-1002</td>\n",
       "      <td>startseq the cavity and the left upper lobe ha...</td>\n",
       "      <td>Cavitation lung upper lobe right</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CXR2721_IM-1183</td>\n",
       "      <td>CXR2721_IM-1183-1001</td>\n",
       "      <td>CXR2721_IM-1183-2001</td>\n",
       "      <td>startseq heart size at the upper limits of nor...</td>\n",
       "      <td>Deformity thoracic vertebrae multiple mild</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CXR2160_IM-0778</td>\n",
       "      <td>CXR2160_IM-0778-1001</td>\n",
       "      <td>CXR2160_IM-0778-2001</td>\n",
       "      <td>startseq the cardiomediastinal silhouette is w...</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CXR3586_IM-1764</td>\n",
       "      <td>CXR3586_IM-1764-1001</td>\n",
       "      <td>CXR3586_IM-1764-2001</td>\n",
       "      <td>startseq the heart is mildly enlarged the medi...</td>\n",
       "      <td>Cardiomegaly mild</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CXR3139_IM-1476</td>\n",
       "      <td>CXR3139_IM-1476-1001</td>\n",
       "      <td>CXR3139_IM-1476-2001</td>\n",
       "      <td>startseq no focal areas of consolidation no pl...</td>\n",
       "      <td>normal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        patient_id             image1_id             image2_id  \\\n",
       "0  CXR3080_IM-1440  CXR3080_IM-1440-1001  CXR3080_IM-1440-1002   \n",
       "1  CXR2721_IM-1183  CXR2721_IM-1183-1001  CXR2721_IM-1183-2001   \n",
       "2  CXR2160_IM-0778  CXR2160_IM-0778-1001  CXR2160_IM-0778-2001   \n",
       "3  CXR3586_IM-1764  CXR3586_IM-1764-1001  CXR3586_IM-1764-2001   \n",
       "4  CXR3139_IM-1476  CXR3139_IM-1476-1001  CXR3139_IM-1476-2001   \n",
       "\n",
       "                                              report  \\\n",
       "0  startseq the cavity and the left upper lobe ha...   \n",
       "1  startseq heart size at the upper limits of nor...   \n",
       "2  startseq the cardiomediastinal silhouette is w...   \n",
       "3  startseq the heart is mildly enlarged the medi...   \n",
       "4  startseq no focal areas of consolidation no pl...   \n",
       "\n",
       "                                         tags  \n",
       "0            Cavitation lung upper lobe right  \n",
       "1  Deformity thoracic vertebrae multiple mild  \n",
       "2                                      normal  \n",
       "3                           Cardiomegaly mild  \n",
       "4                                      normal  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new_2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new_2.to_csv('df_coatt.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 128, 128, 3) (20, 30)\n"
     ]
    }
   ],
   "source": [
    "def create_tag_mapping(mapping_csv):\n",
    "    labels = set()\n",
    "    for i in range(len(mapping_csv)):\n",
    "        tags = mapping_csv['tags'][i].split(' ')\n",
    "        labels.update(tags)\n",
    "    labels = list(labels)\n",
    "    labels.sort()\n",
    "    labels_map = {labels[i]:i for i in range(len(labels))}\n",
    "    inv_labels_map = {i:labels[i] for i in range(len(labels))}\n",
    "    return labels_map, inv_labels_map\n",
    " \n",
    "def create_file_mapping(mapping_csv):\n",
    "    mapping = dict()\n",
    "    for i in range(len(mapping_csv)):\n",
    "        name, tags = mapping_csv['image1_id'][i], mapping_csv['tags'][i]\n",
    "        mapping[name] = tags.split(' ')\n",
    "    return mapping\n",
    " \n",
    "\n",
    "def one_hot_encode(tags, mapping):\n",
    "    encoding = np.zeros(len(mapping), dtype='uint8')\n",
    "    for tag in tags:\n",
    "        encoding[mapping[tag]] = 1\n",
    "    return encoding\n",
    " \n",
    "def load_dataset(path, file_mapping, tag_mapping):\n",
    "    photos, targets = list(), list()\n",
    "    for i in file_mapping:\n",
    "        for filename in os.listdir(folder):\n",
    "            image_name = os.path.splitext(filename)[0]\n",
    "            if(i==image_name):\n",
    "                photo = Image.open(path + image_name + '.png')\n",
    "                photo=img_to_array(photo)\n",
    "                photo = preprocess_input(photo)\n",
    "                photo=cv2.resize(photo,(128,128))\n",
    "                photo = img_to_array(photo, dtype='uint8')\n",
    "                tags = file_mapping[image_name]\n",
    "                target = one_hot_encode(tags, tag_mapping)\n",
    "                photos.append(photo)\n",
    "                targets.append(target)\n",
    "    X = np.asarray(photos, dtype='uint8')\n",
    "    y = np.asarray(targets, dtype='uint8')\n",
    "    return X, y\n",
    " \n",
    "filename = 'df_coatt.csv'\n",
    "mapping_csv = pd.read_csv(filename)\n",
    "tag_mapping, _ = create_tag_mapping(mapping_csv)\n",
    "file_mapping = create_file_mapping(mapping_csv)\n",
    "folder = 'C:/Users/INDIRA/Downloads/data rescon/images/'\n",
    "X, y = load_dataset(folder, file_mapping, tag_mapping)\n",
    "print(X.shape, y.shape)\n",
    "savez_compressed('data_coatt.npz', X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset():\n",
    "    data = np.load('data_coatt.npz')\n",
    "    X, y = data['arr_0'], data['arr_1']\n",
    "    trainX, trainY = X, y\n",
    "    return trainX, trainY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((20, 128, 128, 3), (20, 30))"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainX.shape , trainY.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "def define_model(in_shape=(128, 128, 3), out_shape=30):\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same', input_shape=in_shape))\n",
    "    model.add(Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
    "    model.add(MaxPooling2D((2, 2)))\n",
    "    model.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
    "    model.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
    "    model.add(MaxPooling2D((2, 2)))\n",
    "    model.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
    "    model.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))\n",
    "    model.add(MaxPooling2D((2, 2)))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(128, activation='relu', kernel_initializer='he_uniform'))\n",
    "    model.add(Dense(out_shape, activation='softmax'))\n",
    "    # compile model\n",
    "    opt = tf.keras.optimizers.SGD(lr=0.001, momentum=0.9)\n",
    "    model.compile(optimizer=opt, loss='binary_crossentropy', metrics='accuracy')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "datagen = ImageDataGenerator(rescale=1.0/255.0)\n",
    "trainX, trainY = load_dataset()\n",
    "train_it = datagen.flow(trainX, trainY, batch_size=5)\n",
    "model = define_model()\n",
    "history = model.fit_generator(train_it, steps_per_epoch=len(train_it), epochs=10, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> loss=0.225, fbeta=0.500\n"
     ]
    }
   ],
   "source": [
    "loss, fbeta = model.evaluate_generator(train_it, steps=len(train_it), verbose=0)\n",
    "print('> loss=%.3f, fbeta=%.3f' % (loss, fbeta))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat = model.predict(trainX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 30)"
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yhat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat_list = yhat.max(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(yhat_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.        , 0.89129025, 1.        , 1.        , 1.        ,\n",
       "       1.        , 0.9939778 , 1.        , 1.        , 1.        ,\n",
       "       1.        , 1.        , 0.99420446, 0.99827516, 1.        ,\n",
       "       1.        , 1.        , 1.        , 1.        , 1.        ],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yhat_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = np.argmax(yhat, axis=-1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([24, 22, 24, 22, 24, 22,  8, 24, 24, 24, 24, 24,  6, 21, 22, 24, 24,\n",
       "       24, 21, 24], dtype=int64)"
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, inv_tag_mapping = create_tag_mapping(mapping_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'Airspace',\n",
       " 1: 'Bone',\n",
       " 2: 'Bones',\n",
       " 3: 'Cardiomegaly',\n",
       " 4: 'Cavitation',\n",
       " 5: 'Cicatrix',\n",
       " 6: 'Deformity',\n",
       " 7: 'Density',\n",
       " 8: 'Disease',\n",
       " 9: 'Granulomatous',\n",
       " 10: 'Hernia,',\n",
       " 11: 'Hiatal',\n",
       " 12: 'Lung',\n",
       " 13: 'Thoracic',\n",
       " 14: 'Vertebrae',\n",
       " 15: 'and',\n",
       " 16: 'base',\n",
       " 17: 'degenerative',\n",
       " 18: 'hypoinflation',\n",
       " 19: 'left',\n",
       " 20: 'lobe',\n",
       " 21: 'lung',\n",
       " 22: 'mild',\n",
       " 23: 'multiple',\n",
       " 24: 'normal',\n",
       " 25: 'right',\n",
       " 26: 'thoracic',\n",
       " 27: 'thorax',\n",
       " 28: 'upper',\n",
       " 29: 'vertebrae'}"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inv_tag_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [],
   "source": [
    "semantic_feat = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [],
   "source": [
    "for keys in inv_tag_mapping.keys():\n",
    "    for item in labels:\n",
    "        if item == keys:\n",
    "            semantic_feat.append(inv_tag_mapping[keys])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Deformity',\n",
       " 'Disease',\n",
       " 'lung',\n",
       " 'lung',\n",
       " 'mild',\n",
       " 'mild',\n",
       " 'mild',\n",
       " 'mild',\n",
       " 'normal',\n",
       " 'normal',\n",
       " 'normal',\n",
       " 'normal',\n",
       " 'normal',\n",
       " 'normal',\n",
       " 'normal',\n",
       " 'normal',\n",
       " 'normal',\n",
       " 'normal',\n",
       " 'normal',\n",
       " 'normal']"
      ]
     },
     "execution_count": 333,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "semantic_feat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {},
   "outputs": [],
   "source": [
    " tag_mapping, _ = create_tag_mapping(mapping_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [],
   "source": [
    "semantic_encoding = one_hot_encode(semantic_feat,tag_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [],
   "source": [
    "semantic_feat_encoding = list(semantic_encoding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Semantic Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 339,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(semantic_feat_encoding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [],
   "source": [
    "co_att_feat = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visual Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 341,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_feat_list_new)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Concatenating two features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [],
   "source": [
    "co_att_feat = train_feat_list_new + semantic_feat_encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 343,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(co_att_feat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [],
   "source": [
    "co_att_feat_arr = x = np.asarray(co_att_feat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50,)"
      ]
     },
     "execution_count": 345,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "co_att_feat_arr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 350,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(co_att_feat_arr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Co-attention layer to generate context vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {},
   "outputs": [],
   "source": [
    "inp = Input(shape=co_att_feat_arr.shape[0])\n",
    "hidden = Dense(8, activation='relu')(inp)\n",
    "flat = Flatten()(hidden)\n",
    "out = Dense(2)(flat)\n",
    "model = Model(inputs=inp, outputs=out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}