dc_benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "import deepchem as dc\n",
    "from deepchem.feat import Featurizer\n",
    "\n",
    "from dataloader import *\n",
    "import model\n",
    "import model_old\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "from pprint import pprint\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Source code for fc models (tf for classification, tf_regression for regression)\n",
    "# https://github.com/deepchem/deepchem/blob/master/deepchem/models/tensorgraph/fcnet.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def benchmark(dir_path, ckpt_file, tasks, _model, featurizer, metric_type):\n",
    "    \n",
    "    header = ckpt_file.split('.')[0]\n",
    "    _ckpt = header+'_'+featurizer\n",
    "    checkpoint = torch.load(join(dir_path, ckpt_file), map_location=torch.device('cuda'))\n",
    "    args = checkpoint['args']\n",
    "    _args = vars(args)\n",
    "    args.batch_size = 1\n",
    "    args.test_batch_size = 1\n",
    "    args.act = 'gelu'\n",
    "#     comet = model.Encoder(args)\n",
    "    comet = model_old.Encoder(args)\n",
    "    comet.load_state_dict(checkpoint['encoder'])\n",
    "    comet.eval()\n",
    "\n",
    "    def mol_to_graph(mol):\n",
    "\n",
    "        adj = Chem.rdmolops.GetAdjacencyMatrix(mol)\n",
    "        list_feature = list()\n",
    "        for atom in mol.GetAtoms():\n",
    "            list_feature.append(atom_feature(atom))\n",
    "        return np.array(list_feature), adj\n",
    "\n",
    "    class MyEncoder(Featurizer):\n",
    "        name = ['comet_encoder']\n",
    "        def __init__(self, model):\n",
    "            self.device = 'cuda'\n",
    "\n",
    "            self.model = model.to(self.device)\n",
    "\n",
    "        def _featurize(self, mol):\n",
    "            X, A = mol_to_graph(mol)\n",
    "            X = Variable(torch.unsqueeze(torch.from_numpy(X), dim=0)).to(self.device).long()\n",
    "            A = Variable(torch.unsqueeze(torch.from_numpy(A.astype(float)), dim=0)).to(self.device).float()\n",
    "            _, _, molvec = self.model(X, A)\n",
    "            return torch.squeeze(molvec).detach().cpu().numpy()\n",
    "\n",
    "    if metric_type == 'reg':\n",
    "        _metric = [dc.metrics.Metric(dc.metrics.mae_score, np.mean)]\n",
    "    if metric_type == 'cls':\n",
    "        _metric = [dc.metrics.Metric(dc.metrics.roc_auc_score, np.mean)]\n",
    "    \n",
    "    if featurizer == 'comet':\n",
    "        feat = MyEncoder(comet)\n",
    "    if featurizer == 'raw':\n",
    "        feat = dc.feat.raw_featurizer.RawFeaturizer()\n",
    "    if featurizer == 'fingerprint':\n",
    "        feat = dc.feat.fingerprints.CircularFingerprint(size=256)\n",
    "    if featurizer == 'default':\n",
    "        feat = None\n",
    "    \n",
    "\n",
    "    task_result = dc.molnet.run_benchmark(\n",
    "                                 ckpt = _ckpt,\n",
    "                                 arg = _args,\n",
    "                                 datasets = tasks,\n",
    "                                 model = _model,\n",
    "                                 split = None,\n",
    "                                 metric = _metric,\n",
    "                                 n_features = 256,\n",
    "                                 featurizer = feat,\n",
    "                                 out_path= './results',\n",
    "                                 hyper_parameters = None,\n",
    "                                 test = True,\n",
    "                                 reload = False,\n",
    "                                 seed = 123\n",
    "                                 )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## task types\n",
    "---\n",
    "1) classification: muv, hiv, bace_c, bbbp, tox21, toxcast, sider, clintox\n",
    "\n",
    "\n",
    "2) regression: lipo, qm7, qm8, delaney, sampl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "classification = ['bace_c', 'bbbp', 'clintox', 'hiv', 'muv', 'pcba', 'sider', 'tox21', 'toxcast']\n",
    "cls_models =  ['rf', 'tf', 'tf_robust', 'logreg', 'irv', 'graphconv', 'dag', 'xgb',\n",
    "      'weave', 'kernelsvm']\n",
    "\n",
    "regression = ['bace_r', 'chembl', 'clearance', 'delaney', 'hopv', 'kaggle', 'lipo',\n",
    "                'nci', 'pdbbind', 'ppb', 'qm7', 'qm7b', 'qm8', 'qm9', 'sampl']\n",
    "reg_models = ['tf_regression', 'tf_regression_ft', 'rf_regression', 'graphconvreg',\n",
    "      'dtnn', 'dag_regression', 'xgb_regression', 'weave_regression', 'krr', 'ani']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "cls_tasks = ['bace_c', 'bbbp', 'tox21', 'sider', 'clintox']\n",
    "reg_tasks = ['qm7', 'qm8'] #['lipo', 'qm7', 'qm8', 'delaney', 'sampl']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fingerprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fingerprint_result = dc.molnet.run_benchmark(\n",
    "                                 ckpt = 'fingerprint',\n",
    "                                 arg = {'input':'fingerprint'},\n",
    "                                 datasets = reg_tasks,\n",
    "                                 model = 'tf_regression',\n",
    "                                 split = None,\n",
    "                                 metric =  [dc.metrics.Metric(dc.metrics.mae_score, np.mean)],\n",
    "                                 n_features = 256,\n",
    "                                 featurizer = dc.feat.fingerprints.CircularFingerprint(size=256),\n",
    "                                 out_path= './benchmark',\n",
    "                                 hyper_parameters = None,\n",
    "                                 test = True,\n",
    "                                 reload = False,\n",
    "                                 seed = 123\n",
    "                                 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fingerprint_result = dc.molnet.run_benchmark(\n",
    "                                 ckpt = 'fingerprint',\n",
    "                                 arg = {'input':'fingerprint'},\n",
    "                                 datasets = cls_tasks,\n",
    "                                 model = 'tf',\n",
    "                                 split = None,\n",
    "                                 metric = None,\n",
    "                                 n_features = 256,\n",
    "                                 featurizer = dc.feat.fingerprints.CircularFingerprint(size=256),\n",
    "                                 out_path= './benchmark/',\n",
    "                                 hyper_parameters = None,\n",
    "                                 test = True,\n",
    "                                 reload = False,\n",
    "                                 seed = 123\n",
    "                                 )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## model_8_256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_8_256 = ['model_8_256_001_000004000.tar','model_8_256_005_000020000.tar','model_8_256_010_000040000.tar',\n",
    "               'model_8_256_015_000060000.tar','model_8_256_020_000080000.tar','model_8_256_025_000100000.tar' ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "benchmark_models('model_8_256', model_8_256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_4_256 = ['model_4_256_s_no_auxilary_001_000004000.tar', 'model_4_256_s_no_auxilary_002_000008000.tar', \n",
    "'model_4_256_s_no_auxilary_003_000012000.tar', 'model_4_256_s_no_auxilary_004_000016000.tar']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "benchmark_models('model_4_256', model_4_256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_4_256_fc = [\n",
    "'model_4_256_s_fc_001_000004000.tar',\n",
    "'model_4_256_s_fc_002_000008000.tar',\n",
    "'model_4_256_s_fc_003_000012000.tar',\n",
    "'model_4_256_s_fc_004_000016000.tar',\n",
    "'model_4_256_s_fc_005_000020000.tar',\n",
    "'model_4_256_s_fc_006_000024000.tar',\n",
    "'model_4_256_s_fc_007_000028000.tar',\n",
    "'model_4_256_s_fc_008_000032000.tar']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cls_benchmark_models(dir_path, files, tasks=['sider'], skip=3):\n",
    "    print(\"Total {} results\".format(len(files)//skip))\n",
    "    for i, file in tqdm(enumerate(files)):\n",
    "        if i % skip == 0:\n",
    "            print(\"process\", file)\n",
    "            benchmark(dir_path, file, tasks, 'tf', 'comet', 'cls')\n",
    "\n",
    "def reg_benchmark_models(dir_path, files, tasks=['qm8'], skip=3):\n",
    "    print(\"Total {} results\".format(len(files)//skip))\n",
    "    for i, file in tqdm(enumerate(files)):\n",
    "        if i % skip == 0:\n",
    "            print(\"process\", file)\n",
    "            benchmark(dir_path, file, tasks, 'tf_regression', 'comet', 'reg')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_filelist(dir_path):\n",
    "    file_list = [f for f in listdir(dir_path) if isfile(join(dir_path, f)) if 'tar' in f]\n",
    "    file_list.sort()\n",
    "    return file_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "0it [00:00, ?it/s]\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 7 results\n",
      "process model_4_256_s_fc_001_000004000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: qm8\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/qm8.csv\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "Featurizing sample 8000\n",
      "TIMING: featurizing shard 0 took 99.195 s\n",
      "Loading shard 2 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "Featurizing sample 8000\n",
      "TIMING: featurizing shard 1 took 99.408 s\n",
      "Loading shard 3 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "TIMING: featurizing shard 2 took 65.506 s\n",
      "TIMING: dataset construction took 265.175 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.972 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.262 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.258 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.921 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.117 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.122 s\n",
      "Loading dataset from disk.\n",
      "-----------------------------\n",
      "Start fitting: tf_regression\n",
      "computed_metrics: [0.00829700612105413, 0.00859912275578078, 0.01365075257613988, 0.02795742806554798, 0.008284561998634683, 0.0084335508184599, 0.01077860630664361, 0.021041014210918557, 0.008299764348998557, 0.00844143185921137, 0.010750174007921566, 0.021041439378663273, 0.00789163398055747, 0.007999149465444659, 0.01246538216181497, 0.02456183272884653]\n",
      "computed_metrics: [0.009694500139948155, 0.009664197255711823, 0.016429786462674345, 0.03216183627376514, 0.009683372752577172, 0.00985862895158821, 0.013005644801750545, 0.024557820776882667, 0.009694556472270386, 0.009869760878021147, 0.012978713968493578, 0.02456702434809564, 0.009329701613634762, 0.00909462790743857, 0.014854691934240688, 0.02813186463787126]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "1it [05:58, 358.79s/it]\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.009564908059126001, 0.0094294156278122, 0.01626261029803162, 0.032372075818478965, 0.009726381646059922, 0.009307862499316762, 0.012846826816409213, 0.02457030190703601, 0.009750409586901682, 0.009328142506072232, 0.012816028100452166, 0.024576254862306676, 0.009086381581742927, 0.008734133222058642, 0.015649619733733425, 0.028698140324061833]\n",
      "process model_4_256_s_fc_004_000016000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: qm8\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/qm8.csv\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "Featurizing sample 8000\n",
      "TIMING: featurizing shard 0 took 99.353 s\n",
      "Loading shard 2 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "Featurizing sample 8000\n",
      "TIMING: featurizing shard 1 took 99.452 s\n",
      "Loading shard 3 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "TIMING: featurizing shard 2 took 65.420 s\n",
      "TIMING: dataset construction took 265.290 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.979 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.262 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.259 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.904 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.117 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.117 s\n",
      "Loading dataset from disk.\n",
      "-----------------------------\n",
      "Start fitting: tf_regression\n",
      "computed_metrics: [0.00972991023373246, 0.010071334570826006, 0.0137883312949345, 0.02862437384789778, 0.009992380130528498, 0.010177874634962995, 0.012038377774758607, 0.020689694213807047, 0.009987789516463466, 0.010202364696404181, 0.012055553116245243, 0.02069424054688357, 0.00939934405531062, 0.00950459765373287, 0.013797566026610784, 0.024015007400767386]\n",
      "computed_metrics: [0.010820696136172632, 0.011238325196262007, 0.01600395805933367, 0.033073088795006846, 0.011103405459173647, 0.011337487303923973, 0.01407143996718874, 0.025526429457876157, 0.011090656848222299, 0.011355222045917896, 0.014081080974300888, 0.025526220145325067, 0.010534438873575695, 0.01063994758126581, 0.016441940204511705, 0.02922865809725367]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "4it [12:02, 287.48s/it]\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.011067019434270225, 0.011169165933946956, 0.016216883719514633, 0.03297947779920917, 0.011501217802609251, 0.0114993466720942, 0.013196676350547314, 0.024849603125416158, 0.011502487229798023, 0.01152323231983077, 0.0132173000384011, 0.024851526499907624, 0.010787624831681494, 0.010470300890314697, 0.015029546683516663, 0.028621889737562547]\n",
      "process model_4_256_s_fc_007_000028000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: qm8\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/qm8.csv\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "Featurizing sample 8000\n",
      "TIMING: featurizing shard 0 took 99.829 s\n",
      "Loading shard 2 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "Featurizing sample 8000\n",
      "TIMING: featurizing shard 1 took 100.141 s\n",
      "Loading shard 3 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "TIMING: featurizing shard 2 took 65.674 s\n",
      "TIMING: dataset construction took 266.717 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.962 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.259 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.262 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.910 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.119 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.122 s\n",
      "Loading dataset from disk.\n",
      "-----------------------------\n",
      "Start fitting: tf_regression\n",
      "computed_metrics: [0.008659540888215543, 0.009006732210836962, 0.01346089451007404, 0.02998410703917284, 0.008992391402353582, 0.008790342197406726, 0.013105205462554437, 0.02064662624368447, 0.008995383548773179, 0.00878824688242249, 0.013099848560567966, 0.020616790481342427, 0.008439609684955208, 0.00846688897779466, 0.013742724574246364, 0.02376809390871746]\n",
      "computed_metrics: [0.009832886319997937, 0.009916000903367538, 0.015693124051052616, 0.03553067142379797, 0.010096707259142298, 0.009758465896983395, 0.015305122094811541, 0.025349048722260802, 0.010101494290472251, 0.009758367931785424, 0.015294593598066827, 0.025337067992842396, 0.009496099146700193, 0.009328204663409483, 0.01635743739414534, 0.027929726038953454]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "7it [18:07, 237.78s/it]\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.009352418063662373, 0.009702554961608859, 0.01640175301003344, 0.033557261807831945, 0.009699419072447278, 0.009594723399957433, 0.015423485178659452, 0.02749364721137895, 0.009696420781674897, 0.009592068290309836, 0.01541425840540688, 0.027496639376314472, 0.00904610649018594, 0.009000533932317869, 0.016417793781930767, 0.02939575808006283]\n",
      "process model_4_256_s_fc_010_000040000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: qm8\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/qm8.csv\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "Featurizing sample 8000\n",
      "TIMING: featurizing shard 0 took 99.150 s\n",
      "Loading shard 2 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "Featurizing sample 8000\n",
      "TIMING: featurizing shard 1 took 99.606 s\n",
      "Loading shard 3 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "TIMING: featurizing shard 2 took 65.268 s\n",
      "TIMING: dataset construction took 265.087 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.978 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.256 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.258 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.897 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.121 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.117 s\n",
      "Loading dataset from disk.\n",
      "-----------------------------\n",
      "Start fitting: tf_regression\n",
      "computed_metrics: [0.008158548201852682, 0.008712155417504416, 0.01308978484149153, 0.027764715099972092, 0.00822363231040538, 0.00841295935638397, 0.01088683342557101, 0.02020979215888606, 0.008224271125882022, 0.00841783765879497, 0.010885094040349678, 0.020214147676706106, 0.007794787544098018, 0.008246119226189412, 0.012228506896260366, 0.023400213574435686]\n",
      "computed_metrics: [0.00904128074931357, 0.00974967908954041, 0.016317665419991797, 0.03221414340203951, 0.009184604739373338, 0.009542398275272543, 0.013865766567219485, 0.025166483680957388, 0.009186609544145168, 0.009542412130674407, 0.01386071097558057, 0.025167671889736867, 0.008657733148149927, 0.009210990122230213, 0.015092272742441097, 0.028137655390100928]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "10it [24:15, 203.29s/it]\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.009284849408250577, 0.009731696310265863, 0.015935331271720674, 0.03207175932727116, 0.009251951546989389, 0.009598604586160504, 0.013767920620134075, 0.02498712929488009, 0.009246113459439077, 0.009604057128537695, 0.013767217178523973, 0.02500838713222565, 0.008795766371793088, 0.00917263887514384, 0.015571016709400278, 0.027887338242222226]\n",
      "process model_4_256_s_fc_013_000052000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: qm8\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/qm8.csv\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "Featurizing sample 8000\n",
      "TIMING: featurizing shard 0 took 99.674 s\n",
      "Loading shard 2 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "Featurizing sample 8000\n",
      "TIMING: featurizing shard 1 took 99.808 s\n",
      "Loading shard 3 of size 8192.\n",
      "Featurizing sample 0\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-c563921915fb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mmodel_file_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_filelist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdir_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# cls_benchmark_models(dir_path, model_file_list, tasks=['tox21'])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mreg_benchmark_models\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdir_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_file_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtasks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'qm8'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-21-797438a5e955>\u001b[0m in \u001b[0;36mreg_benchmark_models\u001b[0;34m(dir_path, files, tasks, skip)\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mskip\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"process\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m             \u001b[0mbenchmark\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdir_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtasks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'tf_regression'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'comet'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'reg'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-2-a874cc93b492>\u001b[0m in \u001b[0;36mbenchmark\u001b[0;34m(dir_path, ckpt_file, tasks, _model, featurizer, metric_type)\u001b[0m\n\u001b[1;32m     63\u001b[0m                                  \u001b[0mtest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m                                  \u001b[0mreload\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m                                  \u001b[0mseed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m123\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m                                  )\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/deepchem/molnet/run_benchmark.py\u001b[0m in \u001b[0;36mrun_benchmark\u001b[0;34m(ckpt, arg, datasets, model, split, metric, direction, featurizer, n_features, out_path, hyper_parameters, hyper_param_search, max_iter, search_range, test, reload, seed)\u001b[0m\n\u001b[1;32m    157\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    158\u001b[0m       tasks, all_dataset, transformers = loading_functions[dataset](\n\u001b[0;32m--> 159\u001b[0;31m           featurizer=featurizer, reload=reload)\n\u001b[0m\u001b[1;32m    160\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    161\u001b[0m     \u001b[0mtrain_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_dataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mall_dataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/deepchem/molnet/load_function/qm8_datasets.py\u001b[0m in \u001b[0;36mload_qm8\u001b[0;34m(featurizer, split, reload, move_mean)\u001b[0m\n\u001b[1;32m     73\u001b[0m         tasks=qm8_tasks, smiles_field=\"smiles\", featurizer=featurizer)\n\u001b[1;32m     74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 75\u001b[0;31m   \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeaturize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     76\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0msplit\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/deepchem/data/data_loader.py\u001b[0m in \u001b[0;36mfeaturize\u001b[0;34m(self, input_files, data_dir, shard_size)\u001b[0m\n\u001b[1;32m    232\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m     return DiskDataset.create_dataset(\n\u001b[0;32m--> 234\u001b[0;31m         shard_generator(), data_dir, self.tasks, verbose=self.verbose)\n\u001b[0m\u001b[1;32m    235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mget_shards\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_files\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshard_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/deepchem/data/datasets.py\u001b[0m in \u001b[0;36mcreate_dataset\u001b[0;34m(shard_generator, data_dir, tasks, verbose)\u001b[0m\n\u001b[1;32m    555\u001b[0m     \u001b[0mmetadata_rows\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    556\u001b[0m     \u001b[0mtime1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 557\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mshard_num\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mw\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mids\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshard_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    558\u001b[0m       \u001b[0mbasename\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"shard-%d\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mshard_num\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    559\u001b[0m       metadata_rows.append(\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/deepchem/data/data_loader.py\u001b[0m in \u001b[0;36mshard_generator\u001b[0;34m()\u001b[0m\n\u001b[1;32m    211\u001b[0m           self.get_shards(input_files, shard_size)):\n\u001b[1;32m    212\u001b[0m         \u001b[0mtime1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 213\u001b[0;31m         \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_inds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeaturize_shard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshard\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    214\u001b[0m         \u001b[0mids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mshard\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mid_field\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m         \u001b[0mids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mids\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mvalid_inds\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/deepchem/data/data_loader.py\u001b[0m in \u001b[0;36mfeaturize_shard\u001b[0;34m(self, shard)\u001b[0m\n\u001b[1;32m    254\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mfeaturize_shard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshard\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m     \u001b[0;34m\"\"\"Featurizes a shard of an input dataframe.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 256\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mfeaturize_smiles_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshard\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeaturizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfield\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msmiles_field\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    257\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    258\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/deepchem/data/data_loader.py\u001b[0m in \u001b[0;36mfeaturize_smiles_df\u001b[0;34m(df, featurizer, field, log_every_N, verbose)\u001b[0m\n\u001b[1;32m     74\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mind\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mlog_every_N\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m       \u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Featurizing sample %d\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mind\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 76\u001b[0;31m     \u001b[0mfeatures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeaturizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeaturize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmol\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     77\u001b[0m   valid_inds = np.array(\n\u001b[1;32m     78\u001b[0m       [1 if elt.size > 0 else 0 for elt in features], dtype=bool)\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/deepchem/feat/base_classes.py\u001b[0m in \u001b[0;36mfeaturize\u001b[0;34m(self, mols, verbose, log_every_n)\u001b[0m\n\u001b[1;32m     73\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmol\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmols\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     74\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mmol\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 75\u001b[0;31m         \u001b[0mfeatures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_featurize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     76\u001b[0m       \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m         \u001b[0mfeatures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-2-a874cc93b492>\u001b[0m in \u001b[0;36m_featurize\u001b[0;34m(self, mol)\u001b[0m\n\u001b[1;32m     32\u001b[0m             \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m             \u001b[0mA\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m             \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmolvec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmolvec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/urp/comet/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_X, A)\u001b[0m\n\u001b[1;32m    187\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    188\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_X\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 189\u001b[0;31m         \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmolvec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_X\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    190\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mact\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfc1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    191\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mact\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfc2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/urp/comet/model.py\u001b[0m in \u001b[0;36mencoder\u001b[0;34m(self, input_X, A)\u001b[0m\n\u001b[1;32m    196\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_embed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_X\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    197\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgconvs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 198\u001b[0;31m             \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    199\u001b[0m         \u001b[0mmolvec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmolvec\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/urp/comet/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, X, A)\u001b[0m\n\u001b[1;32m    135\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    136\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 137\u001b[0;31m         \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgconv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    138\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    139\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msc_type\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'no'\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# no skip-connection\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/urp/comet/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, X, A)\u001b[0m\n\u001b[1;32m     75\u001b[0m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 77\u001b[0;31m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     78\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/urp/comet/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, X, A)\u001b[0m\n\u001b[1;32m     35\u001b[0m         \u001b[0mlist_X_head\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_attn_heads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 37\u001b[0;31m             \u001b[0mX_projected\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprojection\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m             \u001b[0mattn_matrix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattn_coeff\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_projected\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcoef_matrix\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m             \u001b[0mX_head\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mattn_matrix\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_projected\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     65\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mweak_script_method\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 67\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/deepchem/lib/python3.5/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m   1352\u001b[0m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unwrap_optional\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1353\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1354\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1355\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1356\u001b[0m             \u001b[0moutput\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unwrap_optional\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "dir_path = './runs/model_4_256_s_fc_train'\n",
    "model_file_list = get_filelist(dir_path)\n",
    "# cls_benchmark_models(dir_path, model_file_list, tasks=['tox21'])\n",
    "reg_benchmark_models(dir_path, model_file_list, tasks=['qm8'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "0it [00:00, ?it/s]\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 8 results\n",
      "process model_8_256_001_000004000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: sider\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/sider.csv.gz\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "TIMING: featurizing shard 0 took 31.311 s\n",
      "TIMING: dataset construction took 31.387 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.082 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.065 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.020 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.029 s\n",
      "Loading dataset from disk.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "1it [00:33, 33.45s/it]\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.6615958102279729, 0.6188332215729476, 0.7792006754855052, 0.6401828500616311, 0.6463294517224649, 0.6355700897944963, 0.7209765602279998, 0.6552251678748178, 0.6284469065974508, 0.648802932998696, 0.6465173739116794, 0.6188926818860678, 0.662612919792219, 0.6466321243523316, 0.6245822351438741, 0.6813322368421053, 0.6892389424264889, 0.6134361344115726, 0.604423557495058, 0.6356802102284212, 0.657777116750185, 0.6473157385472623, 0.6754182754182754, 0.6323379958979286, 0.6648711169861203, 0.7418959330143541, 0.6124430169912971]\n",
      "computed_metrics: [0.6441851265822787, 0.49905704856199906, 0.6830985915492958, 0.5433826469412235, 0.5927873779113448, 0.5927318295739348, 0.5692137320044297, 0.5430508474576272, 0.5392287234042553, 0.6127508854781581, 0.5596846846846847, 0.5695592286501377, 0.6386554621848739, 0.5040650406504066, 0.5498525073746312, 0.6118332586284178, 0.6033464566929134, 0.5788446526151444, 0.6451162790697674, 0.5429033998920669, 0.5437209302325581, 0.5681719496309162, 0.6653944020356235, 0.5507955936352509, 0.6147467503361721, 0.5152671755725191, 0.5758738277919864]\n",
      "computed_metrics: [0.6890822784810128, 0.6309676044330776, 0.5, 0.4576470588235294, 0.6863278886872353, 0.5626984126984127, 0.7217027559055118, 0.5, 0.6290249433106576, 0.5522300469483568, 0.5120808678500985, 0.5536458333333334, 0.6125423728813559, 0.44747899159663873, 0.5378359264497878, 0.5423728813559322, 0.5821661998132586, 0.484593837535014, 0.5103058510638299, 0.5872230080150873, 0.5725141120277899, 0.633516931864545, 0.4925925925925926, 0.5353107344632768, 0.4389937106918239, 0.7109375, 0.5036908380373426]\n",
      "process model_8_256_004_000016000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: sider\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/sider.csv.gz\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "TIMING: featurizing shard 0 took 31.212 s\n",
      "TIMING: dataset construction took 31.284 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.073 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.067 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.021 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.020 s\n",
      "Loading dataset from disk.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "4it [01:06, 26.74s/it]\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.6957701786814541, 0.6288842874722749, 0.7565906745473308, 0.6493912688106835, 0.6593377001455605, 0.6458888348492657, 0.7344413424067935, 0.6538133765940407, 0.6431139348390347, 0.6773086292168007, 0.6564671738959917, 0.6273735865158949, 0.6888423037742362, 0.6416038624587848, 0.6342281879194631, 0.6940480679217808, 0.7119718309859154, 0.6263750062785675, 0.6450971616127709, 0.635049524964625, 0.6531321267037351, 0.6603049614627473, 0.6882147453576025, 0.6084146356486015, 0.661753690240141, 0.7414972089314196, 0.625880646498135]\n",
      "computed_metrics: [0.6817642405063291, 0.5386610089580386, 0.6690140845070423, 0.5669732107157137, 0.6435011269722013, 0.5100250626566416, 0.5370985603543743, 0.5420338983050847, 0.6046099290780143, 0.6119637937819756, 0.6007882882882882, 0.6212121212121211, 0.6512605042016806, 0.590650406504065, 0.5943952802359882, 0.6777229941730165, 0.6658464566929134, 0.5714285714285714, 0.6237209302325581, 0.5890447922288181, 0.5604651162790698, 0.6291793313069909, 0.5896946564885497, 0.5903712770297838, 0.6564320932317347, 0.5858778625954199, 0.527919863597613]\n",
      "computed_metrics: [0.6789952531645569, 0.6802003410059676, 0.3900709219858156, 0.49058823529411766, 0.7894736842105262, 0.6459183673469386, 0.7133366141732284, 0.4981325863678805, 0.6092970521541949, 0.6003521126760565, 0.5044378698224852, 0.6338541666666666, 0.5728813559322035, 0.4372082166199813, 0.5911126826968411, 0.6454802259887007, 0.6143790849673203, 0.47805788982259567, 0.6266622340425532, 0.6614804337576614, 0.579244463742944, 0.6496328029375765, 0.36851851851851847, 0.5732445520581114, 0.48700209643605874, 0.7307291666666667, 0.5742509769865393]\n",
      "process model_8_256_007_000028000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: sider\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/sider.csv.gz\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "TIMING: featurizing shard 0 took 31.149 s\n",
      "TIMING: dataset construction took 31.222 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.083 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.066 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.020 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.027 s\n",
      "Loading dataset from disk.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "7it [01:39, 22.04s/it]\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.6720086260012323, 0.6201485553434973, 0.7887231447602965, 0.655168777066621, 0.6586074721009219, 0.6383094752429969, 0.7193382965935748, 0.6556820748711419, 0.6445821872699802, 0.6739253463251396, 0.6409051690328653, 0.61163857478131, 0.6828732351678328, 0.6716556759302872, 0.6306913385684139, 0.6714280606407322, 0.7013466765505312, 0.6317745743131247, 0.6346872552310618, 0.6374833232261978, 0.645748956974731, 0.6489573254145673, 0.6868909726052583, 0.6218826413397636, 0.6560549313358303, 0.7157695374800639, 0.6172891628677994]\n",
      "computed_metrics: [0.6376582278481012, 0.5099009900990099, 0.823943661971831, 0.5479808076769292, 0.6348610067618332, 0.4842105263157895, 0.5337763012181618, 0.5650847457627118, 0.5693705673758865, 0.5852026761117671, 0.548704954954955, 0.5550964187327823, 0.6418067226890756, 0.4764227642276423, 0.5454277286135693, 0.6131779471089198, 0.6899606299212598, 0.5878220140515222, 0.5874418604651163, 0.5906637884511602, 0.5013953488372093, 0.6189752496743378, 0.6723918575063614, 0.5771113831089352, 0.6284177498879426, 0.4821882951653944, 0.5196078431372548]\n",
      "computed_metrics: [0.63310917721519, 0.5832267689684569, 0.41843971631205673, 0.4868627450980392, 0.7528735632183908, 0.6168934240362811, 0.7113681102362205, 0.44864612511671337, 0.6002267573696145, 0.602112676056338, 0.4869329388560158, 0.6354166666666666, 0.5745762711864406, 0.488562091503268, 0.5262847713342762, 0.5639628732849072, 0.6661998132586369, 0.5695611577964519, 0.6259973404255319, 0.6152758132956153, 0.5564481111593573, 0.6273969808241535, 0.43981481481481477, 0.5232041969330105, 0.48511530398322855, 0.7197916666666666, 0.5523230568823274]\n",
      "process model_8_256_010_000040000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: sider\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/sider.csv.gz\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "TIMING: featurizing shard 0 took 31.421 s\n",
      "TIMING: dataset construction took 31.491 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.078 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.065 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.020 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.020 s\n",
      "Loading dataset from disk.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "10it [02:13, 18.78s/it]\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.6704343807763401, 0.6161362346820723, 0.7785908621821935, 0.6498658616166477, 0.639061135371179, 0.6233240659412452, 0.7121551406580198, 0.6515647781177485, 0.6442490218107155, 0.6621545975738785, 0.6424778413993255, 0.604224450608065, 0.6656900630582847, 0.6672868582195007, 0.6306234093235152, 0.6845924563137091, 0.7025574499629355, 0.632583253804812, 0.614039013837604, 0.6350171821305843, 0.6283969826632801, 0.6504187381135098, 0.6928479499908071, 0.6230067074410097, 0.6592898582654034, 0.7045454545454546, 0.623224892941014]\n",
      "computed_metrics: [0.626384493670886, 0.49929278642149927, 0.8873239436619719, 0.5479808076769292, 0.581893313298272, 0.5095238095238095, 0.4939091915836102, 0.5623728813559322, 0.5538563829787234, 0.6225895316804407, 0.5878378378378378, 0.37258953168044073, 0.6505602240896359, 0.5378048780487805, 0.5292035398230088, 0.605333930972658, 0.6500984251968505, 0.5960187353629977, 0.6018604651162791, 0.5677280086346465, 0.5062790697674417, 0.6131133304385585, 0.7143765903307888, 0.5822113423092614, 0.6212460779919318, 0.4611959287531807, 0.5091645353793691]\n",
      "computed_metrics: [0.6473496835443038, 0.6183930093776642, 0.48936170212765956, 0.4645098039215686, 0.7704174228675136, 0.5908163265306123, 0.7320374015748031, 0.4738562091503268, 0.608843537414966, 0.5757042253521127, 0.479043392504931, 0.48020833333333335, 0.5572881355932204, 0.45821661998132585, 0.4633427628477133, 0.5851493139628732, 0.5986227824463118, 0.5784313725490196, 0.6463874113475178, 0.5787364450730788, 0.49500651324359535, 0.6375968992248062, 0.47962962962962963, 0.5276432606941082, 0.4561844863731656, 0.6859375000000001, 0.5555796786799826]\n",
      "process model_8_256_013_000052000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: sider\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/sider.csv.gz\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "TIMING: featurizing shard 0 took 31.403 s\n",
      "TIMING: dataset construction took 31.475 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.080 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.072 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.021 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.021 s\n",
      "Loading dataset from disk.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "13it [02:46, 16.49s/it]\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.6833518176216883, 0.6191095521970127, 0.7701003846514682, 0.6519520924928646, 0.6506477438136826, 0.6267249326722755, 0.709789836949146, 0.6484434359406124, 0.6409677294386549, 0.6605152432272828, 0.6390187465683583, 0.6268802005547258, 0.6743742580759224, 0.6800459255770137, 0.6091713537845647, 0.6930956807780321, 0.6922905856189769, 0.6379225475915415, 0.6177874752899929, 0.6259571457448958, 0.63922385190948, 0.6485502652564146, 0.6928571428571428, 0.6296433168880924, 0.6536975839024748, 0.7031798245614035, 0.6090343970161625]\n",
      "computed_metrics: [0.6669303797468353, 0.5299387081565299, 0.9225352112676056, 0.5309876049580169, 0.6555221637866266, 0.5258145363408522, 0.5033222591362125, 0.5396610169491526, 0.5682624113475176, 0.5883510428964975, 0.5869932432432432, 0.47451790633608815, 0.5973389355742297, 0.5288617886178861, 0.5392330383480826, 0.684670551322277, 0.6835629921259843, 0.5323965651834504, 0.6234883720930232, 0.5717754991905019, 0.5339534883720931, 0.653061224489796, 0.6450381679389312, 0.584047327621379, 0.639399372478709, 0.5534351145038168, 0.5300511508951407]\n",
      "computed_metrics: [0.6578322784810127, 0.6203111679454391, 0.5567375886524824, 0.46529411764705886, 0.765577737447066, 0.6060090702947845, 0.733513779527559, 0.48319327731092443, 0.6226757369614513, 0.5764866979655712, 0.45290927021696253, 0.5718749999999999, 0.5725423728813559, 0.40452847805788983, 0.4911598302687411, 0.6408393866020985, 0.6223155929038282, 0.6377217553688141, 0.5943040780141844, 0.6628948609146628, 0.5660008684324794, 0.6294369645042841, 0.5351851851851852, 0.5153349475383373, 0.4331236897274633, 0.7395833333333333, 0.5277898393399914]\n",
      "process model_8_256_016_000064000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: sider\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/sider.csv.gz\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "TIMING: featurizing shard 0 took 31.751 s\n",
      "TIMING: dataset construction took 31.825 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.078 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.064 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.020 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.022 s\n",
      "Loading dataset from disk.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "16it [03:20, 14.92s/it]\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.6834288354898337, 0.6132292365169076, 0.7887231447602965, 0.6437752539401089, 0.6259946627850558, 0.6241007847037218, 0.7036051493825006, 0.6501273179045937, 0.6304342773021345, 0.6499532491818607, 0.6318260255706329, 0.6235065073607852, 0.6747699509172498, 0.6742934526613282, 0.616865472923403, 0.686962047534845, 0.6932913269088213, 0.6423878647847707, 0.6233374361269628, 0.6176248231251263, 0.6249113333180789, 0.6406159287311067, 0.7037874609303181, 0.6260309071983345, 0.6518396122493942, 0.7168361244019139, 0.6096456692913386]\n",
      "computed_metrics: [0.6550632911392404, 0.5256954266855257, 0.823943661971831, 0.5363854458216712, 0.6697971450037565, 0.5233082706766918, 0.5343300110741971, 0.5416949152542373, 0.5585106382978724, 0.6395120031483668, 0.5557432432432432, 0.5020661157024793, 0.6274509803921569, 0.4821138211382114, 0.4994100294985251, 0.6454504706409683, 0.6279527559055118, 0.5480093676814988, 0.5986046511627907, 0.5377765785213168, 0.5244186046511627, 0.6326530612244898, 0.61323155216285, 0.5744594043247655, 0.5916629314208874, 0.532442748091603, 0.5206734867860188]\n",
      "computed_metrics: [0.6774129746835443, 0.5335677749360614, 0.3971631205673759, 0.4576470588235294, 0.7326073805202662, 0.5726757369614512, 0.6990649606299213, 0.4584500466853408, 0.6233560090702948, 0.5608372456964006, 0.43762327416173574, 0.565625, 0.5532203389830508, 0.5063025210084033, 0.5001178689297501, 0.5857546408393866, 0.6022408963585435, 0.5681605975723623, 0.5971852836879432, 0.5412541254125414, 0.5256187581415545, 0.6420848633210934, 0.5055555555555555, 0.5314769975786924, 0.4050314465408805, 0.7083333333333333, 0.5130264871906209]\n",
      "process model_8_256_019_000076000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: sider\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/sider.csv.gz\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "TIMING: featurizing shard 0 took 31.231 s\n",
      "TIMING: dataset construction took 31.304 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.081 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.067 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.020 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.020 s\n",
      "Loading dataset from disk.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "19it [03:53, 13.78s/it]\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.6742914356130623, 0.6160514932906924, 0.7733370860305845, 0.6443651991641892, 0.6381052886948083, 0.6238866426150016, 0.7215000290816029, 0.6496088054705629, 0.6311858365939642, 0.6446138136364755, 0.6386344026982509, 0.6204928525709409, 0.6683167900049671, 0.6771431935939708, 0.6112681031437655, 0.686962047534845, 0.6952063256733383, 0.6465492993118691, 0.6084181865652158, 0.6158055387103296, 0.6376221312038075, 0.6480773080644623, 0.7058742415885273, 0.6301822526900596, 0.6570904017037527, 0.7174441786283892, 0.6224409448818897]\n",
      "computed_metrics: [0.6281645569620253, 0.4884488448844884, 0.7112676056338028, 0.5249900039984006, 0.6149511645379413, 0.5152882205513785, 0.566998892580288, 0.5315254237288136, 0.544104609929078, 0.6107831562377017, 0.5737612612612611, 0.4511019283746556, 0.6446078431372548, 0.5317073170731708, 0.5339233038348082, 0.6544150605109815, 0.6486220472440944, 0.5569867291178766, 0.5490697674418604, 0.5045871559633028, 0.5053488372093023, 0.6118106817194963, 0.6513994910941476, 0.5232558139534883, 0.5264455401165397, 0.5597964376590331, 0.5008525149190111]\n",
      "computed_metrics: [0.6710838607594937, 0.5393222506393862, 0.2446808510638298, 0.4809803921568627, 0.7192982456140351, 0.568140589569161, 0.7576279527559056, 0.4780578898225957, 0.6335600907029478, 0.5535993740219093, 0.498767258382643, 0.6359375, 0.543728813559322, 0.48762838468720826, 0.5272277227722773, 0.6353914447134787, 0.6311858076563959, 0.6638655462184874, 0.5592863475177305, 0.6218175388967468, 0.5851063829787235, 0.5906772745818033, 0.500925925925926, 0.5260290556900726, 0.4220125786163522, 0.7604166666666667, 0.5275727312201477]\n",
      "process model_8_256_022_000088000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: sider\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/sider.csv.gz\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "TIMING: featurizing shard 0 took 31.644 s\n",
      "TIMING: dataset construction took 31.714 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.080 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.066 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.020 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.027 s\n",
      "Loading dataset from disk.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "22it [04:27, 13.01s/it]\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.6817629390018485, 0.6264710000221063, 0.7655033305188105, 0.6512006538834216, 0.637493934983018, 0.6339077664617194, 0.7291484906648054, 0.6538185103805163, 0.6311393483903459, 0.6520847174036072, 0.6345399639187386, 0.6456688713462769, 0.6765589876998459, 0.6520960904380593, 0.6155929317356374, 0.6953970121697525, 0.6878675562144798, 0.6320608769903059, 0.6212226325015852, 0.6317828987265008, 0.6428925550496152, 0.658139534883721, 0.6849880492737636, 0.6303885881113842, 0.6696151868987295, 0.7300189393939394, 0.6235426163834784]\n",
      "computed_metrics: [0.6635680379746836, 0.5794436586515794, 0.823943661971831, 0.5561775289884046, 0.6716754320060105, 0.5260651629072681, 0.5681063122923589, 0.512542372881356, 0.5378989361702127, 0.6499409681227863, 0.5498310810810811, 0.6260330578512396, 0.6358543417366946, 0.49959349593495933, 0.5014749262536873, 0.6671896010757508, 0.6402559055118111, 0.539032006245121, 0.5520930232558139, 0.5593631948192122, 0.5702325581395349, 0.650238818931828, 0.6202290076335877, 0.5775193798449613, 0.5392200806813088, 0.6431297709923665, 0.5701193520886616]\n",
      "computed_metrics: [0.6710838607594936, 0.601768968456948, 0.6134751773049645, 0.4950980392156863, 0.7416817906836055, 0.5837868480725623, 0.7123523622047244, 0.5247432306255836, 0.6374149659863946, 0.6007433489827856, 0.47386587771203154, 0.6010416666666667, 0.5972881355932204, 0.4969654528478058, 0.5227487034417728, 0.6123890234059726, 0.6031746031746031, 0.6017740429505136, 0.5546320921985817, 0.6296558227251297, 0.547112462006079, 0.6161770705834353, 0.46759259259259256, 0.533091202582728, 0.4171907756813417, 0.71875, 0.5186712983065567]\n",
      "process model_8_256_025_000100000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: sider\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/sider.csv.gz\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "TIMING: featurizing shard 0 took 30.922 s\n",
      "TIMING: dataset construction took 30.996 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.078 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.073 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.031 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.020 s\n",
      "Loading dataset from disk.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "25it [05:00, 12.40s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.6879975354282193, 0.6360025643481912, 0.7665353222628764, 0.6539526330013381, 0.6337481804949054, 0.6402485500039925, 0.7378245022198957, 0.6627102285561739, 0.6438345019951187, 0.6533396077852416, 0.6506245587889246, 0.6161635018846454, 0.6782975105026983, 0.6614107395195479, 0.606350025813113, 0.7069654280216351, 0.690449715838893, 0.6336882816816515, 0.6324792062959232, 0.6369820092985647, 0.6418094867629225, 0.6621901171132094, 0.7016731016731017, 0.6354114697857192, 0.6575420430344422, 0.7406598883572568, 0.6245959386655615]\n",
      "computed_metrics: [0.6615901898734178, 0.5304101838755304, 0.8309859154929577, 0.5723710515793683, 0.6040570999248686, 0.5338345864661654, 0.5509413067552602, 0.5372881355932203, 0.53302304964539, 0.6188508461235733, 0.5230855855855856, 0.559228650137741, 0.5927871148459384, 0.49796747967479676, 0.5448377581120945, 0.6613626176602421, 0.6062992125984252, 0.5362997658079625, 0.6313953488372093, 0.5860766324878575, 0.5197674418604651, 0.6131133304385584, 0.6055979643765903, 0.5528355773153815, 0.5945764231286419, 0.5916030534351144, 0.5394288150042625]\n",
      "computed_metrics: [0.6629746835443038, 0.5418797953964195, 0.6134751773049645, 0.4886274509803922, 0.6799758015728977, 0.5801587301587301, 0.6572342519685039, 0.4915966386554622, 0.6365079365079365, 0.573356807511737, 0.48742603550295854, 0.6708333333333334, 0.5396610169491525, 0.48389355742296924, 0.5647100424328146, 0.6202582728006456, 0.5877684407096172, 0.6325863678804855, 0.5072030141843972, 0.5827439886845827, 0.5586191923577941, 0.6088331293349654, 0.45555555555555555, 0.5028248587570622, 0.4490566037735849, 0.7317708333333334, 0.4826313504125054]\n"
     ]
    }
   ],
   "source": [
    "dir_path = './runs/model_8_256_train'\n",
    "model_file_list = get_filelist(dir_path)\n",
    "cls_benchmark_models(dir_path, model_file_list, tasks=['sider'])\n",
    "# reg_benchmark_models(dir_path, model_file_list, tasks=['qm8'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "0it [00:00, ?it/s]\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['model_4_256_s_no_auxilary_001_000004000.tar', 'model_4_256_s_no_auxilary_002_000008000.tar', 'model_4_256_s_no_auxilary_003_000012000.tar', 'model_4_256_s_no_auxilary_004_000016000.tar', 'model_4_256_s_no_auxilary_005_000020000.tar', 'model_4_256_s_no_auxilary_006_000024000.tar']\n",
      "Total 0 results\n",
      "process model_4_256_s_no_auxilary_006_000024000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: sider\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/sider.csv.gz\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "TIMING: featurizing shard 0 took 17.236 s\n",
      "TIMING: dataset construction took 17.315 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.084 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.068 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.022 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.022 s\n",
      "Loading dataset from disk.\n",
      "computed_metrics: [0.6851139864448552, 0.6318502361705733, 0.7817337461300309, 0.6665886664601308, 0.6728748180494906, 0.6578535703656385, 0.7595484596444289, 0.6764328398053269, 0.6488668500368031, 0.6830017716099506, 0.7016236567573928, 0.675058672925112, 0.7042701150876839, 0.6754651436646255, 0.6431269190011684, 0.6985597175993343, 0.7068692858907832, 0.6547239941734893, 0.628953601133863, 0.6369415807560137, 0.673588388287608, 0.6668612992559475, 0.7183857326714469, 0.6499658160719896, 0.671506205478446, 0.7658094098883572, 0.6340067688907307]\n",
      "computed_metrics: [0.6590189873417722, 0.4764262140499764, 0.704225352112676, 0.5797680927628949, 0.6990984222389182, 0.5320802005012532, 0.5852713178294574, 0.5277966101694915, 0.5921985815602837, 0.6151121605667061, 0.6213400900900901, 0.668732782369146, 0.680672268907563, 0.5955284552845528, 0.6002949852507375, 0.6169878978036754, 0.6373031496062993, 0.5163934426229508, 0.5388372093023255, 0.5920129519697788, 0.5951162790697675, 0.5816326530612245, 0.6132315521628499, 0.5283557731538148, 0.6125056028686688, 0.6361323155216285, 0.49232736572890023]\n",
      "computed_metrics: [0.6532832278481013, 0.6137041773231031, 0.36524822695035464, 0.47019607843137257, 0.7199032062915911, 0.5885487528344671, 0.7389271653543308, 0.434640522875817, 0.6129251700680273, 0.5766823161189358, 0.5069033530571991, 0.5562499999999999, 0.6020338983050848, 0.4740896358543418, 0.5717821782178217, 0.5599273607748184, 0.5639589169000934, 0.5214752567693743, 0.6503767730496455, 0.6084394153701084, 0.5794615718627878, 0.6137290901672786, 0.45, 0.5328894269572235, 0.5163522012578616, 0.6317708333333334, 0.6098567086409031]\n",
      "-------------------------------------\n",
      "Benchmark on dataset: tox21\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/tox21.csv.gz\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "TIMING: featurizing shard 0 took 89.431 s\n",
      "TIMING: dataset construction took 89.825 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.387 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.325 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.087 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.086 s\n",
      "Loading dataset from disk.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "1it [01:56, 116.30s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "0it [00:00, ?it/s]\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.8496779086659647, 0.8913199462106616, 0.8836952079632604, 0.8263722591606577, 0.7379901444714105, 0.8250580986290856, 0.8563595517112346, 0.7735099964014629, 0.8439690151235706, 0.8011255486355673, 0.857122644563204, 0.8184145829837222]\n",
      "computed_metrics: [0.7628259041211102, 0.789021164021164, 0.8415783006416967, 0.7586426029486527, 0.6563636363636363, 0.7325938163795138, 0.7398753894080996, 0.7605157471993236, 0.7510713444784554, 0.656665767709457, 0.8063666195597499, 0.7733850129198967]\n",
      "computed_metrics: [0.8035417819590481, 0.8191985612319452, 0.894658279134236, 0.7765333891563744, 0.7043072833830804, 0.7375208541875209, 0.7560410460112545, 0.7252862423368194, 0.7818083010665036, 0.725609756097561, 0.8288524692384436, 0.7973484848484849]\n",
      "Total 0 results\n",
      "process model_4_256_s_no_auxilary_006_000024000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: qm8\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/qm8.csv\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "Featurizing sample 8000\n",
      "TIMING: featurizing shard 0 took 90.723 s\n",
      "Loading shard 2 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "Featurizing sample 8000\n",
      "TIMING: featurizing shard 1 took 91.065 s\n",
      "Loading shard 3 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "TIMING: featurizing shard 2 took 60.436 s\n",
      "TIMING: dataset construction took 243.277 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.962 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.260 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.262 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.900 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.121 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.118 s\n",
      "Loading dataset from disk.\n",
      "-----------------------------\n",
      "Start fitting: tf_regression\n",
      "computed_metrics: [0.009404590299796843, 0.009457525808658005, 0.014513049347333709, 0.028652037342950246, 0.009385724124174696, 0.009326936946143357, 0.011931527533686103, 0.021767991797919207, 0.009411181467653185, 0.009301450813469135, 0.011937670027704357, 0.021755054446762505, 0.008899244502211913, 0.008797366220200867, 0.015024605316850508, 0.025541459909248585]\n",
      "computed_metrics: [0.01029839568546022, 0.010276520757667216, 0.01626962647064543, 0.03273736324902603, 0.010428984119107816, 0.010280674279131784, 0.013765346956542118, 0.02597613265008788, 0.010447207864237334, 0.01025719676278053, 0.013763082831003451, 0.025962976576051514, 0.00984839089620781, 0.009689220581655401, 0.01699092577505753, 0.029996185961884688]\n",
      "computed_metrics: [0.010332427108485689, 0.010823781377423717, 0.01751361769702413, 0.03400961848630955, 0.010422462885572845, 0.010521686032823425, 0.01451073626785069, 0.02614454447725698, 0.01043669817455604, 0.010505708567223528, 0.014525652927334239, 0.026149130453214372, 0.0097695406860797, 0.009948978769747336, 0.01792776178824691, 0.030609419427414283]\n",
      "-------------------------------------\n",
      "Benchmark on dataset: qm7\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/qm7.csv\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "TIMING: featurizing shard 0 took 76.987 s\n",
      "TIMING: dataset construction took 77.283 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.278 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.084 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.080 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.267 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.035 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.036 s\n",
      "Loading dataset from disk.\n",
      "-----------------------------\n",
      "Start fitting: tf_regression\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "1it [07:35, 455.23s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [109.01008632797965]\n",
      "computed_metrics: [110.73501766565828]\n",
      "computed_metrics: [113.36072444954212]\n"
     ]
    }
   ],
   "source": [
    "dir_path = './runs/model_4_256_s_no_auxilary_train'\n",
    "model_file_list = get_filelist(dir_path)\n",
    "print(model_file_list)\n",
    "cls_benchmark_models(dir_path, [model_file_list[-1]], tasks=['sider', 'tox21'])\n",
    "reg_benchmark_models(dir_path, [model_file_list[-1]], tasks=['qm8', 'qm7'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "0it [00:00, ?it/s]\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['model_8_256_001_000004000.tar', 'model_8_256_002_000008000.tar', 'model_8_256_003_000012000.tar', 'model_8_256_004_000016000.tar', 'model_8_256_005_000020000.tar', 'model_8_256_006_000024000.tar', 'model_8_256_007_000028000.tar', 'model_8_256_008_000032000.tar', 'model_8_256_009_000036000.tar', 'model_8_256_010_000040000.tar', 'model_8_256_011_000044000.tar', 'model_8_256_012_000048000.tar', 'model_8_256_013_000052000.tar', 'model_8_256_014_000056000.tar', 'model_8_256_015_000060000.tar', 'model_8_256_016_000064000.tar', 'model_8_256_017_000068000.tar', 'model_8_256_018_000072000.tar', 'model_8_256_019_000076000.tar', 'model_8_256_020_000080000.tar', 'model_8_256_021_000084000.tar', 'model_8_256_022_000088000.tar', 'model_8_256_023_000092000.tar', 'model_8_256_024_000096000.tar', 'model_8_256_025_000100000.tar']\n",
      "Total 0 results\n",
      "process model_8_256_025_000100000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: sider\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/sider.csv.gz\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "TIMING: featurizing shard 0 took 31.484 s\n",
      "TIMING: dataset construction took 31.559 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.076 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.064 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.020 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.020 s\n",
      "Loading dataset from disk.\n",
      "computed_metrics: [0.6905853357979052, 0.6388285055302969, 0.7582324795947087, 0.6496153820801667, 0.6377753517709849, 0.6425932243991318, 0.7394046026483645, 0.6704160420559788, 0.6457482663774067, 0.6535456804704609, 0.6427641383637932, 0.6364790022046796, 0.6774135158571801, 0.6535150730098916, 0.6115307628907064, 0.7106937148949448, 0.6868297504324191, 0.6408558943191521, 0.6291559434560442, 0.6369173236304833, 0.6376678946525409, 0.6633979513529746, 0.7021695164552307, 0.6361475021095487, 0.6556510244547258, 0.7342404306220096, 0.6269417391904959]\n",
      "computed_metrics: [0.651503164556962, 0.5325318246110325, 0.6901408450704225, 0.5667732906837265, 0.615702479338843, 0.5335839598997493, 0.5658914728682171, 0.5149152542372881, 0.5254875886524824, 0.6221959858323495, 0.5199887387387387, 0.6253443526170799, 0.5822829131652661, 0.5044715447154472, 0.5294985250737463, 0.6786194531600178, 0.5915354330708662, 0.524199843871975, 0.6311627906976744, 0.5701565029681598, 0.49976744186046507, 0.6072514112027789, 0.6017811704834606, 0.5746634026927785, 0.6006275212909009, 0.5833333333333333, 0.5398550724637681]\n",
      "computed_metrics: [0.6722705696202531, 0.5651108269394713, 0.39361702127659576, 0.4827450980392157, 0.6872353297035692, 0.5905895691609977, 0.6572342519685039, 0.49859943977591037, 0.6324263038548752, 0.5670970266040689, 0.43639053254437876, 0.5708333333333333, 0.5966101694915253, 0.4736227824463119, 0.5496228194247996, 0.6446731234866828, 0.607843137254902, 0.4901960784313726, 0.5391179078014184, 0.5959453088165959, 0.546461137646548, 0.5870053039575684, 0.462037037037037, 0.5153349475383373, 0.449685534591195, 0.7510416666666666, 0.49283543204515856]\n",
      "-------------------------------------\n",
      "Benchmark on dataset: tox21\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/tox21.csv.gz\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "TIMING: featurizing shard 0 took 168.321 s\n",
      "TIMING: dataset construction took 168.678 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.384 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.324 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.088 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.088 s\n",
      "Loading dataset from disk.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "1it [03:30, 210.46s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "0it [00:00, ?it/s]\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [0.8415428465744517, 0.8862235067437378, 0.8575353924260067, 0.7951018933841447, 0.7281757113674732, 0.8148050231328487, 0.8265620246758038, 0.7362271430125614, 0.8236217054209614, 0.7767671703415853, 0.8244330731902234, 0.7992463780276333]\n",
      "computed_metrics: [0.7635185276802058, 0.8000992063492063, 0.8183452418877712, 0.7015760040671073, 0.6780909090909091, 0.7425851624577138, 0.7446348217376255, 0.7593414594048709, 0.7868142218036261, 0.7019732110751529, 0.7628685717023163, 0.788975021533161]\n",
      "computed_metrics: [0.7895591219332225, 0.8861349969088967, 0.8815438746134983, 0.7628741888214361, 0.7294167628924377, 0.7904738071404738, 0.655246607083747, 0.7008992967904797, 0.7914543848923308, 0.6984591298615689, 0.7865345443964084, 0.7820583887657059]\n",
      "Total 0 results\n",
      "process model_8_256_025_000100000.tar\n",
      "-------------------------------------\n",
      "Benchmark on dataset: qm8\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/qm8.csv\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "Featurizing sample 8000\n",
      "TIMING: featurizing shard 0 took 175.965 s\n",
      "Loading shard 2 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "Featurizing sample 7000\n",
      "Featurizing sample 8000\n",
      "TIMING: featurizing shard 1 took 176.152 s\n",
      "Loading shard 3 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "TIMING: featurizing shard 2 took 115.912 s\n",
      "TIMING: dataset construction took 469.084 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.971 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.259 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.258 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.908 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.117 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.122 s\n",
      "Loading dataset from disk.\n",
      "-----------------------------\n",
      "Start fitting: tf_regression\n",
      "computed_metrics: [0.013316593358605034, 0.011848579195075126, 0.018722369551312353, 0.036640748926261145, 0.013804495927468668, 0.012529123508691599, 0.016501592853316698, 0.028865621454723732, 0.013798880117282215, 0.012517703907986719, 0.016488987846560603, 0.028901731500757956, 0.013025504416376402, 0.011523018938519593, 0.01899583015087123, 0.03407720046641989]\n",
      "computed_metrics: [0.014158269168571632, 0.012472306072878617, 0.01944700037424681, 0.037369376139014955, 0.014795192832937046, 0.013268984702188798, 0.016754804003551747, 0.030774823453302957, 0.014782356123616495, 0.013253278561409735, 0.016742905889018132, 0.03081485414341149, 0.014191238636773348, 0.012269244406842204, 0.01898953686085083, 0.0347408846211657]\n",
      "computed_metrics: [0.014708846072474505, 0.012763885680930176, 0.02009904674060781, 0.03833293688822087, 0.015141256382703418, 0.013587071990838669, 0.018639718734583227, 0.029983982264719024, 0.015154919138044775, 0.013575158713241556, 0.018632077850519905, 0.030024027903466442, 0.01428355075551068, 0.012438781599078854, 0.021052022525586602, 0.035379515602706454]\n",
      "-------------------------------------\n",
      "Benchmark on dataset: qm7\n",
      "-------------------------------------\n",
      "Loading raw samples now.\n",
      "shard_size: 8192\n",
      "About to start loading CSV from /tmp/qm7.csv\n",
      "Loading shard 1 of size 8192.\n",
      "Featurizing sample 0\n",
      "Featurizing sample 1000\n",
      "Featurizing sample 2000\n",
      "Featurizing sample 3000\n",
      "Featurizing sample 4000\n",
      "Featurizing sample 5000\n",
      "Featurizing sample 6000\n",
      "TIMING: featurizing shard 0 took 146.126 s\n",
      "TIMING: dataset construction took 146.418 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.277 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.076 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.074 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.265 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.036 s\n",
      "Loading dataset from disk.\n",
      "TIMING: dataset construction took 0.035 s\n",
      "Loading dataset from disk.\n",
      "-----------------------------\n",
      "Start fitting: tf_regression\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "1it [12:29, 749.38s/it]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computed_metrics: [108.55753549618473]\n",
      "computed_metrics: [106.97330430701801]\n",
      "computed_metrics: [105.67685971891085]\n"
     ]
    }
   ],
   "source": [
    "dir_path = './runs/model_8_256_train'\n",
    "model_file_list = get_filelist(dir_path)\n",
    "print(model_file_list)\n",
    "cls_benchmark_models(dir_path, [model_file_list[-1]], tasks=['sider', 'tox21'])\n",
    "reg_benchmark_models(dir_path, [model_file_list[-1]], tasks=['qm8', 'qm7'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (deepchem)",
   "language": "python",
   "name": "deepchem"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}