play_with_summarizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "from datetime import datetime\n",
    "import json\n",
    "import os\n",
    "import pickle\n",
    "import random\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "from pytorch_transformers import ConstantLRSchedule, GPT2Config, GPT2LMHeadModel,AdamW, GPT2Tokenizer, WarmupLinearSchedule\n",
    "from tensorboardX import SummaryWriter\n",
    "import torch\n",
    "from torch.nn import CrossEntropyLoss\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
    "from tqdm import tnrange, tqdm_notebook\n",
    "\n",
    "from dataset import GPT21024Dataset \n",
    "from utils import add_special_tokens, beam_search, generate_beam_sample, generate_sample, sample_seq, set_seed, top_k_top_p_filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#please change default arguments if needed\n",
    "\n",
    "parser = argparse.ArgumentParser()\n",
    "parser.add_argument(\"--lr\",default=5e-5, type=float, help=\"learning rate\")\n",
    "parser.add_argument(\"--seed\",default=42, type=int,  help=\"seed to replicate results\")\n",
    "parser.add_argument(\"--n_gpu\",default=1, type=int,  help=\"no of gpu available\")\n",
    "parser.add_argument(\"--gradient_accumulation_steps\",default=32, type=int, help=\"gradient_accumulation_steps\")\n",
    "parser.add_argument(\"--batch_size\",default=1, type=int,  help=\"batch_size\")\n",
    "parser.add_argument(\"--num_workers\",default=4, type=int,  help=\"num of cpus available\")\n",
    "parser.add_argument(\"--device\",default=torch.device('cuda'), help=\"torch.device object\")\n",
    "parser.add_argument(\"--num_train_epochs\",default=5, type=int,  help=\"no of epochs of training\")\n",
    "parser.add_argument(\"--output_dir\",default='./output', type=str,  help=\"path to save evaluation results\")\n",
    "parser.add_argument(\"--model_dir\",default='./weights', type=str,  help=\"path to save trained model\")\n",
    "parser.add_argument(\"--max_grad_norm\",default=1.0, type=float, help=\"max gradient norm.\")\n",
    "parser.add_argument(\"--root_dir\",default='./CNN/gpt2_1024_data', type=str, help=\"location of json dataset.\")\n",
    "parser.add_argument(\"--ids_file\",default='./CNN/ids.json', type=str, help=\"location of train, valid and test file indexes\")\n",
    "args = parser.parse_args([])\n",
    "print(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using the same validation and training data as during training\n",
    "tokenizer = add_special_tokens()\n",
    "train_data = GPT21024Dataset(args.root_dir,args.ids_file,mode='train',length=3000)\n",
    "valid_data = GPT21024Dataset(args.root_dir,args.ids_file,mode='valid',length=500)\n",
    "test_data = GPT21024Dataset(args.root_dir,args.ids_file,mode='test',length=100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model_file and config_file are files used to load finetuned model, change these name as per your file names\n",
    "\n",
    "model_file = os.path.join(args.model_dir, 'model_data{}_trained_after_{}_epochs_only_sum_loss_ignr_pad.bin'.format(len(train_data),args.num_train_epochs))\n",
    "config_file = os.path.join(args.model_dir, 'config_data{}_trained_after_{}_epochs_only_sum_loss_ignr_pad.json'.format(len(train_data),args.num_train_epochs))\n",
    "\n",
    "config = GPT2Config.from_json_file(config_file)\n",
    "model = GPT2LMHeadModel(config)\n",
    "state_dict = torch.load(model_file)\n",
    "model.load_state_dict(state_dict)\n",
    "model.to(args['device'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_sample(test_data, tokenizer, num=4, length=100, temperature=1, top_k=10, top_p=0.5, device=args.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_beam_sample(data, tokenizer, num=4, length=100, beam_size=3, device=args.device)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (fastai2019)",
   "language": "python",
   "name": "fastai2019"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}