summary.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import PyPDF2\n",
    "\n",
    "def extract_text_from_pdf(pdf_path):\n",
    "    text = \"\"\n",
    "    with open(pdf_path, 'rb') as file:\n",
    "        reader = PyPDF2.PdfReader(file)\n",
    "        for page in reader.pages:\n",
    "            page_text = page.extract_text()\n",
    "            if page_text:\n",
    "                text += page_text\n",
    "        return text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import PyPDF2\n",
    "\n",
    "def extract_text_from_pdf(pdf_path):\n",
    "    text = \"\"\n",
    "    with open(pdf_path, 'rb') as file:\n",
    "        reader = PyPDF2.PdfReader(file)\n",
    "        for page in reader.pages:\n",
    "            page_text = page.extract_text()\n",
    "            if page_text:\n",
    "                text += page_text\n",
    "        return text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\aryan\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python38\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n",
    "\n",
    "def summarize_text(text, model_name='t5-small', max_length=150, min_length=40):\n",
    "    # Load the tokenizer and model\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "    model = AutoModelForSeq2SeqLM.from_pretrained(model_name)\n",
    "    \n",
    "    # Prepare the text for summarization\n",
    "    inputs = tokenizer.encode(\"summarize: \" + text, return_tensors='pt', max_length=512, truncation=True)\n",
    "    \n",
    "    # Generate the summary\n",
    "    summary_ids = model.generate(\n",
    "        inputs,\n",
    "        max_length=max_length,\n",
    "        min_length=min_length,\n",
    "        length_penalty=2.0,\n",
    "        num_beams=4,\n",
    "        early_stopping=True,\n",
    "        no_repeat_ngram_size=2\n",
    "    )\n",
    "    \n",
    "    # Decode the summary\n",
    "    summary = tokenizer.decode(summary_ids[0], skip_special_tokens=True)\n",
    "    \n",
    "    return summary\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_text_into_chunks(text, max_chunk_size=500):\n",
    "    sentences = text.split('. ')\n",
    "    current_chunk = ''\n",
    "    chunks = []\n",
    "    for sentence in sentences:\n",
    "        if len(current_chunk) + len(sentence) <= max_chunk_size:\n",
    "            current_chunk += sentence + '. '\n",
    "        else:\n",
    "            chunks.append(current_chunk)\n",
    "            current_chunk = sentence + '. '\n",
    "    if current_chunk:\n",
    "        chunks.append(current_chunk)\n",
    "    return chunks\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def summarize_large_text(text, model_name='t5-small'):\n",
    "    chunks = split_text_into_chunks(text)\n",
    "    summaries = []\n",
    "    for chunk in chunks:\n",
    "        summary = summarize_text(chunk, model_name=model_name)\n",
    "        summaries.append(summary)\n",
    "    final_summary = ' '.join(summaries)\n",
    "    return final_summary\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing file: E:\\OSPO\\bioarxiv\\2024.03.01.582919v1.full.pdf\n",
      "Summarizing chunk 1/112...\n",
      "Summarizing chunk 2/112...\n",
      "Summarizing chunk 3/112...\n",
      "Summarizing chunk 4/112...\n",
      "Summarizing chunk 5/112...\n",
      "Summarizing chunk 6/112...\n",
      "Summarizing chunk 7/112...\n",
      "Summarizing chunk 8/112...\n",
      "Summarizing chunk 9/112...\n",
      "Summarizing chunk 10/112...\n",
      "Summarizing chunk 11/112...\n",
      "Summarizing chunk 12/112...\n",
      "Summarizing chunk 13/112...\n",
      "Summarizing chunk 14/112...\n",
      "Summarizing chunk 15/112...\n",
      "Summarizing chunk 16/112...\n",
      "Summarizing chunk 17/112...\n",
      "Summarizing chunk 18/112...\n",
      "Summarizing chunk 19/112...\n",
      "Summarizing chunk 20/112...\n",
      "Summarizing chunk 21/112...\n",
      "Summarizing chunk 22/112...\n",
      "Summarizing chunk 23/112...\n",
      "Summarizing chunk 24/112...\n",
      "Summarizing chunk 25/112...\n",
      "Summarizing chunk 26/112...\n",
      "Summarizing chunk 27/112...\n",
      "Summarizing chunk 28/112...\n",
      "Summarizing chunk 29/112...\n",
      "Summarizing chunk 30/112...\n",
      "Summarizing chunk 31/112...\n",
      "Summarizing chunk 32/112...\n",
      "Summarizing chunk 33/112...\n",
      "Summarizing chunk 34/112...\n",
      "Summarizing chunk 35/112...\n",
      "Summarizing chunk 36/112...\n",
      "Summarizing chunk 37/112...\n",
      "Summarizing chunk 38/112...\n",
      "Summarizing chunk 39/112...\n",
      "Summarizing chunk 40/112...\n",
      "Summarizing chunk 41/112...\n",
      "Summarizing chunk 42/112...\n",
      "Summarizing chunk 43/112...\n",
      "Summarizing chunk 44/112...\n",
      "Summarizing chunk 45/112...\n",
      "Summarizing chunk 46/112...\n",
      "Summarizing chunk 47/112...\n",
      "Summarizing chunk 48/112...\n",
      "Summarizing chunk 49/112...\n",
      "Summarizing chunk 50/112...\n",
      "Summarizing chunk 51/112...\n",
      "Summarizing chunk 52/112...\n",
      "Summarizing chunk 53/112...\n",
      "Summarizing chunk 54/112...\n",
      "Summarizing chunk 55/112...\n",
      "Summarizing chunk 56/112...\n",
      "Summarizing chunk 57/112...\n",
      "Summarizing chunk 58/112...\n",
      "Summarizing chunk 59/112...\n",
      "Summarizing chunk 60/112...\n",
      "Summarizing chunk 61/112...\n",
      "Summarizing chunk 62/112...\n",
      "Summarizing chunk 63/112...\n",
      "Summarizing chunk 64/112...\n",
      "Summarizing chunk 65/112...\n",
      "Summarizing chunk 66/112...\n",
      "Summarizing chunk 67/112...\n",
      "Summarizing chunk 68/112...\n",
      "Summarizing chunk 69/112...\n",
      "Summarizing chunk 70/112...\n",
      "Summarizing chunk 71/112...\n",
      "Summarizing chunk 72/112...\n",
      "Summarizing chunk 73/112...\n",
      "Summarizing chunk 74/112...\n",
      "Summarizing chunk 75/112...\n",
      "Summarizing chunk 76/112...\n",
      "Summarizing chunk 77/112...\n",
      "Summarizing chunk 78/112...\n",
      "Summarizing chunk 79/112...\n",
      "Summarizing chunk 80/112...\n",
      "Summarizing chunk 81/112...\n",
      "Summarizing chunk 82/112...\n",
      "Summarizing chunk 83/112...\n",
      "Summarizing chunk 84/112...\n",
      "Summarizing chunk 85/112...\n",
      "Summarizing chunk 86/112...\n",
      "Summarizing chunk 87/112...\n",
      "Summarizing chunk 88/112...\n",
      "Summarizing chunk 89/112...\n",
      "Summarizing chunk 90/112...\n",
      "Summarizing chunk 91/112...\n",
      "Summarizing chunk 92/112...\n",
      "Summarizing chunk 93/112...\n",
      "Summarizing chunk 94/112...\n",
      "Summarizing chunk 95/112...\n",
      "Summarizing chunk 96/112...\n",
      "Summarizing chunk 97/112...\n",
      "Summarizing chunk 98/112...\n",
      "Summarizing chunk 99/112...\n",
      "Summarizing chunk 100/112...\n",
      "Summarizing chunk 101/112...\n",
      "Summarizing chunk 102/112...\n",
      "Summarizing chunk 103/112...\n",
      "Summarizing chunk 104/112...\n",
      "Summarizing chunk 105/112...\n",
      "Summarizing chunk 106/112...\n",
      "Summarizing chunk 107/112...\n",
      "Summarizing chunk 108/112...\n",
      "Summarizing chunk 109/112...\n",
      "Summarizing chunk 110/112...\n",
      "Summarizing chunk 111/112...\n",
      "Summarizing chunk 112/112...\n",
      "\n",
      "Summary:\n",
      "the order Schizomida 36 (Arachnida) is heavily understudied and the phylogeny of the group is poorly understood. most of them 41 documenting setation patterns on pedipalps, flagellum and CC-BY 4.0 international license made available under a is the author/funder. bioRxiv preprint 346 fossil, down to genus level easily. currently, there are 380 described species live in or on leaf litter, in cavities under\n",
      "\n",
      "Processing file: E:\\OSPO\\eartharxiv\\jones_et_al_2024.pdf\n",
      "Summarizing chunk 1/52...\n",
      "Summarizing chunk 2/52...\n",
      "Summarizing chunk 3/52...\n",
      "Summarizing chunk 4/52...\n",
      "Summarizing chunk 5/52...\n",
      "Summarizing chunk 6/52...\n",
      "Summarizing chunk 7/52...\n",
      "Summarizing chunk 8/52...\n",
      "Summarizing chunk 9/52...\n",
      "Summarizing chunk 10/52...\n",
      "Summarizing chunk 11/52...\n",
      "Summarizing chunk 12/52...\n",
      "Summarizing chunk 13/52...\n",
      "Summarizing chunk 14/52...\n",
      "Summarizing chunk 15/52...\n",
      "Summarizing chunk 16/52...\n",
      "Summarizing chunk 17/52...\n",
      "Summarizing chunk 18/52...\n",
      "Summarizing chunk 19/52...\n",
      "Summarizing chunk 20/52...\n",
      "Summarizing chunk 21/52...\n",
      "Summarizing chunk 22/52...\n",
      "Summarizing chunk 23/52...\n",
      "Summarizing chunk 24/52...\n",
      "Summarizing chunk 25/52...\n",
      "Summarizing chunk 26/52...\n",
      "Summarizing chunk 27/52...\n",
      "Summarizing chunk 28/52...\n",
      "Summarizing chunk 29/52...\n",
      "Summarizing chunk 30/52...\n",
      "Summarizing chunk 31/52...\n",
      "Summarizing chunk 32/52...\n",
      "Summarizing chunk 33/52...\n",
      "Summarizing chunk 34/52...\n",
      "Summarizing chunk 35/52...\n",
      "Summarizing chunk 36/52...\n",
      "Summarizing chunk 37/52...\n",
      "Summarizing chunk 38/52...\n",
      "Summarizing chunk 39/52...\n",
      "Summarizing chunk 40/52...\n",
      "Summarizing chunk 41/52...\n",
      "Summarizing chunk 42/52...\n",
      "Summarizing chunk 43/52...\n",
      "Summarizing chunk 44/52...\n",
      "Summarizing chunk 45/52...\n",
      "Summarizing chunk 46/52...\n",
      "Summarizing chunk 47/52...\n",
      "Summarizing chunk 48/52...\n",
      "Summarizing chunk 49/52...\n",
      "Summarizing chunk 50/52...\n",
      "Summarizing chunk 51/52...\n",
      "Summarizing chunk 52/52...\n",
      "\n",
      "Summary:\n",
      "rmacrostrat: an R package for accessing and retrieving 1 data from the Macrostrat geological database 2 Lewis A. Jones1, Christopher D. Dean2, William Gearty3, and Bethany J the final version of this manuscript will be available via the 15 ‘Peer-reviewed publication DOI’ link on this page. it can be used to visualize regional stratigraphic columns, produce regional geologic 33 outcrop maps, and estimate the proportion 39 Historically, 40 researchers were required to synth\n",
      "\n",
      "Processing file: E:\\OSPO\\eartharxiv\\jones_et_al_2022.pdf\n",
      "Summarizing chunk 1/50...\n",
      "Summarizing chunk 2/50...\n",
      "Summarizing chunk 3/50...\n",
      "Summarizing chunk 4/50...\n",
      "Summarizing chunk 5/50...\n",
      "Summarizing chunk 6/50...\n",
      "Summarizing chunk 7/50...\n",
      "Summarizing chunk 8/50...\n",
      "Summarizing chunk 9/50...\n",
      "Summarizing chunk 10/50...\n",
      "Summarizing chunk 11/50...\n",
      "Summarizing chunk 12/50...\n",
      "Summarizing chunk 13/50...\n",
      "Summarizing chunk 14/50...\n",
      "Summarizing chunk 15/50...\n",
      "Summarizing chunk 16/50...\n",
      "Summarizing chunk 17/50...\n",
      "Summarizing chunk 18/50...\n",
      "Summarizing chunk 19/50...\n",
      "Summarizing chunk 20/50...\n",
      "Summarizing chunk 21/50...\n",
      "Summarizing chunk 22/50...\n",
      "Summarizing chunk 23/50...\n",
      "Summarizing chunk 24/50...\n",
      "Summarizing chunk 25/50...\n",
      "Summarizing chunk 26/50...\n",
      "Summarizing chunk 27/50...\n",
      "Summarizing chunk 28/50...\n",
      "Summarizing chunk 29/50...\n",
      "Summarizing chunk 30/50...\n",
      "Summarizing chunk 31/50...\n",
      "Summarizing chunk 32/50...\n",
      "Summarizing chunk 33/50...\n",
      "Summarizing chunk 34/50...\n",
      "Summarizing chunk 35/50...\n",
      "Summarizing chunk 36/50...\n",
      "Summarizing chunk 37/50...\n",
      "Summarizing chunk 38/50...\n",
      "Summarizing chunk 39/50...\n",
      "Summarizing chunk 40/50...\n",
      "Summarizing chunk 41/50...\n",
      "Summarizing chunk 42/50...\n",
      "Summarizing chunk 43/50...\n",
      "Summarizing chunk 44/50...\n",
      "Summarizing chunk 45/50...\n",
      "Summarizing chunk 46/50...\n",
      "Summarizing chunk 47/50...\n",
      "Summarizing chunk 48/50...\n",
      "Summarizing chunk 49/50...\n",
      "Summarizing chunk 50/50...\n",
      "\n",
      "Summary:\n",
      "palaeoverse 49 is the first community-driven R package in the community. it provides a user-friendly platform for preparing data for analysis 54 with well-documented open-source code to enhance transparency. the package contains functions that align with current researcher needs to cleanse, prepare, 99 and explore occurrence datasets for further analysis. this needs were established via survey was 105 distributed via social media (Twitter and email)\n",
      "\n",
      "Processing file: E:\\OSPO\\eartharxiv\\deeptime-preprint.pdf\n",
      "Summarizing chunk 1/52...\n",
      "Summarizing chunk 2/52...\n",
      "Summarizing chunk 3/52...\n",
      "Summarizing chunk 4/52...\n",
      "Summarizing chunk 5/52...\n",
      "Summarizing chunk 6/52...\n",
      "Summarizing chunk 7/52...\n",
      "Summarizing chunk 8/52...\n",
      "Summarizing chunk 9/52...\n",
      "Summarizing chunk 10/52...\n",
      "Summarizing chunk 11/52...\n",
      "Summarizing chunk 12/52...\n",
      "Summarizing chunk 13/52...\n",
      "Summarizing chunk 14/52...\n",
      "Summarizing chunk 15/52...\n",
      "Summarizing chunk 16/52...\n",
      "Summarizing chunk 17/52...\n",
      "Summarizing chunk 18/52...\n",
      "Summarizing chunk 19/52...\n",
      "Summarizing chunk 20/52...\n",
      "Summarizing chunk 21/52...\n",
      "Summarizing chunk 22/52...\n",
      "Summarizing chunk 23/52...\n",
      "Summarizing chunk 24/52...\n",
      "Summarizing chunk 25/52...\n",
      "Summarizing chunk 26/52...\n",
      "Summarizing chunk 27/52...\n",
      "Summarizing chunk 28/52...\n",
      "Summarizing chunk 29/52...\n",
      "Summarizing chunk 30/52...\n",
      "Summarizing chunk 31/52...\n",
      "Summarizing chunk 32/52...\n",
      "Summarizing chunk 33/52...\n",
      "Summarizing chunk 34/52...\n",
      "Summarizing chunk 35/52...\n",
      "Summarizing chunk 36/52...\n",
      "Summarizing chunk 37/52...\n",
      "Summarizing chunk 38/52...\n",
      "Summarizing chunk 39/52...\n",
      "Summarizing chunk 40/52...\n",
      "Summarizing chunk 41/52...\n",
      "Summarizing chunk 42/52...\n",
      "Summarizing chunk 43/52...\n",
      "Summarizing chunk 44/52...\n",
      "Summarizing chunk 45/52...\n",
      "Summarizing chunk 46/52...\n",
      "Summarizing chunk 47/52...\n",
      "Summarizing chunk 48/52...\n",
      "Summarizing chunk 49/52...\n",
      "Summarizing chunk 50/52...\n",
      "Summarizing chunk 51/52...\n",
      "Summarizing chunk 52/52...\n",
      "\n",
      "Summary:\n",
      "deeptime provides easy-to-use functions to facilitate visualizations of geological data over long time intervals (104 – 107 years) the 24 open -source and constantly evolving package is accompanied by exhaustive documentation 25 2 about the myriad options available to users. the oldest preserved geologic map, the Turin Papyrus, dat ing 36 back to 1150 BC (Harrell and Brown, 1992) software 49 packages often have graphical user interfaces and dedicated, paid support staff, but their use also 50 incur\n",
      "\n",
      "Processing file: E:\\OSPO\\evoarxiv\\copy-of-pcm-sensitivity-manuscript.pdf\n",
      "Summarizing chunk 1/103...\n",
      "Summarizing chunk 2/103...\n",
      "Summarizing chunk 3/103...\n",
      "Summarizing chunk 4/103...\n",
      "Summarizing chunk 5/103...\n",
      "Summarizing chunk 6/103...\n",
      "Summarizing chunk 7/103...\n",
      "Summarizing chunk 8/103...\n",
      "Summarizing chunk 9/103...\n",
      "Summarizing chunk 10/103...\n",
      "Summarizing chunk 11/103...\n",
      "Summarizing chunk 12/103...\n",
      "Summarizing chunk 13/103...\n",
      "Summarizing chunk 14/103...\n",
      "Summarizing chunk 15/103...\n",
      "Summarizing chunk 16/103...\n",
      "Summarizing chunk 17/103...\n",
      "Summarizing chunk 18/103...\n",
      "Summarizing chunk 19/103...\n",
      "Summarizing chunk 20/103...\n",
      "Summarizing chunk 21/103...\n",
      "Summarizing chunk 22/103...\n",
      "Summarizing chunk 23/103...\n",
      "Summarizing chunk 24/103...\n",
      "Summarizing chunk 25/103...\n",
      "Summarizing chunk 26/103...\n",
      "Summarizing chunk 27/103...\n",
      "Summarizing chunk 28/103...\n",
      "Summarizing chunk 29/103...\n",
      "Summarizing chunk 30/103...\n",
      "Summarizing chunk 31/103...\n",
      "Summarizing chunk 32/103...\n",
      "Summarizing chunk 33/103...\n",
      "Summarizing chunk 34/103...\n",
      "Summarizing chunk 35/103...\n",
      "Summarizing chunk 36/103...\n",
      "Summarizing chunk 37/103...\n",
      "Summarizing chunk 38/103...\n",
      "Summarizing chunk 39/103...\n",
      "Summarizing chunk 40/103...\n",
      "Summarizing chunk 41/103...\n",
      "Summarizing chunk 42/103...\n",
      "Summarizing chunk 43/103...\n",
      "Summarizing chunk 44/103...\n",
      "Summarizing chunk 45/103...\n",
      "Summarizing chunk 46/103...\n",
      "Summarizing chunk 47/103...\n",
      "Summarizing chunk 48/103...\n",
      "Summarizing chunk 49/103...\n",
      "Summarizing chunk 50/103...\n",
      "Summarizing chunk 51/103...\n",
      "Summarizing chunk 52/103...\n",
      "Summarizing chunk 53/103...\n",
      "Summarizing chunk 54/103...\n",
      "Summarizing chunk 55/103...\n",
      "Summarizing chunk 56/103...\n",
      "Summarizing chunk 57/103...\n",
      "Summarizing chunk 58/103...\n",
      "Summarizing chunk 59/103...\n",
      "Summarizing chunk 60/103...\n",
      "Summarizing chunk 61/103...\n",
      "Summarizing chunk 62/103...\n",
      "Summarizing chunk 63/103...\n",
      "Summarizing chunk 64/103...\n",
      "Summarizing chunk 65/103...\n",
      "Summarizing chunk 66/103...\n",
      "Summarizing chunk 67/103...\n",
      "Summarizing chunk 68/103...\n",
      "Summarizing chunk 69/103...\n",
      "Summarizing chunk 70/103...\n",
      "Summarizing chunk 71/103...\n",
      "Summarizing chunk 72/103...\n",
      "Summarizing chunk 73/103...\n",
      "Summarizing chunk 74/103...\n",
      "Summarizing chunk 75/103...\n",
      "Summarizing chunk 76/103...\n",
      "Summarizing chunk 77/103...\n",
      "Summarizing chunk 78/103...\n",
      "Summarizing chunk 79/103...\n",
      "Summarizing chunk 80/103...\n",
      "Summarizing chunk 81/103...\n",
      "Summarizing chunk 82/103...\n",
      "Summarizing chunk 83/103...\n",
      "Summarizing chunk 84/103...\n",
      "Summarizing chunk 85/103...\n",
      "Summarizing chunk 86/103...\n",
      "Summarizing chunk 87/103...\n",
      "Summarizing chunk 88/103...\n",
      "Summarizing chunk 89/103...\n",
      "Summarizing chunk 90/103...\n",
      "Summarizing chunk 91/103...\n",
      "Summarizing chunk 92/103...\n",
      "Summarizing chunk 93/103...\n",
      "Summarizing chunk 94/103...\n",
      "Summarizing chunk 95/103...\n",
      "Summarizing chunk 96/103...\n",
      "Summarizing chunk 97/103...\n",
      "Summarizing chunk 98/103...\n",
      "Summarizing chunk 99/103...\n",
      "Summarizing chunk 100/103...\n",
      "Summarizing chunk 101/103...\n",
      "Summarizing chunk 102/103...\n",
      "Summarizing chunk 103/103...\n",
      "\n",
      "Summary:\n",
      "we used multiple diversification rates to simulate the evolution of a single trait across each phylogeny using multiple continuous trait evolution models. we then compared the fit of the correct and incorrect models to the simulated traits; increasing the proportion of fossils is far more beneficial, and perhaps more time- and resource-efficient, than increasing number of extant taxa in the dataset.\n"
     ]
    }
   ],
   "source": [
    "import PyPDF2\n",
    "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n",
    "\n",
    "def extract_text_from_pdf(pdf_path):\n",
    "    \"\"\"Extracts text from a PDF file.\"\"\"\n",
    "    text = \"\"\n",
    "    with open(pdf_path, 'rb') as file:\n",
    "        reader = PyPDF2.PdfReader(file)\n",
    "        for page in reader.pages:\n",
    "            page_text = page.extract_text()\n",
    "            if page_text:\n",
    "                text += page_text\n",
    "    return text\n",
    "\n",
    "def summarize_text(text, model, tokenizer, max_length=130, min_length=65):\n",
    "    \"\"\"Summarizes a given text using a pre-loaded model and tokenizer.\"\"\"\n",
    "    # Prepare the text for summarization\n",
    "    inputs = tokenizer.encode(\"summarize: \" + text, return_tensors='pt', max_length=512, truncation=True)\n",
    "    \n",
    "    # Generate the summary\n",
    "    summary_ids = model.generate(\n",
    "        inputs,\n",
    "        max_length=max_length,\n",
    "        min_length=min_length,\n",
    "        length_penalty=2.0,\n",
    "        num_beams=4,\n",
    "        early_stopping=True,\n",
    "        no_repeat_ngram_size=2\n",
    "    )\n",
    "    \n",
    "    # Decode the summary\n",
    "    summary = tokenizer.decode(summary_ids[0], skip_special_tokens=True)\n",
    "    \n",
    "    return summary\n",
    "\n",
    "def split_text_into_chunks(text, max_chunk_size=1000):\n",
    "    \"\"\"Splits text into smaller chunks to fit model input size.\"\"\"\n",
    "    sentences = text.replace('\\n', ' ').split('. ')\n",
    "    chunks = []\n",
    "    current_chunk = ''\n",
    "    for sentence in sentences:\n",
    "        if len(current_chunk) + len(sentence) <= max_chunk_size:\n",
    "            current_chunk += sentence + '. '\n",
    "        else:\n",
    "            chunks.append(current_chunk)\n",
    "            current_chunk = sentence + '. '\n",
    "    if current_chunk:\n",
    "        chunks.append(current_chunk)\n",
    "    return chunks\n",
    "\n",
    "def summarize_large_text(text, model, tokenizer):\n",
    "    \"\"\"Summarizes large text by splitting it into chunks and summarizing the combined summaries.\"\"\"\n",
    "    chunks = split_text_into_chunks(text)\n",
    "    summaries = []\n",
    "    for i, chunk in enumerate(chunks):\n",
    "        print(f\"Summarizing chunk {i+1}/{len(chunks)}...\")\n",
    "        # Summarize each chunk into very short summaries\n",
    "        summary = summarize_text(chunk, model, tokenizer, max_length=50, min_length=30)\n",
    "        summaries.append(summary)\n",
    "    combined_summary = ' '.join(summaries)\n",
    "    # Summarize the combined summaries to get the final summary\n",
    "    final_summary = summarize_text(combined_summary, model, tokenizer, max_length=130, min_length=65)\n",
    "    return final_summary\n",
    "\n",
    "# Load the tokenizer and model once to avoid reloading for each summary\n",
    "model_name = 't5-small'\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(model_name)\n",
    "\n",
    "# List of PDF file paths\n",
    "pdf_paths = [\n",
    "    r\"E:\\OSPO\\bioarxiv\\2024.03.01.582919v1.full.pdf\",\n",
    "    r\"E:\\OSPO\\eartharxiv\\jones_et_al_2024.pdf\",\n",
    "    r\"E:\\OSPO\\eartharxiv\\jones_et_al_2022.pdf\",\n",
    "    r\"E:\\OSPO\\eartharxiv\\deeptime-preprint.pdf\",\n",
    "    r\"E:\\OSPO\\evoarxiv\\copy-of-pcm-sensitivity-manuscript.pdf\"\n",
    "]\n",
    "\n",
    "# Process each PDF file\n",
    "for pdf_path in pdf_paths:\n",
    "    print(f\"\\nProcessing file: {pdf_path}\")\n",
    "    text = extract_text_from_pdf(pdf_path)\n",
    "    if text:\n",
    "        summary = summarize_large_text(text, model, tokenizer)\n",
    "        print(\"\\nSummary:\")\n",
    "        print(summary)\n",
    "    else:\n",
    "        print(\"No text found in the PDF.\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}