astra_vector_search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "145ea0bd-8e6f-40eb-940c-9c296b0054be",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting ray==2.6.1 (from -r requirements.txt (line 1))\n",
      "  Downloading ray-2.6.1-cp310-cp310-macosx_11_0_arm64.whl (55.9 MB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m55.9/55.9 MB\u001B[0m \u001B[31m1.1 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0mm eta \u001B[36m0:00:01\u001B[0m[36m0:00:02\u001B[0mm\n",
      "\u001B[?25hCollecting virtualenv==20.24.2 (from -r requirements.txt (line 2))\n",
      "  Downloading virtualenv-20.24.2-py3-none-any.whl (3.0 MB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m3.0/3.0 MB\u001B[0m \u001B[31m745.2 kB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0mm eta \u001B[36m0:00:01\u001B[0m[36m0:00:01\u001B[0m\n",
      "\u001B[?25hCollecting langchain==0.0.246 (from -r requirements.txt (line 3))\n",
      "  Downloading langchain-0.0.246-py3-none-any.whl (1.4 MB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m1.4/1.4 MB\u001B[0m \u001B[31m1.2 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0mm eta \u001B[36m0:00:01\u001B[0m[36m0:00:01\u001B[0m\n",
      "\u001B[?25hCollecting sentence-transformers==2.2.2 (from -r requirements.txt (line 4))\n",
      "  Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m86.0/86.0 kB\u001B[0m \u001B[31m1.3 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m MB/s\u001B[0m eta \u001B[36m0:00:01\u001B[0m\n",
      "\u001B[?25h  Preparing metadata (setup.py) ... \u001B[?25ldone\n",
      "\u001B[?25hCollecting pymupdf==1.22.5 (from -r requirements.txt (line 5))\n",
      "  Downloading PyMuPDF-1.22.5-cp310-cp310-macosx_11_0_arm64.whl (12.7 MB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m12.7/12.7 MB\u001B[0m \u001B[31m1.3 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0mm eta \u001B[36m0:00:01\u001B[0m[36m0:00:01\u001B[0m\n",
      "\u001B[?25hCollecting arxiv==1.4.8 (from -r requirements.txt (line 6))\n",
      "  Downloading arxiv-1.4.8-py3-none-any.whl (12 kB)\n",
      "Requirement already satisfied: click>=7.0 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from ray==2.6.1->-r requirements.txt (line 1)) (8.1.3)\n",
      "Requirement already satisfied: filelock in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from ray==2.6.1->-r requirements.txt (line 1)) (3.12.2)\n",
      "Requirement already satisfied: jsonschema in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from ray==2.6.1->-r requirements.txt (line 1)) (4.17.3)\n",
      "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from ray==2.6.1->-r requirements.txt (line 1)) (1.0.5)\n",
      "Requirement already satisfied: packaging in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from ray==2.6.1->-r requirements.txt (line 1)) (23.0)\n",
      "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from ray==2.6.1->-r requirements.txt (line 1)) (4.23.4)\n",
      "Requirement already satisfied: pyyaml in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from ray==2.6.1->-r requirements.txt (line 1)) (6.0)\n",
      "Requirement already satisfied: aiosignal in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from ray==2.6.1->-r requirements.txt (line 1)) (1.3.1)\n",
      "Requirement already satisfied: frozenlist in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from ray==2.6.1->-r requirements.txt (line 1)) (1.3.3)\n",
      "Requirement already satisfied: requests in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from ray==2.6.1->-r requirements.txt (line 1)) (2.29.0)\n",
      "Requirement already satisfied: grpcio>=1.42.0 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from ray==2.6.1->-r requirements.txt (line 1)) (1.48.2)\n",
      "Requirement already satisfied: numpy>=1.19.3 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from ray==2.6.1->-r requirements.txt (line 1)) (1.24.2)\n",
      "Requirement already satisfied: distlib<1,>=0.3.7 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from virtualenv==20.24.2->-r requirements.txt (line 2)) (0.3.7)\n",
      "Requirement already satisfied: platformdirs<4,>=3.9.1 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from virtualenv==20.24.2->-r requirements.txt (line 2)) (3.9.1)\n",
      "Collecting SQLAlchemy<3,>=1.4 (from langchain==0.0.246->-r requirements.txt (line 3))\n",
      "  Downloading SQLAlchemy-2.0.19-cp310-cp310-macosx_11_0_arm64.whl (2.0 MB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m2.0/2.0 MB\u001B[0m \u001B[31m1.5 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m[31m1.5 MB/s\u001B[0m eta \u001B[36m0:00:01\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from langchain==0.0.246->-r requirements.txt (line 3)) (3.8.4)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from langchain==0.0.246->-r requirements.txt (line 3)) (4.0.2)\n",
      "Collecting dataclasses-json<0.6.0,>=0.5.7 (from langchain==0.0.246->-r requirements.txt (line 3))\n",
      "  Downloading dataclasses_json-0.5.13-py3-none-any.whl (26 kB)\n",
      "Collecting langsmith<0.1.0,>=0.0.11 (from langchain==0.0.246->-r requirements.txt (line 3))\n",
      "  Downloading langsmith-0.0.15-py3-none-any.whl (30 kB)\n",
      "Collecting numexpr<3.0.0,>=2.8.4 (from langchain==0.0.246->-r requirements.txt (line 3))\n",
      "  Downloading numexpr-2.8.4-cp310-cp310-macosx_11_0_arm64.whl (89 kB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m89.2/89.2 kB\u001B[0m \u001B[31m618.9 kB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m1m691.4 kB/s\u001B[0m eta \u001B[36m0:00:01\u001B[0m\n",
      "\u001B[?25hCollecting openapi-schema-pydantic<2.0,>=1.2 (from langchain==0.0.246->-r requirements.txt (line 3))\n",
      "  Downloading openapi_schema_pydantic-1.2.4-py3-none-any.whl (90 kB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m90.0/90.0 kB\u001B[0m \u001B[31m1.4 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m MB/s\u001B[0m eta \u001B[36m0:00:01\u001B[0m\n",
      "\u001B[?25hCollecting pydantic<2,>=1 (from langchain==0.0.246->-r requirements.txt (line 3))\n",
      "  Downloading pydantic-1.10.12-cp310-cp310-macosx_11_0_arm64.whl (2.5 MB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m2.5/2.5 MB\u001B[0m \u001B[31m1.7 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0mm eta \u001B[36m0:00:01\u001B[0m0:01\u001B[0m:01\u001B[0m\n",
      "\u001B[?25hCollecting tenacity<9.0.0,>=8.1.0 (from langchain==0.0.246->-r requirements.txt (line 3))\n",
      "  Downloading tenacity-8.2.2-py3-none-any.whl (24 kB)\n",
      "Collecting transformers<5.0.0,>=4.6.0 (from sentence-transformers==2.2.2->-r requirements.txt (line 4))\n",
      "  Downloading transformers-4.31.0-py3-none-any.whl (7.4 MB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m7.4/7.4 MB\u001B[0m \u001B[31m1.1 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0mm eta \u001B[36m0:00:01\u001B[0m[36m0:00:01\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: tqdm in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from sentence-transformers==2.2.2->-r requirements.txt (line 4)) (4.65.0)\n",
      "Collecting torch>=1.6.0 (from sentence-transformers==2.2.2->-r requirements.txt (line 4))\n",
      "  Downloading torch-2.0.1-cp310-none-macosx_11_0_arm64.whl (55.8 MB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m55.8/55.8 MB\u001B[0m \u001B[31m1.4 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0mm eta \u001B[36m0:00:01\u001B[0m[36m0:00:01\u001B[0mm\n",
      "\u001B[?25hCollecting torchvision (from sentence-transformers==2.2.2->-r requirements.txt (line 4))\n",
      "  Downloading torchvision-0.15.2-cp310-cp310-macosx_11_0_arm64.whl (1.4 MB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m1.4/1.4 MB\u001B[0m \u001B[31m1.5 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0mm eta \u001B[36m0:00:01\u001B[0m[36m0:00:01\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: scikit-learn in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from sentence-transformers==2.2.2->-r requirements.txt (line 4)) (1.2.2)\n",
      "Requirement already satisfied: scipy in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from sentence-transformers==2.2.2->-r requirements.txt (line 4)) (1.10.1)\n",
      "Collecting nltk (from sentence-transformers==2.2.2->-r requirements.txt (line 4))\n",
      "  Downloading nltk-3.8.1-py3-none-any.whl (1.5 MB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m1.5/1.5 MB\u001B[0m \u001B[31m1.3 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m[36m0:00:01\u001B[0mm eta \u001B[36m0:00:01\u001B[0m0m\n",
      "\u001B[?25hCollecting sentencepiece (from sentence-transformers==2.2.2->-r requirements.txt (line 4))\n",
      "  Downloading sentencepiece-0.1.99-cp310-cp310-macosx_11_0_arm64.whl (1.2 MB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m1.2/1.2 MB\u001B[0m \u001B[31m1.7 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m[36m0:00:01\u001B[0mm eta \u001B[36m0:00:01\u001B[0m\n",
      "\u001B[?25hCollecting huggingface-hub>=0.4.0 (from sentence-transformers==2.2.2->-r requirements.txt (line 4))\n",
      "  Downloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m268.8/268.8 kB\u001B[0m \u001B[31m1.6 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m MB/s\u001B[0m eta \u001B[36m0:00:01\u001B[0m:01\u001B[0m\n",
      "\u001B[?25hCollecting feedparser (from arxiv==1.4.8->-r requirements.txt (line 6))\n",
      "  Downloading feedparser-6.0.10-py3-none-any.whl (81 kB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m81.1/81.1 kB\u001B[0m \u001B[31m1.3 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m MB/s\u001B[0m eta \u001B[36m0:00:01\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: attrs>=17.3.0 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.246->-r requirements.txt (line 3)) (22.2.0)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.246->-r requirements.txt (line 3)) (2.0.4)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.246->-r requirements.txt (line 3)) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.246->-r requirements.txt (line 3)) (1.9.2)\n",
      "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.246->-r requirements.txt (line 3))\n",
      "  Downloading marshmallow-3.20.1-py3-none-any.whl (49 kB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m49.4/49.4 kB\u001B[0m \u001B[31m1.3 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\n",
      "\u001B[?25hCollecting typing-inspect<1,>=0.4.0 (from dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.246->-r requirements.txt (line 3))\n",
      "  Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
      "Requirement already satisfied: six>=1.5.2 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from grpcio>=1.42.0->ray==2.6.1->-r requirements.txt (line 1)) (1.16.0)\n",
      "Collecting fsspec (from huggingface-hub>=0.4.0->sentence-transformers==2.2.2->-r requirements.txt (line 4))\n",
      "  Downloading fsspec-2023.6.0-py3-none-any.whl (163 kB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m163.8/163.8 kB\u001B[0m \u001B[31m1.5 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m[31m1.7 MB/s\u001B[0m eta \u001B[36m0:00:01\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers==2.2.2->-r requirements.txt (line 4)) (4.6.3)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from requests->ray==2.6.1->-r requirements.txt (line 1)) (3.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from requests->ray==2.6.1->-r requirements.txt (line 1)) (1.26.16)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from requests->ray==2.6.1->-r requirements.txt (line 1)) (2023.5.7)\n",
      "Collecting sympy (from torch>=1.6.0->sentence-transformers==2.2.2->-r requirements.txt (line 4))\n",
      "  Downloading sympy-1.12-py3-none-any.whl (5.7 MB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m5.7/5.7 MB\u001B[0m \u001B[31m1.6 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0mm eta \u001B[36m0:00:01\u001B[0m[36m0:00:01\u001B[0m\n",
      "\u001B[?25hCollecting networkx (from torch>=1.6.0->sentence-transformers==2.2.2->-r requirements.txt (line 4))\n",
      "  Downloading networkx-3.1-py3-none-any.whl (2.1 MB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m2.1/2.1 MB\u001B[0m \u001B[31m1.7 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0mm eta \u001B[36m0:00:01\u001B[0m0:01\u001B[0m:01\u001B[0m\n",
      "\u001B[?25hRequirement already satisfied: jinja2 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from torch>=1.6.0->sentence-transformers==2.2.2->-r requirements.txt (line 4)) (3.1.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers==2.2.2->-r requirements.txt (line 4)) (2023.6.3)\n",
      "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers<5.0.0,>=4.6.0->sentence-transformers==2.2.2->-r requirements.txt (line 4))\n",
      "  Downloading tokenizers-0.13.3-cp310-cp310-macosx_12_0_arm64.whl (3.9 MB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m3.9/3.9 MB\u001B[0m \u001B[31m1.5 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0mm eta \u001B[36m0:00:01\u001B[0m0:01\u001B[0m:01\u001B[0m\n",
      "\u001B[?25hCollecting safetensors>=0.3.1 (from transformers<5.0.0,>=4.6.0->sentence-transformers==2.2.2->-r requirements.txt (line 4))\n",
      "  Downloading safetensors-0.3.1-cp310-cp310-macosx_12_0_arm64.whl (401 kB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m401.4/401.4 kB\u001B[0m \u001B[31m1.4 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0mm eta \u001B[36m0:00:01\u001B[0m[36m0:00:01\u001B[0m\n",
      "\u001B[?25hCollecting sgmllib3k (from feedparser->arxiv==1.4.8->-r requirements.txt (line 6))\n",
      "  Downloading sgmllib3k-1.0.0.tar.gz (5.8 kB)\n",
      "  Preparing metadata (setup.py) ... \u001B[?25ldone\n",
      "\u001B[?25hRequirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from jsonschema->ray==2.6.1->-r requirements.txt (line 1)) (0.19.3)\n",
      "Requirement already satisfied: joblib in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from nltk->sentence-transformers==2.2.2->-r requirements.txt (line 4)) (1.2.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from scikit-learn->sentence-transformers==2.2.2->-r requirements.txt (line 4)) (3.1.0)\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from torchvision->sentence-transformers==2.2.2->-r requirements.txt (line 4)) (9.5.0)\n",
      "Collecting mypy-extensions>=0.3.0 (from typing-inspect<1,>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.246->-r requirements.txt (line 3))\n",
      "  Using cached mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (from jinja2->torch>=1.6.0->sentence-transformers==2.2.2->-r requirements.txt (line 4)) (2.1.3)\n",
      "Collecting mpmath>=0.19 (from sympy->torch>=1.6.0->sentence-transformers==2.2.2->-r requirements.txt (line 4))\n",
      "  Downloading mpmath-1.3.0-py3-none-any.whl (536 kB)\n",
      "\u001B[2K     \u001B[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m536.2/536.2 kB\u001B[0m \u001B[31m1.3 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0mm eta \u001B[36m0:00:01\u001B[0m0:01\u001B[0m:01\u001B[0m\n",
      "\u001B[?25hBuilding wheels for collected packages: sentence-transformers, sgmllib3k\n",
      "  Building wheel for sentence-transformers (setup.py) ... \u001B[?25ldone\n",
      "\u001B[?25h  Created wheel for sentence-transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125925 sha256=d74537421d8519b9a9b373c5b92079cf9f0439cf2b08cf5d790f93da9276f804\n",
      "  Stored in directory: /Users/rihabbenaicha/Library/Caches/pip/wheels/62/f2/10/1e606fd5f02395388f74e7462910fe851042f97238cbbd902f\n",
      "  Building wheel for sgmllib3k (setup.py) ... \u001B[?25ldone\n",
      "\u001B[?25h  Created wheel for sgmllib3k: filename=sgmllib3k-1.0.0-py3-none-any.whl size=6048 sha256=578f1490fa84796b3032087a910f9fc79a646f5fe5cfd89cd4958b87d9fbb1ab\n",
      "  Stored in directory: /Users/rihabbenaicha/Library/Caches/pip/wheels/f0/69/93/a47e9d621be168e9e33c7ce60524393c0b92ae83cf6c6e89c5\n",
      "Successfully built sentence-transformers sgmllib3k\n",
      "Installing collected packages: tokenizers, sgmllib3k, sentencepiece, safetensors, mpmath, virtualenv, tenacity, sympy, SQLAlchemy, pymupdf, pydantic, numexpr, nltk, networkx, mypy-extensions, marshmallow, fsspec, feedparser, typing-inspect, torch, ray, openapi-schema-pydantic, langsmith, huggingface-hub, arxiv, transformers, torchvision, dataclasses-json, sentence-transformers, langchain\n",
      "  Attempting uninstall: virtualenv\n",
      "    Found existing installation: virtualenv 20.24.0\n",
      "    Uninstalling virtualenv-20.24.0:\n",
      "      Successfully uninstalled virtualenv-20.24.0\n",
      "  Attempting uninstall: ray\n",
      "    Found existing installation: ray 2.5.1\n",
      "    Uninstalling ray-2.5.1:\n",
      "      Successfully uninstalled ray-2.5.1\n",
      "Successfully installed SQLAlchemy-2.0.19 arxiv-1.4.8 dataclasses-json-0.5.13 feedparser-6.0.10 fsspec-2023.6.0 huggingface-hub-0.16.4 langchain-0.0.246 langsmith-0.0.15 marshmallow-3.20.1 mpmath-1.3.0 mypy-extensions-1.0.0 networkx-3.1 nltk-3.8.1 numexpr-2.8.4 openapi-schema-pydantic-1.2.4 pydantic-1.10.12 pymupdf-1.22.5 ray-2.6.1 safetensors-0.3.1 sentence-transformers-2.2.2 sentencepiece-0.1.99 sgmllib3k-1.0.0 sympy-1.12 tenacity-8.2.2 tokenizers-0.13.3 torch-2.0.1 torchvision-0.15.2 transformers-4.31.0 typing-inspect-0.9.0 virtualenv-20.24.2\n"
     ]
    }
   ],
   "source": [
    "!pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "22f82576-e5c4-4b70-8fd7-4d4bd47691d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-28 14:01:24,955\tINFO worker.py:1621 -- Started a local Ray instance.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69ad527ae8474975a2b35fd91616882e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<div class=\"lm-Widget p-Widget lm-Panel p-Panel jp-Cell-outputWrapper\">\n",
       "    <div style=\"margin-left: 50px;display: flex;flex-direction: row;align-items: center\">\n",
       "        <div class=\"jp-RenderedHTMLCommon\" style=\"display: flex; flex-direction: row;\">\n",
       "  <svg viewBox=\"0 0 567 224\" fill=\"none\" xmlns=\"http://www.w3.org/2000/svg\" style=\"height: 3em;\">\n",
       "    <g clip-path=\"url(#clip0_4338_178347)\">\n",
       "        <path d=\"M341.29 165.561H355.29L330.13 129.051C345.63 123.991 354.21 112.051 354.21 94.2307C354.21 71.3707 338.72 58.1807 311.88 58.1807H271V165.561H283.27V131.661H311.8C314.25 131.661 316.71 131.501 319.01 131.351L341.25 165.561H341.29ZM283.29 119.851V70.0007H311.82C331.3 70.0007 342.34 78.2907 342.34 94.5507C342.34 111.271 331.34 119.861 311.82 119.861L283.29 119.851ZM451.4 138.411L463.4 165.561H476.74L428.74 58.1807H416L367.83 165.561H380.83L392.83 138.411H451.4ZM446.19 126.601H398L422 72.1407L446.24 126.601H446.19ZM526.11 128.741L566.91 58.1807H554.35L519.99 114.181L485.17 58.1807H472.44L514.01 129.181V165.541H526.13V128.741H526.11Z\" fill=\"var(--jp-ui-font-color0)\"/>\n",
       "        <path d=\"M82.35 104.44C84.0187 97.8827 87.8248 92.0678 93.1671 87.9146C98.5094 83.7614 105.083 81.5067 111.85 81.5067C118.617 81.5067 125.191 83.7614 130.533 87.9146C135.875 92.0678 139.681 97.8827 141.35 104.44H163.75C164.476 101.562 165.622 98.8057 167.15 96.2605L127.45 56.5605C121.071 60.3522 113.526 61.6823 106.235 60.3005C98.9443 58.9187 92.4094 54.9203 87.8602 49.0574C83.3109 43.1946 81.0609 35.8714 81.5332 28.4656C82.0056 21.0599 85.1679 14.0819 90.4252 8.8446C95.6824 3.60726 102.672 0.471508 110.08 0.0272655C117.487 -0.416977 124.802 1.86091 130.647 6.4324C136.493 11.0039 140.467 17.5539 141.821 24.8501C143.175 32.1463 141.816 39.6859 138 46.0505L177.69 85.7505C182.31 82.9877 187.58 81.4995 192.962 81.4375C198.345 81.3755 203.648 82.742 208.33 85.3976C213.012 88.0532 216.907 91.9029 219.616 96.5544C222.326 101.206 223.753 106.492 223.753 111.875C223.753 117.258 222.326 122.545 219.616 127.197C216.907 131.848 213.012 135.698 208.33 138.353C203.648 141.009 198.345 142.375 192.962 142.313C187.58 142.251 182.31 140.763 177.69 138L138 177.7C141.808 184.071 143.155 191.614 141.79 198.91C140.424 206.205 136.44 212.75 130.585 217.313C124.731 221.875 117.412 224.141 110.004 223.683C102.596 223.226 95.6103 220.077 90.3621 214.828C85.1139 209.58 81.9647 202.595 81.5072 195.187C81.0497 187.779 83.3154 180.459 87.878 174.605C92.4405 168.751 98.9853 164.766 106.281 163.401C113.576 162.035 121.119 163.383 127.49 167.19L167.19 127.49C165.664 124.941 164.518 122.182 163.79 119.3H141.39C139.721 125.858 135.915 131.673 130.573 135.826C125.231 139.98 118.657 142.234 111.89 142.234C105.123 142.234 98.5494 139.98 93.2071 135.826C87.8648 131.673 84.0587 125.858 82.39 119.3H60C58.1878 126.495 53.8086 132.78 47.6863 136.971C41.5641 141.163 34.1211 142.972 26.7579 142.059C19.3947 141.146 12.6191 137.574 7.70605 132.014C2.79302 126.454 0.0813599 119.29 0.0813599 111.87C0.0813599 104.451 2.79302 97.2871 7.70605 91.7272C12.6191 86.1673 19.3947 82.5947 26.7579 81.6817C34.1211 80.7686 41.5641 82.5781 47.6863 86.7696C53.8086 90.9611 58.1878 97.2456 60 104.44H82.35ZM100.86 204.32C103.407 206.868 106.759 208.453 110.345 208.806C113.93 209.159 117.527 208.258 120.522 206.256C123.517 204.254 125.725 201.276 126.771 197.828C127.816 194.38 127.633 190.677 126.253 187.349C124.874 184.021 122.383 181.274 119.205 179.577C116.027 177.88 112.359 177.337 108.826 178.042C105.293 178.746 102.113 180.654 99.8291 183.44C97.5451 186.226 96.2979 189.718 96.3 193.32C96.2985 195.364 96.7006 197.388 97.4831 199.275C98.2656 201.163 99.4132 202.877 100.86 204.32ZM204.32 122.88C206.868 120.333 208.453 116.981 208.806 113.396C209.159 109.811 208.258 106.214 206.256 103.219C204.254 100.223 201.275 98.0151 197.827 96.97C194.38 95.9249 190.676 96.1077 187.348 97.4873C184.02 98.8669 181.274 101.358 179.577 104.536C177.879 107.714 177.337 111.382 178.041 114.915C178.746 118.448 180.653 121.627 183.439 123.911C186.226 126.195 189.717 127.443 193.32 127.44C195.364 127.443 197.388 127.042 199.275 126.259C201.163 125.476 202.878 124.328 204.32 122.88ZM122.88 19.4205C120.333 16.8729 116.981 15.2876 113.395 14.9347C109.81 14.5817 106.213 15.483 103.218 17.4849C100.223 19.4868 98.0146 22.4654 96.9696 25.9131C95.9245 29.3608 96.1073 33.0642 97.4869 36.3922C98.8665 39.7202 101.358 42.4668 104.535 44.1639C107.713 45.861 111.381 46.4036 114.914 45.6992C118.447 44.9949 121.627 43.0871 123.911 40.301C126.195 37.515 127.442 34.0231 127.44 30.4205C127.44 28.3772 127.038 26.3539 126.255 24.4664C125.473 22.5788 124.326 20.8642 122.88 19.4205ZM19.42 100.86C16.8725 103.408 15.2872 106.76 14.9342 110.345C14.5813 113.93 15.4826 117.527 17.4844 120.522C19.4863 123.518 22.4649 125.726 25.9127 126.771C29.3604 127.816 33.0638 127.633 36.3918 126.254C39.7198 124.874 42.4664 122.383 44.1635 119.205C45.8606 116.027 46.4032 112.359 45.6988 108.826C44.9944 105.293 43.0866 102.114 40.3006 99.8296C37.5145 97.5455 34.0227 96.2983 30.42 96.3005C26.2938 96.3018 22.337 97.9421 19.42 100.86ZM100.86 100.86C98.3125 103.408 96.7272 106.76 96.3742 110.345C96.0213 113.93 96.9226 117.527 98.9244 120.522C100.926 123.518 103.905 125.726 107.353 126.771C110.8 127.816 114.504 127.633 117.832 126.254C121.16 124.874 123.906 122.383 125.604 119.205C127.301 116.027 127.843 112.359 127.139 108.826C126.434 105.293 124.527 102.114 121.741 99.8296C118.955 97.5455 115.463 96.2983 111.86 96.3005C109.817 96.299 107.793 96.701 105.905 97.4835C104.018 98.2661 102.303 99.4136 100.86 100.86Z\" fill=\"#00AEEF\"/>\n",
       "    </g>\n",
       "    <defs>\n",
       "        <clipPath id=\"clip0_4338_178347\">\n",
       "            <rect width=\"566.93\" height=\"223.75\" fill=\"white\"/>\n",
       "        </clipPath>\n",
       "    </defs>\n",
       "  </svg>\n",
       "</div>\n",
       "\n",
       "        <table class=\"jp-RenderedHTMLCommon\" style=\"border-collapse: collapse;color: var(--jp-ui-font-color1);font-size: var(--jp-ui-font-size1);\">\n",
       "    <tr>\n",
       "        <td style=\"text-align: left\"><b>Python version:</b></td>\n",
       "        <td style=\"text-align: left\"><b>3.10.11</b></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"text-align: left\"><b>Ray version:</b></td>\n",
       "        <td style=\"text-align: left\"><b>2.6.1</b></td>\n",
       "    </tr>\n",
       "    \n",
       "</table>\n",
       "\n",
       "    </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "RayContext(dashboard_url='', python_version='3.10.11', ray_version='2.6.1', ray_commit='d68bf04883af2e430fc3a50fd544bb7c84aff2e9', protocol_version=None)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import ray\n",
    "ray.init(runtime_env={\"pip\": [\"langchain\", \"pypdf\", \"sentence_transformers\", \"transformers\"]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4eeea11e-ab5c-4e4c-91d9-419125a137a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.document_loaders import ArxivLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ecf364f2-5a93-4e30-aef6-5725987c7888",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "        chunk_size=1000, chunk_overlap=100, length_function=len\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "25e81ff3-732d-40bc-9794-8dbbf423676f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: certifi in /opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages (2023.5.7)\n"
     ]
    }
   ],
   "source": [
    "!pip install certifi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c3aa9a7b-0189-4b14-ab86-338aecd8673d",
   "metadata": {},
   "outputs": [
    {
     "ename": "URLError",
     "evalue": "<urlopen error [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1007)>",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mSSLCertVerificationError\u001B[0m                  Traceback (most recent call last)",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/urllib/request.py:1348\u001B[0m, in \u001B[0;36mAbstractHTTPHandler.do_open\u001B[0;34m(self, http_class, req, **http_conn_args)\u001B[0m\n\u001B[1;32m   1347\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m-> 1348\u001B[0m     \u001B[43mh\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrequest\u001B[49m\u001B[43m(\u001B[49m\u001B[43mreq\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mget_method\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mreq\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mselector\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mreq\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1349\u001B[0m \u001B[43m              \u001B[49m\u001B[43mencode_chunked\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mreq\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mhas_header\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mTransfer-encoding\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m)\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1350\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mOSError\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m err: \u001B[38;5;66;03m# timeout error\u001B[39;00m\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/http/client.py:1283\u001B[0m, in \u001B[0;36mHTTPConnection.request\u001B[0;34m(self, method, url, body, headers, encode_chunked)\u001B[0m\n\u001B[1;32m   1282\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"Send a complete request to the server.\"\"\"\u001B[39;00m\n\u001B[0;32m-> 1283\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_send_request\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmethod\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mbody\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mheaders\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mencode_chunked\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/http/client.py:1329\u001B[0m, in \u001B[0;36mHTTPConnection._send_request\u001B[0;34m(self, method, url, body, headers, encode_chunked)\u001B[0m\n\u001B[1;32m   1328\u001B[0m     body \u001B[38;5;241m=\u001B[39m _encode(body, \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mbody\u001B[39m\u001B[38;5;124m'\u001B[39m)\n\u001B[0;32m-> 1329\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mendheaders\u001B[49m\u001B[43m(\u001B[49m\u001B[43mbody\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mencode_chunked\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mencode_chunked\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/http/client.py:1278\u001B[0m, in \u001B[0;36mHTTPConnection.endheaders\u001B[0;34m(self, message_body, encode_chunked)\u001B[0m\n\u001B[1;32m   1277\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m CannotSendHeader()\n\u001B[0;32m-> 1278\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_send_output\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmessage_body\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mencode_chunked\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mencode_chunked\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/http/client.py:1038\u001B[0m, in \u001B[0;36mHTTPConnection._send_output\u001B[0;34m(self, message_body, encode_chunked)\u001B[0m\n\u001B[1;32m   1037\u001B[0m \u001B[38;5;28;01mdel\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_buffer[:]\n\u001B[0;32m-> 1038\u001B[0m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msend\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmsg\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1040\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m message_body \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m   1041\u001B[0m \n\u001B[1;32m   1042\u001B[0m     \u001B[38;5;66;03m# create a consistent interface to message_body\u001B[39;00m\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/http/client.py:976\u001B[0m, in \u001B[0;36mHTTPConnection.send\u001B[0;34m(self, data)\u001B[0m\n\u001B[1;32m    975\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mauto_open:\n\u001B[0;32m--> 976\u001B[0m     \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mconnect\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    977\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/http/client.py:1455\u001B[0m, in \u001B[0;36mHTTPSConnection.connect\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m   1453\u001B[0m     server_hostname \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mhost\n\u001B[0;32m-> 1455\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39msock \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_context\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mwrap_socket\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msock\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1456\u001B[0m \u001B[43m                                      \u001B[49m\u001B[43mserver_hostname\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mserver_hostname\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/ssl.py:513\u001B[0m, in \u001B[0;36mSSLContext.wrap_socket\u001B[0;34m(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)\u001B[0m\n\u001B[1;32m    507\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mwrap_socket\u001B[39m(\u001B[38;5;28mself\u001B[39m, sock, server_side\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m,\n\u001B[1;32m    508\u001B[0m                 do_handshake_on_connect\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m,\n\u001B[1;32m    509\u001B[0m                 suppress_ragged_eofs\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m,\n\u001B[1;32m    510\u001B[0m                 server_hostname\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m, session\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m):\n\u001B[1;32m    511\u001B[0m     \u001B[38;5;66;03m# SSLSocket class handles server_hostname encoding before it calls\u001B[39;00m\n\u001B[1;32m    512\u001B[0m     \u001B[38;5;66;03m# ctx._wrap_socket()\u001B[39;00m\n\u001B[0;32m--> 513\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msslsocket_class\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_create\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m    514\u001B[0m \u001B[43m        \u001B[49m\u001B[43msock\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msock\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    515\u001B[0m \u001B[43m        \u001B[49m\u001B[43mserver_side\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mserver_side\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    516\u001B[0m \u001B[43m        \u001B[49m\u001B[43mdo_handshake_on_connect\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mdo_handshake_on_connect\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    517\u001B[0m \u001B[43m        \u001B[49m\u001B[43msuppress_ragged_eofs\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msuppress_ragged_eofs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    518\u001B[0m \u001B[43m        \u001B[49m\u001B[43mserver_hostname\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mserver_hostname\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    519\u001B[0m \u001B[43m        \u001B[49m\u001B[43mcontext\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[1;32m    520\u001B[0m \u001B[43m        \u001B[49m\u001B[43msession\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msession\u001B[49m\n\u001B[1;32m    521\u001B[0m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/ssl.py:1071\u001B[0m, in \u001B[0;36mSSLSocket._create\u001B[0;34m(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session)\u001B[0m\n\u001B[1;32m   1070\u001B[0m             \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mdo_handshake_on_connect should not be specified for non-blocking sockets\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m-> 1071\u001B[0m         \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdo_handshake\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1072\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (\u001B[38;5;167;01mOSError\u001B[39;00m, \u001B[38;5;167;01mValueError\u001B[39;00m):\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/ssl.py:1342\u001B[0m, in \u001B[0;36mSSLSocket.do_handshake\u001B[0;34m(self, block)\u001B[0m\n\u001B[1;32m   1341\u001B[0m         \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39msettimeout(\u001B[38;5;28;01mNone\u001B[39;00m)\n\u001B[0;32m-> 1342\u001B[0m     \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_sslobj\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdo_handshake\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1343\u001B[0m \u001B[38;5;28;01mfinally\u001B[39;00m:\n",
      "\u001B[0;31mSSLCertVerificationError\u001B[0m: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1007)",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001B[0;31mURLError\u001B[0m                                  Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[12], line 1\u001B[0m\n\u001B[0;32m----> 1\u001B[0m docs_chunks \u001B[38;5;241m=\u001B[39m \u001B[43mArxivLoader\u001B[49m\u001B[43m(\u001B[49m\u001B[43mquery\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43m1605.08386\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mload_max_docs\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;241;43m2\u001B[39;49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mload_and_split\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtext_splitter\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages/langchain/document_loaders/base.py:43\u001B[0m, in \u001B[0;36mBaseLoader.load_and_split\u001B[0;34m(self, text_splitter)\u001B[0m\n\u001B[1;32m     41\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m     42\u001B[0m     _text_splitter \u001B[38;5;241m=\u001B[39m text_splitter\n\u001B[0;32m---> 43\u001B[0m docs \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mload\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     44\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m _text_splitter\u001B[38;5;241m.\u001B[39msplit_documents(docs)\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages/langchain/document_loaders/arxiv.py:33\u001B[0m, in \u001B[0;36mArxivLoader.load\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m     28\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mload\u001B[39m(\u001B[38;5;28mself\u001B[39m) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m List[Document]:\n\u001B[1;32m     29\u001B[0m     arxiv_client \u001B[38;5;241m=\u001B[39m ArxivAPIWrapper(\n\u001B[1;32m     30\u001B[0m         load_max_docs\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mload_max_docs,\n\u001B[1;32m     31\u001B[0m         load_all_available_meta\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mload_all_available_meta,\n\u001B[1;32m     32\u001B[0m     )\n\u001B[0;32m---> 33\u001B[0m     docs \u001B[38;5;241m=\u001B[39m \u001B[43marxiv_client\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mload\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mquery\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     34\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m docs\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages/langchain/utilities/arxiv.py:114\u001B[0m, in \u001B[0;36mArxivAPIWrapper.load\u001B[0;34m(self, query)\u001B[0m\n\u001B[1;32m    112\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m result \u001B[38;5;129;01min\u001B[39;00m results:\n\u001B[1;32m    113\u001B[0m     \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 114\u001B[0m         doc_file_name: \u001B[38;5;28mstr\u001B[39m \u001B[38;5;241m=\u001B[39m \u001B[43mresult\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdownload_pdf\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    115\u001B[0m         \u001B[38;5;28;01mwith\u001B[39;00m fitz\u001B[38;5;241m.\u001B[39mopen(doc_file_name) \u001B[38;5;28;01mas\u001B[39;00m doc_file:\n\u001B[1;32m    116\u001B[0m             text: \u001B[38;5;28mstr\u001B[39m \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;241m.\u001B[39mjoin(page\u001B[38;5;241m.\u001B[39mget_text() \u001B[38;5;28;01mfor\u001B[39;00m page \u001B[38;5;129;01min\u001B[39;00m doc_file)\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages/arxiv/arxiv.py:210\u001B[0m, in \u001B[0;36mResult.download_pdf\u001B[0;34m(self, dirpath, filename)\u001B[0m\n\u001B[1;32m    208\u001B[0m     filename \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_get_default_filename()\n\u001B[1;32m    209\u001B[0m path \u001B[38;5;241m=\u001B[39m os\u001B[38;5;241m.\u001B[39mpath\u001B[38;5;241m.\u001B[39mjoin(dirpath, filename)\n\u001B[0;32m--> 210\u001B[0m written_path, _ \u001B[38;5;241m=\u001B[39m \u001B[43murlretrieve\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpdf_url\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mpath\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    211\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m written_path\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/urllib/request.py:241\u001B[0m, in \u001B[0;36murlretrieve\u001B[0;34m(url, filename, reporthook, data)\u001B[0m\n\u001B[1;32m    224\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m    225\u001B[0m \u001B[38;5;124;03mRetrieve a URL into a temporary location on disk.\u001B[39;00m\n\u001B[1;32m    226\u001B[0m \n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    237\u001B[0m \u001B[38;5;124;03mdata file as well as the resulting HTTPMessage object.\u001B[39;00m\n\u001B[1;32m    238\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m    239\u001B[0m url_type, path \u001B[38;5;241m=\u001B[39m _splittype(url)\n\u001B[0;32m--> 241\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m contextlib\u001B[38;5;241m.\u001B[39mclosing(\u001B[43murlopen\u001B[49m\u001B[43m(\u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdata\u001B[49m\u001B[43m)\u001B[49m) \u001B[38;5;28;01mas\u001B[39;00m fp:\n\u001B[1;32m    242\u001B[0m     headers \u001B[38;5;241m=\u001B[39m fp\u001B[38;5;241m.\u001B[39minfo()\n\u001B[1;32m    244\u001B[0m     \u001B[38;5;66;03m# Just return the local path and the \"headers\" for file://\u001B[39;00m\n\u001B[1;32m    245\u001B[0m     \u001B[38;5;66;03m# URLs. No sense in performing a copy unless requested.\u001B[39;00m\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/urllib/request.py:216\u001B[0m, in \u001B[0;36murlopen\u001B[0;34m(url, data, timeout, cafile, capath, cadefault, context)\u001B[0m\n\u001B[1;32m    214\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m    215\u001B[0m     opener \u001B[38;5;241m=\u001B[39m _opener\n\u001B[0;32m--> 216\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mopener\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mopen\u001B[49m\u001B[43m(\u001B[49m\u001B[43murl\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/urllib/request.py:525\u001B[0m, in \u001B[0;36mOpenerDirector.open\u001B[0;34m(self, fullurl, data, timeout)\u001B[0m\n\u001B[1;32m    523\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m processor \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mprocess_response\u001B[38;5;241m.\u001B[39mget(protocol, []):\n\u001B[1;32m    524\u001B[0m     meth \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mgetattr\u001B[39m(processor, meth_name)\n\u001B[0;32m--> 525\u001B[0m     response \u001B[38;5;241m=\u001B[39m \u001B[43mmeth\u001B[49m\u001B[43m(\u001B[49m\u001B[43mreq\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mresponse\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    527\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m response\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/urllib/request.py:634\u001B[0m, in \u001B[0;36mHTTPErrorProcessor.http_response\u001B[0;34m(self, request, response)\u001B[0m\n\u001B[1;32m    631\u001B[0m \u001B[38;5;66;03m# According to RFC 2616, \"2xx\" code indicates that the client's\u001B[39;00m\n\u001B[1;32m    632\u001B[0m \u001B[38;5;66;03m# request was successfully received, understood, and accepted.\u001B[39;00m\n\u001B[1;32m    633\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m (\u001B[38;5;241m200\u001B[39m \u001B[38;5;241m<\u001B[39m\u001B[38;5;241m=\u001B[39m code \u001B[38;5;241m<\u001B[39m \u001B[38;5;241m300\u001B[39m):\n\u001B[0;32m--> 634\u001B[0m     response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mparent\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43merror\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m    635\u001B[0m \u001B[43m        \u001B[49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mhttp\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrequest\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mresponse\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcode\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmsg\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mhdrs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    637\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m response\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/urllib/request.py:557\u001B[0m, in \u001B[0;36mOpenerDirector.error\u001B[0;34m(self, proto, *args)\u001B[0m\n\u001B[1;32m    555\u001B[0m     http_err \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m0\u001B[39m\n\u001B[1;32m    556\u001B[0m args \u001B[38;5;241m=\u001B[39m (\u001B[38;5;28mdict\u001B[39m, proto, meth_name) \u001B[38;5;241m+\u001B[39m args\n\u001B[0;32m--> 557\u001B[0m result \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_call_chain\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    558\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m result:\n\u001B[1;32m    559\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m result\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/urllib/request.py:496\u001B[0m, in \u001B[0;36mOpenerDirector._call_chain\u001B[0;34m(self, chain, kind, meth_name, *args)\u001B[0m\n\u001B[1;32m    494\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m handler \u001B[38;5;129;01min\u001B[39;00m handlers:\n\u001B[1;32m    495\u001B[0m     func \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mgetattr\u001B[39m(handler, meth_name)\n\u001B[0;32m--> 496\u001B[0m     result \u001B[38;5;241m=\u001B[39m \u001B[43mfunc\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    497\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m result \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m    498\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m result\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/urllib/request.py:749\u001B[0m, in \u001B[0;36mHTTPRedirectHandler.http_error_302\u001B[0;34m(self, req, fp, code, msg, headers)\u001B[0m\n\u001B[1;32m    746\u001B[0m fp\u001B[38;5;241m.\u001B[39mread()\n\u001B[1;32m    747\u001B[0m fp\u001B[38;5;241m.\u001B[39mclose()\n\u001B[0;32m--> 749\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mparent\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mopen\u001B[49m\u001B[43m(\u001B[49m\u001B[43mnew\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtimeout\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mreq\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtimeout\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/urllib/request.py:519\u001B[0m, in \u001B[0;36mOpenerDirector.open\u001B[0;34m(self, fullurl, data, timeout)\u001B[0m\n\u001B[1;32m    516\u001B[0m     req \u001B[38;5;241m=\u001B[39m meth(req)\n\u001B[1;32m    518\u001B[0m sys\u001B[38;5;241m.\u001B[39maudit(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124murllib.Request\u001B[39m\u001B[38;5;124m'\u001B[39m, req\u001B[38;5;241m.\u001B[39mfull_url, req\u001B[38;5;241m.\u001B[39mdata, req\u001B[38;5;241m.\u001B[39mheaders, req\u001B[38;5;241m.\u001B[39mget_method())\n\u001B[0;32m--> 519\u001B[0m response \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_open\u001B[49m\u001B[43m(\u001B[49m\u001B[43mreq\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdata\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    521\u001B[0m \u001B[38;5;66;03m# post-process response\u001B[39;00m\n\u001B[1;32m    522\u001B[0m meth_name \u001B[38;5;241m=\u001B[39m protocol\u001B[38;5;241m+\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m_response\u001B[39m\u001B[38;5;124m\"\u001B[39m\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/urllib/request.py:536\u001B[0m, in \u001B[0;36mOpenerDirector._open\u001B[0;34m(self, req, data)\u001B[0m\n\u001B[1;32m    533\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m result\n\u001B[1;32m    535\u001B[0m protocol \u001B[38;5;241m=\u001B[39m req\u001B[38;5;241m.\u001B[39mtype\n\u001B[0;32m--> 536\u001B[0m result \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_call_chain\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mhandle_open\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mprotocol\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mprotocol\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m+\u001B[39;49m\n\u001B[1;32m    537\u001B[0m \u001B[43m                          \u001B[49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43m_open\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mreq\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    538\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m result:\n\u001B[1;32m    539\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m result\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/urllib/request.py:496\u001B[0m, in \u001B[0;36mOpenerDirector._call_chain\u001B[0;34m(self, chain, kind, meth_name, *args)\u001B[0m\n\u001B[1;32m    494\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m handler \u001B[38;5;129;01min\u001B[39;00m handlers:\n\u001B[1;32m    495\u001B[0m     func \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mgetattr\u001B[39m(handler, meth_name)\n\u001B[0;32m--> 496\u001B[0m     result \u001B[38;5;241m=\u001B[39m \u001B[43mfunc\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    497\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m result \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m    498\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m result\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/urllib/request.py:1391\u001B[0m, in \u001B[0;36mHTTPSHandler.https_open\u001B[0;34m(self, req)\u001B[0m\n\u001B[1;32m   1390\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mhttps_open\u001B[39m(\u001B[38;5;28mself\u001B[39m, req):\n\u001B[0;32m-> 1391\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdo_open\u001B[49m\u001B[43m(\u001B[49m\u001B[43mhttp\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mclient\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mHTTPSConnection\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mreq\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1392\u001B[0m \u001B[43m        \u001B[49m\u001B[43mcontext\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_context\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcheck_hostname\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_check_hostname\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/urllib/request.py:1351\u001B[0m, in \u001B[0;36mAbstractHTTPHandler.do_open\u001B[0;34m(self, http_class, req, **http_conn_args)\u001B[0m\n\u001B[1;32m   1348\u001B[0m         h\u001B[38;5;241m.\u001B[39mrequest(req\u001B[38;5;241m.\u001B[39mget_method(), req\u001B[38;5;241m.\u001B[39mselector, req\u001B[38;5;241m.\u001B[39mdata, headers,\n\u001B[1;32m   1349\u001B[0m                   encode_chunked\u001B[38;5;241m=\u001B[39mreq\u001B[38;5;241m.\u001B[39mhas_header(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mTransfer-encoding\u001B[39m\u001B[38;5;124m'\u001B[39m))\n\u001B[1;32m   1350\u001B[0m     \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mOSError\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m err: \u001B[38;5;66;03m# timeout error\u001B[39;00m\n\u001B[0;32m-> 1351\u001B[0m         \u001B[38;5;28;01mraise\u001B[39;00m URLError(err)\n\u001B[1;32m   1352\u001B[0m     r \u001B[38;5;241m=\u001B[39m h\u001B[38;5;241m.\u001B[39mgetresponse()\n\u001B[1;32m   1353\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m:\n",
      "\u001B[0;31mURLError\u001B[0m: <urlopen error [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain (_ssl.c:1007)>"
     ]
    }
   ],
   "source": [
    "docs_chunks = ArxivLoader(query=\"1605.08386\", load_max_docs=2).load_and_split(text_splitter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8521090-a016-439b-ab63-c2d20eba7e06",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = ray.data.from_items(docs_chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bbd6967-04f0-4a15-bd4d-89013aa5fafd",
   "metadata": {},
   "outputs": [],
   "source": [
    "for d in ds.iter_rows():\n",
    "  print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "199babae-c021-464e-8b65-d3c1abe34943",
   "metadata": {},
   "outputs": [],
   "source": [
    "# preprocess chunks by replacing \\n by empty space\n",
    "ds_batch = ds.flat_map(lambda row: [{'item': row[\"item\"].page_content.replace(\"\\n\", \" \")}])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b8e4c70-5f9b-4727-b735-ceb2fcaab22f",
   "metadata": {},
   "outputs": [],
   "source": [
    "for row in ds_batch.iter_rows():\n",
    "  print(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b50b049c-5362-436e-9aa9-426b0784962a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "from typing import List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "787d4403-cd06-4799-811c-b59282719ec0",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name=\"intfloat/multilingual-e5-small\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9d4212a6-974e-498c-ace4-b17471fb5255",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Embed:\n",
    "  def __init__(self):\n",
    "        self.transformer = SentenceTransformer(model_name)\n",
    "\n",
    "  def __call__(self, text_batch: List[str]):\n",
    "      # We manually encode using sentence_transformer since LangChain\n",
    "      # HuggingfaceEmbeddings does not support specifying a batch size yet.\n",
    "      text = text_batch[\"item\"]\n",
    "      embeddings = self.transformer.encode(\n",
    "          text,\n",
    "          batch_size=100 #,  # Large batch size to maximize GPU/CPU utilization.\n",
    "          #device=\"cuda\",\n",
    "      ).tolist()\n",
    "\n",
    "      #return {'results': list((text, embeddings))}\n",
    "      return {'results':list(zip(text, embeddings))}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9de3e326-1e08-473d-b947-f03bdb997ed4",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds_embed = ds_batch.map_batches(\n",
    "    Embed,\n",
    "    compute=ray.data.ActorPoolStrategy(size=1),\n",
    "    # Large batch size to maximize GPU utilization.\n",
    "    # Too large a batch size may result in GPU running out of memory.\n",
    "    # If the chunk size is increased, then decrease batch size.\n",
    "    # If the chunk size is decreased, then increase batch size.\n",
    "    #batch_size=10,  # Large batch size to maximize GPU utilization.\n",
    "    #compute=ray.data.ActorPoolStrategy(min_size=1, max_size=20),  # I have 20 GPUs in my cluster\n",
    "    #num_gpus=0.1,  # 1 GPU for each actor.\n",
    "    num_cpus=0,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "baf7dde7-a447-490e-abbe-bb924e9a2333",
   "metadata": {},
   "outputs": [],
   "source": [
    "for output in ds_embed.iter_rows():\n",
    "    print(len(output[\"results\"][1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0673eb1d-c454-4f1e-ac2c-2be9abf9425e",
   "metadata": {},
   "source": [
    "## Setup connection with Astra DB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f4ad8f7-f096-47b7-a144-dbd42f459c6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from cassandra.cluster import Cluster\n",
    "from cassandra.auth import PlainTextAuthProvider\n",
    "from local_creds_secrets import *\n",
    "\n",
    "cloud_config= {\n",
    "  'secure_connect_bundle': secure_bundle_path\n",
    "}\n",
    "auth_provider = PlainTextAuthProvider(client_id, client_secret)\n",
    "cluster = Cluster(cloud=cloud_config, auth_provider=auth_provider)\n",
    "session = cluster.connect()\n",
    "\n",
    "row = session.execute(\"select release_version from system.local\").one()\n",
    "if row:\n",
    "  print(row[0])\n",
    "else:\n",
    "  print(\"An error occurred.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bff7db72-749b-4747-8980-8463fd3c9b95",
   "metadata": {},
   "outputs": [],
   "source": [
    "table_name = f\"{db_keyspace}.papers\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b29d4068-d041-45ce-8c86-5bc62fd9222f",
   "metadata": {},
   "outputs": [],
   "source": [
    "table_creation_query = f\"\"\"CREATE TABLE IF NOT EXISTS {table_name} (\n",
    "  id int PRIMARY KEY,\n",
    "  name TEXT,\n",
    "  description TEXT,\n",
    "  item_vector VECTOR<FLOAT, 384> //create a 384-dimensional embedding\n",
    ");\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efc31863-c697-4347-b6d3-dd3c77c54e5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "session.execute(table_creation_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0631fcf2-ef0d-4f67-ac59-3b79e7ce4022",
   "metadata": {},
   "outputs": [],
   "source": [
    "sai_index_query= f\"\"\"CREATE CUSTOM INDEX IF NOT EXISTS ann_index\n",
    "  ON {table_name}(item_vector) USING 'StorageAttachedIndex'; \"\"\"\n",
    "  #WITH OPTIONS = { 'similarity_function': 'DOT_PRODUCT' };\"\"\"\n",
    "session.execute(sai_index_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d97e0ee3-4a08-45fe-ae2c-69ff7b27051d",
   "metadata": {},
   "outputs": [],
   "source": [
    "session.execute(sai_index_query)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa811e95-d6f3-4bfd-9b04-ee8a8963cf92",
   "metadata": {},
   "source": [
    "### Insert vector records into AstraDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2a091ea-8bc1-4f52-98b7-dd81091a119f",
   "metadata": {},
   "outputs": [],
   "source": [
    "id=1\n",
    "for output in ds_embed.iter_rows():\n",
    "  query = f\"\"\"INSERT INTO vector_search.papers (id, name, description, item_vector) \n",
    "  VALUES ({id},'paper_1_sample', '{output[\"results\"][0]}',{output[\"results\"][1]});\"\"\"\n",
    "  id+=1\n",
    "  session.execute(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "660304f1-b6c3-4b0b-ab60-d0857a23a563",
   "metadata": {},
   "outputs": [],
   "source": [
    "rows = session.execute(\"select * from vector_search.papers limit 3;\").all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4b6faa3-0c74-48c8-abb7-570ca55a8eff",
   "metadata": {},
   "outputs": [],
   "source": [
    "for row in rows:\n",
    "  print(row)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5399fdb-5b4b-40b2-8baa-96add0c001e4",
   "metadata": {},
   "source": [
    "## Searching a sentence from the table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a555dfe6-5725-4216-84e7-157c7ba463d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SentenceTransformer('intfloat/multilingual-e5-small')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95b4dff3-2ab7-4b47-b053-3ece178d667d",
   "metadata": {},
   "outputs": [],
   "source": [
    "example_sentence=\"two lattice points are connected by an edge if their diﬀerence lies in a ﬁnite\"\n",
    "embedding = list(model.encode(example_sentence))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3f2022d-f5af-4f4d-b2c6-19869828b5a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "search_query = f\"\"\"SELECT * FROM vector_search.papers\n",
    "ORDER BY item_vector ANN OF {embedding}\n",
    "LIMIT 2;\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81575f5c-f57a-401a-84cf-d9890f256fe3",
   "metadata": {},
   "outputs": [],
   "source": [
    "session.execute(search_query).all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b65700e1-a474-40d0-8330-eebe86eed8b8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8cd9d84-2a9e-4cc4-9339-4afdc955d877",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}