{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/envs/rag_llm/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import hopsworks\n",
    "from sentence_transformers import SentenceTransformer\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from langchain_docling import DoclingLoader\n",
    "from langchain_docling.loader import ExportType\n",
    "from docling.chunking import HybridChunker\n",
    "\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PDF_PATH = \"content/Building+Machine+Learning+Systems+with+a+Feature+Store.pdf\"\n",
    "EMBED_MODEL_ID = \"sentence-transformers/all-MiniLM-L6-v2\"\n",
    "EXPORT_TYPE = ExportType.DOC_CHUNKS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-12-02 19:43:33,611 INFO: detected formats: [<InputFormat.PDF: 'pdf'>]\n",
      "2025-12-02 19:43:33,861 INFO: Going to convert document batch...\n",
      "2025-12-02 19:43:33,863 INFO: Initializing pipeline for StandardPdfPipeline with options hash e15bc6f248154cc62f8db15ef18a8ab7\n",
      "2025-12-02 19:43:33,913 WARNING: The plugin langchain_docling will not be loaded because Docling is being executed with allow_external_plugins=false.\n",
      "2025-12-02 19:43:33,914 INFO: Loading plugin 'docling_defaults'\n",
      "2025-12-02 19:43:33,926 INFO: Registered picture descriptions: ['vlm', 'api']\n",
      "2025-12-02 19:43:33,981 WARNING: The plugin langchain_docling will not be loaded because Docling is being executed with allow_external_plugins=false.\n",
      "2025-12-02 19:43:33,982 INFO: Loading plugin 'docling_defaults'\n",
      "2025-12-02 19:43:34,010 INFO: Registered ocr engines: ['auto', 'easyocr', 'ocrmac', 'rapidocr', 'tesserocr', 'tesseract']\n",
      "2025-12-02 19:43:42,281 INFO: Auto OCR model selected ocrmac.\n",
      "2025-12-02 19:43:42,299 WARNING: The plugin langchain_docling will not be loaded because Docling is being executed with allow_external_plugins=false.\n",
      "2025-12-02 19:43:42,299 INFO: Loading plugin 'docling_defaults'\n",
      "2025-12-02 19:43:42,323 INFO: Registered layout engines: ['docling_layout_default', 'docling_experimental_table_crops_layout']\n",
      "2025-12-02 19:43:42,347 INFO: Accelerator device: 'mps'\n",
      "2025-12-02 19:43:57,889 WARNING: The plugin langchain_docling will not be loaded because Docling is being executed with allow_external_plugins=false.\n",
      "2025-12-02 19:43:57,907 INFO: Loading plugin 'docling_defaults'\n",
      "2025-12-02 19:43:57,919 INFO: Registered table structure engines: ['docling_tableformer']\n",
      "2025-12-02 19:44:40,325 INFO: Accelerator device: 'mps'\n",
      "2025-12-02 19:44:41,261 INFO: Processing document Building+Machine+Learning+Systems+with+a+Feature+Store.pdf\n",
      "2025-12-02 19:51:45,276 INFO: Finished converting document Building+Machine+Learning+Systems+with+a+Feature+Store.pdf in 491.52 sec.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (1143 > 512). Running this sequence through the model will result in indexing errors\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 1333 document chunks\n"
     ]
    }
   ],
   "source": [
    "loader = DoclingLoader(\n",
    "    file_path=PDF_PATH,\n",
    "    export_type=EXPORT_TYPE,\n",
    "    chunker=HybridChunker(tokenizer=EMBED_MODEL_ID),\n",
    ")\n",
    "\n",
    "docs = loader.load()\n",
    "print(f\"Loaded {len(docs)} document chunks\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "page_content='Praise for Building Machine Learning Systems with a Feature Store\n",
      "It' s easy to be lost in quality metrics land and forget about the crucial systems aspect to ML. Jim does a great job explaining those aspects and gives a lot of practical tips on how to survive a long deployment.\n",
      "-Hannes Mühleisen, cocreator of DuckDB\n",
      "Building machine learning systems in production has historically involved a lot of black magic and undocumented learnings. Jim Dowling is doing a great service to ML practitioners by sharing the best practices and putting together clear step-by-step guide.' metadata={'source': 'content/Building+Machine+Learning+Systems+with+a+Feature+Store.pdf', 'dl_meta': {'schema_name': 'docling_core.transforms.chunker.DocMeta', 'version': '1.0.0', 'doc_items': [{'self_ref': '#/texts/7', 'parent': {'$ref': '#/body'}, 'children': [], 'content_layer': 'body', 'label': 'text', 'prov': [{'page_no': 1, 'bbox': {'l': 97.75, 't': 162.01999999999998, 'r': 432.0, 'b': 126.02999999999997, 'coord_origin': 'BOTTOMLEFT'}, 'charspan': [0, 213]}]}, {'self_ref': '#/texts/8', 'parent': {'$ref': '#/body'}, 'children': [], 'content_layer': 'body', 'label': 'text', 'prov': [{'page_no': 1, 'bbox': {'l': 264.75, 't': 122.13, 'r': 432.0, 'b': 110.03200000000004, 'coord_origin': 'BOTTOMLEFT'}, 'charspan': [0, 38]}]}, {'self_ref': '#/texts/9', 'parent': {'$ref': '#/body'}, 'children': [], 'content_layer': 'body', 'label': 'text', 'prov': [{'page_no': 2, 'bbox': {'l': 81.2, 't': 608.02, 'r': 432.0, 'b': 572.03, 'coord_origin': 'BOTTOMLEFT'}, 'charspan': [0, 256]}]}], 'headings': ['Praise for Building Machine Learning Systems with a Feature Store'], 'origin': {'mimetype': 'application/pdf', 'binary_hash': 2591788756701469466, 'filename': 'Building+Machine+Learning+Systems+with+a+Feature+Store.pdf'}}}\n"
     ]
    }
   ],
   "source": [
    "print(docs[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created 1333 splits\n",
      "Sample: Praise for Building Machine Learning Systems with a Feature Store\n",
      "I witnessed the rise of feature st...\n"
     ]
    }
   ],
   "source": [
    "if EXPORT_TYPE == ExportType.DOC_CHUNKS:\n",
    "    splits = docs\n",
    "else:\n",
    "    from langchain_text_splitters import MarkdownHeaderTextSplitter\n",
    "    splitter = MarkdownHeaderTextSplitter(\n",
    "        headers_to_split_on=[\n",
    "            (\"#\", \"Header_1\"),\n",
    "            (\"##\", \"Header_2\"),\n",
    "            (\"###\", \"Header_3\"),\n",
    "        ],\n",
    "    )\n",
    "    splits = [split for doc in docs for split in splitter.split_text(doc.page_content)]\n",
    "\n",
    "print(f\"Created {len(splits)} splits\")\n",
    "print(f\"Sample: {splits[0].page_content[:100]}...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-12-02 19:52:07,229 INFO: Use pytorch device_name: mps\n",
      "2025-12-02 19:52:07,232 INFO: Load pretrained SentenceTransformer: sentence-transformers/all-MiniLM-L6-v2\n"
     ]
    }
   ],
   "source": [
    "embeddings = SentenceTransformer(EMBED_MODEL_ID)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batches: 100%|██████████| 42/42 [00:18<00:00,  2.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created 1333 embeddings\n"
     ]
    }
   ],
   "source": [
    "texts = [split.page_content for split in splits]\n",
    "metadatas = [split.metadata for split in splits]\n",
    "\n",
    "vectors = embeddings.encode(texts, show_progress_bar=True, batch_size=32)\n",
    "print(f\"Created {len(vectors)} embeddings\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-12-02 19:52:44,050 INFO: Initializing external client\n",
      "2025-12-02 19:52:44,064 INFO: Base URL: https://c.app.hopsworks.ai:443\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "UserWarning: The installed hopsworks client version 4.4.2 may not be compatible with the connected Hopsworks backend version 4.2.2. \n",
      "To ensure compatibility please install the latest bug fix release matching the minor version of your backend (4.2) by running 'pip install hopsworks==4.2.*'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-12-02 19:52:47,302 INFO: Python Engine initialized.\n",
      "\n",
      "Logged in to project, explore it here https://c.app.hopsworks.ai:443/p/1271977\n"
     ]
    }
   ],
   "source": [
    "project = hopsworks.login()\n",
    "fs = project.get_feature_store()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created dataframe with 1333 rows\n"
     ]
    }
   ],
   "source": [
    "data = []\n",
    "for i, (text, vector, metadata) in enumerate(zip(texts, vectors, metadatas)):\n",
    "    data.append({\n",
    "        'id': i,\n",
    "        'text': text,\n",
    "        'page': metadata.get('page', metadata.get('page_number', 0)),\n",
    "        'embedding': vector\n",
    "    })\n",
    "\n",
    "df = pd.DataFrame(data)\n",
    "print(f\"Created dataframe with {len(df)} rows\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Group created successfully, explore it at \n",
      "https://c.app.hopsworks.ai:443/p/1271977/fs/1258579/fg/1790385\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Uploading Dataframe: 100.00% |██████████| Rows 1333/1333 | Elapsed Time: 00:01 | Remaining Time: 00:00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Launching job: book_embeddings_2_offline_fg_materialization\n",
      "Job started successfully, you can follow the progress at \n",
      "https://c.app.hopsworks.ai:443/p/1271977/jobs/named/book_embeddings_2_offline_fg_materialization/executions\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(Job('book_embeddings_2_offline_fg_materialization', 'SPARK'), None)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "book_fg = fs.get_or_create_feature_group(\n",
    "    name=\"book_embeddings\",\n",
    "    version=2,\n",
    "    primary_key=[\"id\"],\n",
    "    description=\"Book text chunks with embeddings\"\n",
    ")\n",
    "\n",
    "book_fg.insert(df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rag_llm",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}