Update app/policy_vector_db.py

app/policy_vector_db.py

@@ -42,7 +42,7 @@ class PolicyVectorDB:
         except Exception as e:
             logger.warning(f"Could not retrieve existing IDs from ChromaDB: {e}. Assuming no existing IDs for now.")
 
-        new_chunks = [chunk for chunk in chunks if chunk.get('id') and chunk['id'] not in existing_ids]
+        new_chunks = [chunk for chunk in chunks if chunk.get('id') and str(chunk.get('id')) not in existing_ids]
 
         if not new_chunks:
             logger.info("No new chunks to add.")
@@ -53,7 +53,8 @@ class PolicyVectorDB:
         for i in range(0, len(new_chunks), batch_size):
             batch = new_chunks[i:i + batch_size]
             texts = [chunk['text'] for chunk in batch]
-            ids = [chunk['id'] for chunk in batch]
+            # --- FIX: Ensure all IDs are strings before adding to ChromaDB ---
+            ids = [str(chunk['id']) for chunk in batch]
             metadatas = [self._flatten_metadata(chunk['metadata']) if chunk.get('metadata') else {} for chunk in batch]
             embeddings = self.embedding_model.encode(texts, show_progress_bar=False).tolist()
             collection.add(ids=ids, embeddings=embeddings, documents=texts, metadatas=metadatas)
@@ -89,8 +90,10 @@ def ensure_db_populated(db_instance: PolicyVectorDB, chunks_file_path: str):
                 logger.error(f"Chunks file not found at {chunks_file_path}. Cannot populate DB.")
                 return False
             
+            chunks_to_add = []
             with open(chunks_file_path, 'r', encoding='utf-8') as f:
-                chunks_to_add = json.load(f)
+                for line in f:
+                    chunks_to_add.append(json.loads(line))
             
             if not chunks_to_add:
                 logger.warning(f"Chunks file at {chunks_file_path} is empty. No data to add to DB.")