`Refactor Warbler pack ingestion to process in batches for memory efficiency and add progress updates.`

app.py

@@ -174,13 +174,49 @@ if WARBLER_AVAILABLE:
 
             if pack_docs and pack_manager.should_ingest_packs(api, len(pack_docs)):
                 print(f"[INFO] Ingesting {len(pack_docs)} documents from Warbler packs...")
-                for doc in pack_docs:
-                    success = api.add_document(doc["id"], doc["content"], doc["metadata"])
-                    if not success:
-                        print(f"[WARN] Failed to add document {doc['id']}")
-                packs_loaded = len(pack_docs)
+                total_docs = len(pack_docs)
+                processed = 0
+                failed = 0
+                start_time = time.time()
+                batch_size = 1000
+
+                # Process in batches to avoid memory issues and provide progress
+                for batch_start in range(0, total_docs, batch_size):
+                    batch_end = min(batch_start + batch_size, total_docs)
+                    batch = pack_docs[batch_start:batch_end]
+
+                    batch_processed = 0
+                    batch_failed = 0
+
+                    for doc in batch:
+                        success = api.add_document(doc["id"], doc["content"], doc["metadata"])
+                        if not success:
+                            batch_failed += 1
+                            failed += 1
+                            if failed <= 5:  # Log first few failures
+                                print(f"[WARN] Failed to add document {doc['id']}")
+
+                        batch_processed += 1
+                        processed += 1
+
+                    # Progress update after each batch
+                    elapsed = time.time() - start_time
+                    rate = processed / elapsed if elapsed > 0 else 0
+                    eta = (total_docs - processed) / rate if rate > 0 else 0
+                    print(f"[PROGRESS] {processed}/{total_docs} documents ingested "
+                          f"({processed/total_docs*100:.1f}%) - "
+                          f"{rate:.1f} docs/sec - ETA: {eta/60:.1f} min")
+
+                    # Force garbage collection after large batches to free memory
+                    if processed % 10000 == 0:
+                        import gc
+                        gc.collect()
+
+                packs_loaded = processed
                 pack_manager.mark_packs_ingested(1, packs_loaded)
-                print(f"[OK] Loaded {packs_loaded} documents from Warbler packs")
+                total_time = time.time() - start_time
+                print(f"[OK] Loaded {packs_loaded} documents from Warbler packs "
+                      f"({failed} failed) in {total_time:.1f} seconds")
 
             elif pack_docs:
                 packs_loaded = len(pack_docs)