Harshavard21 commited on
Commit
2929eee
·
1 Parent(s): 923d3a8

Pin pyarrow==18.1.0 to resolve NumPy 2.0 conflict

Browse files
Files changed (2) hide show
  1. requirements.txt +1 -0
  2. scripts/migrate_qdrant.py +46 -14
requirements.txt CHANGED
@@ -28,6 +28,7 @@ langchain-community==0.3.12 # Community integrations
28
  # --- Evaluation ---
29
  ragas==0.2.6 # RAG evaluation framework (Faithfulness, Relevance etc.)
30
  datasets==3.2.0 # HuggingFace datasets (used by RAGAS)
 
31
 
32
  # --- UI / API ---
33
  fastapi==0.111.0 # REST API backend
 
28
  # --- Evaluation ---
29
  ragas==0.2.6 # RAG evaluation framework (Faithfulness, Relevance etc.)
30
  datasets==3.2.0 # HuggingFace datasets (used by RAGAS)
31
+ pyarrow==18.1.0 # Pin compatible with numpy 1.26.4
32
 
33
  # --- UI / API ---
34
  fastapi==0.111.0 # REST API backend
scripts/migrate_qdrant.py CHANGED
@@ -26,7 +26,7 @@ def migrate():
26
  print("Please add them to your .env file.")
27
  return
28
 
29
- cloud_client = QdrantClient(url=url, port=443, api_key=api_key, timeout=60)
30
 
31
  print(f"Connected to Local ({local_path}) and Cloud ({url})")
32
 
@@ -49,11 +49,32 @@ def migrate():
49
  cloud_client.create_payload_index(collection_name=collection_name, field_name=f, field_schema=PayloadSchemaType.KEYWORD)
50
  cloud_client.create_payload_index(collection_name=collection_name, field_name='page_number', field_schema=PayloadSchemaType.INTEGER)
51
 
52
- # 4. Migrate points in batches
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
53
  print("Starting migration...")
54
- batch_size = 100
 
55
  offset = None
56
- total_migrated = 0
57
 
58
  while True:
59
  # Scroll through local points
@@ -68,26 +89,37 @@ def migrate():
68
  if not records:
69
  break
70
 
71
- # Convert Record objects to PointStruct objects
72
  point_structs = [
73
  PointStruct(id=record.id, vector=record.vector, payload=record.payload)
74
  for record in records
 
75
  ]
76
 
77
- # Upsert to cloud
78
- cloud_client.upsert(
79
- collection_name=collection_name,
80
- points=point_structs
81
- )
82
-
83
- total_migrated += len(records)
84
- print(f"Migrated {total_migrated} points...")
 
 
 
 
 
 
 
 
 
 
85
 
86
  offset = next_offset
87
  if offset is None:
88
  break
89
 
90
- print(f"\nMigration complete! Successfully moved {total_migrated} vectors to Qdrant Cloud.")
91
 
92
  if __name__ == "__main__":
93
  migrate()
 
26
  print("Please add them to your .env file.")
27
  return
28
 
29
+ cloud_client = QdrantClient(url=url, port=443, api_key=api_key, timeout=120)
30
 
31
  print(f"Connected to Local ({local_path}) and Cloud ({url})")
32
 
 
49
  cloud_client.create_payload_index(collection_name=collection_name, field_name=f, field_schema=PayloadSchemaType.KEYWORD)
50
  cloud_client.create_payload_index(collection_name=collection_name, field_name='page_number', field_schema=PayloadSchemaType.INTEGER)
51
 
52
+ # 4. Fetch existing points in Cloud to resume without re-uploading
53
+ print("Checking existing points in Cloud for fast resume...")
54
+ existing_ids = set()
55
+ cloud_offset = None
56
+ while True:
57
+ records, next_c_offset = cloud_client.scroll(
58
+ collection_name=collection_name,
59
+ limit=1000,
60
+ offset=cloud_offset,
61
+ with_payload=False,
62
+ with_vectors=False
63
+ )
64
+ for r in records:
65
+ existing_ids.add(r.id)
66
+ cloud_offset = next_c_offset
67
+ if cloud_offset is None or not records:
68
+ break
69
+
70
+ print(f"Found {len(existing_ids)} points already migrated. Only uploading remaining points...")
71
+
72
+ # 5. Migrate remaining points in batches
73
  print("Starting migration...")
74
+ import time
75
+ batch_size = 25
76
  offset = None
77
+ total_migrated = len(existing_ids)
78
 
79
  while True:
80
  # Scroll through local points
 
89
  if not records:
90
  break
91
 
92
+ # Filter out points that already exist in cloud
93
  point_structs = [
94
  PointStruct(id=record.id, vector=record.vector, payload=record.payload)
95
  for record in records
96
+ if record.id not in existing_ids
97
  ]
98
 
99
+ if point_structs:
100
+ # Upsert to cloud with retries
101
+ for attempt in range(1, 5):
102
+ try:
103
+ cloud_client.upsert(
104
+ collection_name=collection_name,
105
+ points=point_structs,
106
+ wait=True
107
+ )
108
+ break
109
+ except Exception as e:
110
+ if attempt == 4:
111
+ raise e
112
+ print(f"Batch write retry {attempt}/4 due to: {e}... waiting 3s")
113
+ time.sleep(3)
114
+
115
+ total_migrated += len(point_structs)
116
+ print(f"Progress: {total_migrated}/5180 points...")
117
 
118
  offset = next_offset
119
  if offset is None:
120
  break
121
 
122
+ print(f"\nMigration complete! All {total_migrated} vectors are live on Qdrant Cloud.")
123
 
124
  if __name__ == "__main__":
125
  migrate()