Fix: Rettet vTiger pagination - bruger ID-baseret filtering i stedet for LIMIT OFFSET

This commit is contained in:
Christian 2025-12-22 12:59:12 +01:00
parent 94781227b2
commit e5dc0f64d3

View File

@ -36,27 +36,33 @@ async def sync_from_vtiger() -> Dict[str, Any]:
vtiger = get_vtiger_service() vtiger = get_vtiger_service()
# Fetch ALL accounts using pagination (vTiger limits to ~200 per request) # Fetch ALL accounts - vTiger query API doesn't support LIMIT/OFFSET
# Instead, use recursive queries based on ID to get all records
all_accounts = [] all_accounts = []
offset = 0 last_id = None
batch_size = 200 batch_size = 100 # vTiger typically returns ~100 records per query
while True: while True:
query = f"SELECT id, accountname, email1, siccode, cf_accounts_cvr, website, bill_city, bill_code, bill_country FROM Accounts LIMIT {batch_size} OFFSET {offset};" # Build query with ID filter to paginate
if last_id is None:
query = f"SELECT id, accountname, email1, siccode, cf_accounts_cvr, website, bill_city, bill_code, bill_country FROM Accounts ORDER BY id LIMIT {batch_size};"
else:
# Get records with ID > last_id to continue pagination
query = f"SELECT id, accountname, email1, siccode, cf_accounts_cvr, website, bill_city, bill_code, bill_country FROM Accounts WHERE id > '{last_id}' ORDER BY id LIMIT {batch_size};"
batch = await vtiger.query(query) batch = await vtiger.query(query)
if not batch or len(batch) == 0: if not batch or len(batch) == 0:
break break
all_accounts.extend(batch) all_accounts.extend(batch)
logger.info(f"📥 Fetched batch: {len(batch)} accounts (total so far: {len(all_accounts)})") last_id = batch[-1].get('id') # Track last ID for next query
logger.info(f"📥 Fetched batch: {len(batch)} accounts (last ID: {last_id}, total: {len(all_accounts)})")
# If we got less than batch_size, we've reached the end # If we got less than batch_size, we've reached the end
if len(batch) < batch_size: if len(batch) < batch_size:
break break
offset += batch_size
logger.info(f"📥 Fetched total of {len(all_accounts)} accounts from vTiger") logger.info(f"📥 Fetched total of {len(all_accounts)} accounts from vTiger")
accounts = all_accounts accounts = all_accounts