bmc_hub/app/system/backend/sync_router.py

706 lines
30 KiB
Python
Raw Normal View History

"""
System Sync Router
API endpoints for syncing data between vTiger, e-conomic and Hub
"""
import logging
from fastapi import APIRouter, HTTPException
from typing import Dict, Any
from app.core.database import execute_query
from app.services.vtiger_service import get_vtiger_service
import re
logger = logging.getLogger(__name__)
router = APIRouter()
def normalize_name(name: str) -> str:
"""Normalize company name for matching"""
if not name:
return ""
# Remove common suffixes and punctuation
name = re.sub(r'\b(A/S|ApS|I/S|IVS|v/)\b', '', name, flags=re.IGNORECASE)
name = re.sub(r'[^\w\s]', '', name) # Remove punctuation
return name.lower().strip()
@router.post("/sync/vtiger")
async def sync_from_vtiger() -> Dict[str, Any]:
"""
Link vTiger accounts to existing Hub customers
Matches by CVR or normalized name, updates vtiger_id
"""
try:
logger.info("🔄 Starting vTiger link sync...")
vtiger = get_vtiger_service()
# Fetch ALL accounts - vTiger query API doesn't support LIMIT/OFFSET
# Instead, use recursive queries based on ID to get all records
all_accounts = []
last_id = None
batch_size = 100 # vTiger typically returns ~100 records per query
while True:
# Build query with ID filter to paginate
if last_id is None:
query = f"SELECT id, accountname, email1, siccode, cf_accounts_cvr, website, bill_city, bill_code, bill_country FROM Accounts ORDER BY id LIMIT {batch_size};"
else:
# Get records with ID > last_id to continue pagination
query = f"SELECT id, accountname, email1, siccode, cf_accounts_cvr, website, bill_city, bill_code, bill_country FROM Accounts WHERE id > '{last_id}' ORDER BY id LIMIT {batch_size};"
batch = await vtiger.query(query)
if not batch or len(batch) == 0:
break
all_accounts.extend(batch)
last_id = batch[-1].get('id') # Track last ID for next query
logger.info(f"📥 Fetched batch: {len(batch)} accounts (last ID: {last_id}, total: {len(all_accounts)})")
# If we got less than batch_size, we've reached the end
if len(batch) < batch_size:
break
logger.info(f"📥 Fetched total of {len(all_accounts)} accounts from vTiger")
accounts = all_accounts
linked_count = 0
updated_count = 0
not_found_count = 0
for account in accounts:
vtiger_id = account.get('id')
name = account.get('accountname', '').strip()
cvr = account.get('cf_accounts_cvr') or account.get('siccode')
if not name or not vtiger_id:
not_found_count += 1
continue
# Clean CVR number
if cvr:
cvr = re.sub(r'\D', '', str(cvr))[:8] # Remove non-digits, max 8 chars
if len(cvr) != 8:
cvr = None
# Find existing Hub customer by CVR or normalized name
existing = None
match_method = None
if cvr:
existing = execute_query(
"SELECT id, name, vtiger_id FROM customers WHERE cvr_number = %s",
(cvr,)
)
if existing:
match_method = "CVR"
if not existing:
# Match by normalized name
normalized = normalize_name(name)
all_customers = execute_query("SELECT id, name, vtiger_id FROM customers")
for customer in all_customers:
if normalize_name(customer['name']) == normalized:
existing = [customer]
match_method = "navn"
break
if existing:
# Link vTiger ID to existing customer
current_vtiger_id = existing[0].get('vtiger_id')
# Check if this vtiger_id is already assigned to another customer
vtiger_owner = execute_query(
"SELECT id, name FROM customers WHERE vtiger_id = %s",
(vtiger_id,)
)
if vtiger_owner and vtiger_owner[0]['id'] != existing[0]['id']:
# vTiger ID already belongs to another customer - skip
logger.warning(f"⚠️ vTiger #{vtiger_id} allerede tilknyttet '{vtiger_owner[0]['name']}' - springer over '{existing[0]['name']}'")
not_found_count += 1
continue
if current_vtiger_id is None:
execute_query(
"UPDATE customers SET vtiger_id = %s, last_synced_at = NOW() WHERE id = %s",
(vtiger_id, existing[0]['id'])
)
linked_count += 1
logger.info(f"🔗 Linket: {existing[0]['name']} → vTiger #{vtiger_id} (match: {match_method}, CVR: {cvr or 'ingen'})")
elif current_vtiger_id != vtiger_id:
# Update if different vTiger ID
execute_query(
"UPDATE customers SET vtiger_id = %s, last_synced_at = NOW() WHERE id = %s",
(vtiger_id, existing[0]['id'])
)
updated_count += 1
logger.info(f"✏️ Opdateret vTiger ID: {existing[0]['name']}{vtiger_id} (var: {current_vtiger_id})")
else:
# Already linked, just update timestamp
execute_query("UPDATE customers SET last_synced_at = NOW() WHERE id = %s", (existing[0]['id'],))
else:
not_found_count += 1
logger.debug(f"⏭️ Ikke fundet i Hub: '{name}' (CVR: {cvr or 'ingen'}, normalized: '{normalize_name(name)}')")
logger.info(f"✅ vTiger link sync fuldført: {linked_count} nye linket, {updated_count} opdateret, {not_found_count} ikke fundet af {len(accounts)} totalt")
return {
"status": "success",
"linked": linked_count,
"updated": updated_count,
"not_found": not_found_count,
"total_processed": len(accounts)
}
except Exception as e:
logger.error(f"❌ vTiger sync error: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.post("/sync/vtiger-contacts")
async def sync_vtiger_contacts() -> Dict[str, Any]:
"""
SIMPEL TILGANG - Sync contacts from vTiger and link to customers
Step 1: Fetch all contacts from vTiger
Step 2: Create/update contact in Hub
Step 3: Link to customer IF account_id matches customer.vtiger_id
"""
try:
logger.info("🔄 Starting vTiger contacts sync (SIMPEL VERSION)...")
vtiger = get_vtiger_service()
# ===== STEP 1: FETCH ALL CONTACTS =====
logger.info("📥 STEP 1: Fetching contacts from vTiger...")
all_contacts = []
last_id = None
batch_size = 100
while True:
if last_id is None:
query = f"SELECT id, firstname, lastname, email, phone, mobile, title, department, account_id FROM Contacts ORDER BY id LIMIT {batch_size};"
else:
query = f"SELECT id, firstname, lastname, email, phone, mobile, title, department, account_id FROM Contacts WHERE id > '{last_id}' ORDER BY id LIMIT {batch_size};"
batch = await vtiger.query(query)
if not batch or len(batch) == 0:
break
all_contacts.extend(batch)
last_id = batch[-1].get('id')
logger.info(f" → Batch: {len(batch)} contacts (ID: {last_id}, total: {len(all_contacts)})")
if len(batch) < batch_size:
break
logger.info(f"✅ STEP 1 DONE: {len(all_contacts)} contacts fetched")
# ===== STEP 2 & 3: PROCESS EACH CONTACT =====
logger.info("🔄 STEP 2 & 3: Creating/updating contacts and linking to customers...")
created_count = 0
updated_count = 0
skipped_count = 0
linked_count = 0
customers_created_count = 0
debug_count = 0
for i, contact in enumerate(all_contacts, 1):
vtiger_id = contact.get('id')
first_name = (contact.get('firstname') or '').strip()
last_name = (contact.get('lastname') or '').strip()
email = contact.get('email')
account_id = contact.get('account_id') # This is the vTiger Account ID
# Show progress every 100 contacts
if i % 100 == 0:
logger.info(f" → Progress: {i}/{len(all_contacts)} contacts processed...")
# Must have name
if not first_name and not last_name:
skipped_count += 1
continue
# --- STEP 2A: Check if contact exists ---
existing = execute_query(
"SELECT id FROM contacts WHERE vtiger_id = %s",
(vtiger_id,)
) if vtiger_id else None
# --- STEP 2B: Create or update contact ---
contact_id = None
if existing:
# UPDATE existing
execute_query("""
UPDATE contacts
SET first_name = %s, last_name = %s, email = %s,
phone = %s, mobile = %s, title = %s, department = %s,
updated_at = NOW()
WHERE id = %s
""", (
first_name, last_name, email,
contact.get('phone'), contact.get('mobile'),
contact.get('title'), contact.get('department'),
existing[0]['id']
))
contact_id = existing[0]['id']
updated_count += 1
else:
# CREATE new
result = execute_query("""
INSERT INTO contacts
(first_name, last_name, email, phone, mobile, title, department, vtiger_id)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
RETURNING id
""", (
first_name, last_name, email,
contact.get('phone'), contact.get('mobile'),
contact.get('title'), contact.get('department'),
vtiger_id
))
if result and len(result) > 0:
contact_id = result[0]['id']
created_count += 1
else:
skipped_count += 1
continue
# --- DEBUG LOGGING FOR FIRST 20 CONTACTS ---
if debug_count < 20:
logger.warning(f"🔍 DEBUG {debug_count+1}: Contact '{first_name} {last_name}' (vtiger_id={vtiger_id}, contact_id={contact_id}, account_id={account_id})")
debug_count += 1
# --- STEP 3: LINK TO CUSTOMER ---
if not account_id:
# No account_id means contact is not linked to any account in vTiger
if debug_count <= 20:
logger.warning(f" ↳ No account_id - cannot link")
continue
# Find Hub customer with matching vtiger_id
customer_result = execute_query(
"SELECT id, name FROM customers WHERE vtiger_id = %s",
(account_id,)
)
if not customer_result or len(customer_result) == 0:
# FALLBACK: Create customer from vTiger account if not found
try:
account_query = f"SELECT id, accountname, email1, siccode, cf_accounts_cvr, website, bill_city, bill_code, bill_country FROM Accounts WHERE id = '{account_id}';"
account_data = await vtiger.query(account_query)
if account_data and len(account_data) > 0:
account = account_data[0]
account_name = account.get('accountname', '').strip()
cvr = account.get('cf_accounts_cvr') or account.get('siccode')
if cvr:
cvr = re.sub(r'\D', '', str(cvr))[:8]
if len(cvr) != 8:
cvr = None
# Check if customer already exists by CVR (to avoid duplicates)
if cvr:
existing_by_cvr = execute_query(
"SELECT id, name FROM customers WHERE cvr_number = %s",
(cvr,)
)
if existing_by_cvr:
# Found by CVR - update vtiger_id and use this customer
execute_query(
"UPDATE customers SET vtiger_id = %s, last_synced_at = NOW() WHERE id = %s",
(account_id, existing_by_cvr[0]['id'])
)
customer_id = existing_by_cvr[0]['id']
customer_name = existing_by_cvr[0]['name']
logger.info(f"✅ Matched by CVR and updated vtiger_id: {customer_name}{account_id}")
else:
# Create new customer from vTiger
country = (account.get('bill_country') or 'DK').strip().upper()[:2]
new_customer = execute_query("""
INSERT INTO customers
(name, cvr_number, email, website, city, postal_code, country, vtiger_id, last_synced_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, NOW())
RETURNING id, name
""", (
account_name,
cvr,
account.get('email1'),
account.get('website'),
account.get('bill_city'),
account.get('bill_code'),
country,
account_id
))
if new_customer:
customer_id = new_customer[0]['id']
customer_name = new_customer[0]['name']
customers_created_count += 1
logger.info(f"✨ Created customer from vTiger: {customer_name} (vtiger_id={account_id})")
else:
if debug_count <= 20:
logger.warning(f" ↳ Failed to create customer from account_id={account_id}")
continue
else:
# No CVR - create customer anyway (might be foreign company)
country = (account.get('bill_country') or 'DK').strip().upper()[:2]
new_customer = execute_query("""
INSERT INTO customers
(name, email, website, city, postal_code, country, vtiger_id, last_synced_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, NOW())
RETURNING id, name
""", (
account_name,
account.get('email1'),
account.get('website'),
account.get('bill_city'),
account.get('bill_code'),
country,
account_id
))
if new_customer:
customer_id = new_customer[0]['id']
customer_name = new_customer[0]['name']
customers_created_count += 1
logger.info(f"✨ Created customer from vTiger (no CVR): {customer_name} (vtiger_id={account_id})")
else:
if debug_count <= 20:
logger.warning(f" ↳ Failed to create customer from account_id={account_id}")
continue
else:
if debug_count <= 20:
logger.warning(f" ↳ Account not found in vTiger: {account_id}")
continue
except Exception as e:
logger.error(f"❌ Failed to fetch/create customer from vTiger account {account_id}: {e}")
continue
else:
customer_id = customer_result[0]['id']
customer_name = customer_result[0]['name']
# Check if link already exists
existing_link = execute_query(
"SELECT id FROM contact_companies WHERE contact_id = %s AND customer_id = %s",
(contact_id, customer_id)
)
if existing_link:
# Already linked
if debug_count <= 20:
logger.warning(f" ↳ Already linked to '{customer_name}'")
continue
# CREATE LINK
execute_query(
"INSERT INTO contact_companies (contact_id, customer_id, is_primary) VALUES (%s, %s, false)",
(contact_id, customer_id)
)
linked_count += 1
if linked_count <= 10: # Log first 10 successful links
logger.info(f"🔗 LINKED: {first_name} {last_name}{customer_name}")
logger.info(f"✅ SYNC COMPLETE: created={created_count}, updated={updated_count}, linked={linked_count}, customers_created={customers_created_count}, skipped={skipped_count}, total={len(all_contacts)}")
return {
"status": "success",
"created": created_count,
"updated": updated_count,
"linked": linked_count,
"customers_created": customers_created_count,
"skipped": skipped_count,
"total_processed": len(all_contacts)
}
except Exception as e:
logger.error(f"❌ vTiger contacts sync error: {e}", exc_info=True)
raise HTTPException(status_code=500, detail=str(e))
@router.post("/sync/economic")
async def sync_from_economic() -> Dict[str, Any]:
"""
Sync customers from e-conomic (PRIMARY SOURCE)
Creates/updates Hub customers with e-conomic data
"""
try:
logger.info("🔄 Starting e-conomic customer sync (PRIMARY SOURCE)...")
from app.services.economic_service import EconomicService
economic = EconomicService()
# Get all customers from e-conomic (max 1000 per page)
all_customers = []
page = 0
while True:
customers = await economic.get_customers(page=page, page_size=1000)
if not customers:
break
all_customers.extend(customers)
page += 1
if len(customers) < 1000: # Last page
break
economic_customers = all_customers
logger.info(f"📥 Fetched {len(economic_customers)} customers from e-conomic ({page} pages)")
created_count = 0
updated_count = 0
skipped_count = 0
for eco_customer in economic_customers:
customer_number = eco_customer.get('customerNumber')
cvr = eco_customer.get('corporateIdentificationNumber')
name = eco_customer.get('name', '').strip()
address = eco_customer.get('address', '')
city = eco_customer.get('city', '')
zip_code = eco_customer.get('zip', '')
country = eco_customer.get('country', 'DK')
email = eco_customer.get('email', '')
website = eco_customer.get('website', '')
if not customer_number or not name:
skipped_count += 1
continue
# Clean CVR
if cvr:
cvr = re.sub(r'\D', '', str(cvr))[:8]
if len(cvr) != 8:
cvr = None
# Clean country code (max 2 chars for ISO codes)
if country:
country = str(country).strip().upper()[:2]
if not country:
country = 'DK'
else:
country = 'DK'
# Extract email domain
email_domain = email.split('@')[-1] if '@' in email else None
# Check if customer exists by economic_customer_number OR CVR
existing = execute_query(
"SELECT id FROM customers WHERE economic_customer_number = %s",
(customer_number,)
)
# If not found by customer number, try CVR (to avoid duplicates)
if not existing and cvr:
existing = execute_query(
"SELECT id FROM customers WHERE cvr_number = %s",
(cvr,)
)
if existing:
# Update existing customer (always sync economic_customer_number from e-conomic)
update_query = """
UPDATE customers SET
name = %s,
economic_customer_number = %s,
cvr_number = %s,
email_domain = %s,
city = %s,
postal_code = %s,
country = %s,
website = %s,
last_synced_at = NOW()
WHERE id = %s
"""
execute_query(update_query, (
name, customer_number, cvr, email_domain, city, zip_code, country, website, existing[0]['id']
))
updated_count += 1
logger.info(f"✏️ Opdateret: {name} (e-conomic #{customer_number}, CVR: {cvr or 'ingen'})")
else:
# Create new customer from e-conomic
insert_query = """
INSERT INTO customers
(name, economic_customer_number, cvr_number, email_domain,
city, postal_code, country, website, last_synced_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, NOW())
RETURNING id
"""
result = execute_query(insert_query, (
name, customer_number, cvr, email_domain, city, zip_code, country, website
))
if result:
created_count += 1
logger.info(f"✨ Oprettet: {name} (e-conomic #{customer_number}, CVR: {cvr or 'ingen'})")
else:
skipped_count += 1
logger.info(f"✅ e-conomic sync fuldført: {created_count} oprettet, {updated_count} opdateret, {skipped_count} sprunget over af {len(economic_customers)} totalt")
return {
"status": "success",
"created": created_count,
"updated": updated_count,
"skipped": skipped_count,
"total_processed": len(economic_customers)
}
except Exception as e:
logger.error(f"❌ e-conomic sync error: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.post("/sync/cvr-to-economic")
async def sync_cvr_to_economic() -> Dict[str, Any]:
"""
Find customers in Hub with CVR but without e-conomic customer number
Search e-conomic for matching CVR and update Hub
"""
try:
logger.info("🔄 Starting CVR to e-conomic sync...")
from app.services.economic_service import EconomicService
economic = EconomicService()
# Find customers with CVR but no economic_customer_number
customers = execute_query("""
SELECT id, name, cvr_number
FROM customers
WHERE cvr_number IS NOT NULL
AND cvr_number != ''
AND economic_customer_number IS NULL
LIMIT 100
""")
logger.info(f"📥 Found {len(customers)} customers with CVR but no e-conomic number")
found_count = 0
linked_count = 0
for customer in customers:
cvr = customer['cvr_number']
# Search e-conomic for this CVR
eco_customer = await economic.search_customer_by_cvr(cvr)
if eco_customer:
customer_number = eco_customer.get('customerNumber')
if customer_number:
# Update Hub customer
execute_query(
"UPDATE customers SET economic_customer_number = %s, last_synced_at = NOW() WHERE id = %s",
(customer_number, customer['id'])
)
found_count += 1
linked_count += 1
logger.info(f"✅ Fundet og linket: {customer['name']} (CVR: {cvr}) → e-conomic kunde #{customer_number}")
else:
found_count += 1
logger.warning(f"⚠️ Fundet men mangler kundenummer: {customer['name']} (CVR: {cvr})")
logger.info(f"✅ CVR søgning fuldført: {found_count} fundet, {linked_count} linket af {len(customers)} kontrolleret")
return {
"status": "not_implemented",
"message": "Requires e-conomic API search by CVR functionality",
"found": 0,
"candidates": len(customers)
}
except Exception as e:
logger.error(f"❌ CVR to e-conomic sync error: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/sync/diagnostics")
async def sync_diagnostics() -> Dict[str, Any]:
"""
Diagnostics: Check contact linking coverage
Shows why contacts aren't linking to customers
"""
try:
logger.info("🔍 Running sync diagnostics...")
# Check Hub data
hub_stats = execute_query("""
SELECT
COUNT(*) as total_customers,
COUNT(vtiger_id) as customers_with_vtiger
FROM customers
""")
vtiger_ids_in_hub = execute_query(
"SELECT vtiger_id FROM customers WHERE vtiger_id IS NOT NULL"
)
hub_vtiger_set = set(row['vtiger_id'] for row in vtiger_ids_in_hub)
contact_stats = execute_query("""
SELECT
COUNT(*) as total_contacts,
COUNT(vtiger_id) as contacts_with_vtiger
FROM contacts
""")
link_stats = execute_query("""
SELECT COUNT(*) as total_links
FROM contact_companies
""")
# Fetch sample contacts from vTiger
vtiger = get_vtiger_service()
query = "SELECT id, firstname, lastname, account_id FROM Contacts ORDER BY id LIMIT 200;"
contacts = await vtiger.query(query)
# Analyze
with_account = [c for c in contacts if c.get('account_id')]
account_ids = set(c['account_id'] for c in with_account)
matched_accounts = [aid for aid in account_ids if aid in hub_vtiger_set]
unmatched_accounts = [aid for aid in account_ids if aid not in hub_vtiger_set]
result = {
"hub": {
"total_customers": hub_stats[0]['total_customers'],
"customers_with_vtiger_id": hub_stats[0]['customers_with_vtiger'],
"unique_vtiger_ids": len(hub_vtiger_set),
"total_contacts": contact_stats[0]['total_contacts'],
"contacts_with_vtiger_id": contact_stats[0]['contacts_with_vtiger'],
"total_links": link_stats[0]['total_links']
},
"vtiger_sample": {
"sample_size": len(contacts),
"contacts_with_account_id": len(with_account),
"unique_account_ids": len(account_ids),
"account_ids_found_in_hub": len(matched_accounts),
"account_ids_missing_in_hub": len(unmatched_accounts),
"match_rate_percent": round((len(matched_accounts) / len(account_ids) * 100), 1) if account_ids else 0
},
"examples": {
"matched_account_ids": matched_accounts[:5],
"unmatched_account_ids": unmatched_accounts[:10]
},
"recommendation": ""
}
# Add recommendation
match_rate = result['vtiger_sample']['match_rate_percent']
if match_rate < 50:
result['recommendation'] = f"⚠️ LOW MATCH RATE ({match_rate}%) - Kør først /sync/vtiger for at linke flere accounts"
elif match_rate < 90:
result['recommendation'] = f"⚡ MEDIUM MATCH RATE ({match_rate}%) - Nogen accounts mangler stadig at blive linket"
else:
result['recommendation'] = f"✅ HIGH MATCH RATE ({match_rate}%) - De fleste contacts kan linkes"
logger.info(f"✅ Diagnostics complete: {result['recommendation']}")
return result
except Exception as e:
logger.error(f"❌ Diagnostics error: {e}", exc_info=True)
raise HTTPException(status_code=500, detail=str(e))