""" System Sync Router API endpoints for syncing data between vTiger, e-conomic and Hub SYNC ARCHITECTURE - Field Ownership: ===================================== E-CONOMIC owns and syncs: - economic_customer_number (primary key from e-conomic) - address, city, postal_code, country (physical address) - email_domain, website (contact information) - cvr_number (used for matching only, not overwritten if already set) vTIGER owns and syncs: - vtiger_id (primary key from vTiger) - vtiger_account_no - Contact records and contact-company relationships HUB owns (manual or first-sync only): - name (can be synced initially but not overwritten) - cvr_number (used for matching, set once) - Tags, notes, custom fields SYNC RULES: =========== 1. NEVER overwrite source ID if already set (vtiger_id, economic_customer_number) 2. Matching is source-specific (e-conomic: strict economic_customer_number only) 3. Re-sync is idempotent - can run multiple times safely 4. Contact relationships are REPLACED on sync (not added) 5. Each sync only updates fields it owns """ import logging from fastapi import APIRouter, Depends, HTTPException from typing import Dict, Any from app.core.database import execute_query from app.core.auth_dependencies import require_any_permission from app.services.vtiger_service import get_vtiger_service import re logger = logging.getLogger(__name__) router = APIRouter() sync_admin_access = require_any_permission("users.manage", "system.admin") def normalize_name(name: str) -> str: """Normalize company name for matching""" if not name: return "" # Remove common suffixes and punctuation name = re.sub(r'\b(A/S|ApS|I/S|IVS|v/)\b', '', name, flags=re.IGNORECASE) name = re.sub(r'[^\w\s]', '', name) # Remove punctuation return name.lower().strip() @router.post("/sync/vtiger") async def sync_from_vtiger(current_user: dict = Depends(sync_admin_access)) -> Dict[str, Any]: """ Link vTiger accounts to existing Hub customers Matches by CVR or normalized name, updates vtiger_id """ try: logger.info("🔄 Starting vTiger link sync...") vtiger = get_vtiger_service() # Fetch ALL accounts - vTiger query API doesn't support LIMIT/OFFSET # Instead, use recursive queries based on ID to get all records all_accounts = [] last_id = None batch_size = 100 # vTiger typically returns ~100 records per query while True: # Build query with ID filter to paginate if last_id is None: query = f"SELECT id, accountname, email1, siccode, cf_accounts_cvr, website, bill_city, bill_code, bill_country FROM Accounts ORDER BY id LIMIT {batch_size};" else: # Get records with ID > last_id to continue pagination query = f"SELECT id, accountname, email1, siccode, cf_accounts_cvr, website, bill_city, bill_code, bill_country FROM Accounts WHERE id > '{last_id}' ORDER BY id LIMIT {batch_size};" batch = await vtiger.query(query) if not batch or len(batch) == 0: break all_accounts.extend(batch) last_id = batch[-1].get('id') # Track last ID for next query logger.info(f"📥 Fetched batch: {len(batch)} accounts (last ID: {last_id}, total: {len(all_accounts)})") # If we got less than batch_size, we've reached the end if len(batch) < batch_size: break logger.info(f"📥 Fetched total of {len(all_accounts)} accounts from vTiger") accounts = all_accounts linked_count = 0 updated_count = 0 not_found_count = 0 for account in accounts: vtiger_id = account.get('id') name = account.get('accountname', '').strip() cvr = account.get('cf_accounts_cvr') or account.get('siccode') if not name or not vtiger_id: not_found_count += 1 continue # Clean CVR number if cvr: cvr = re.sub(r'\D', '', str(cvr))[:8] # Remove non-digits, max 8 chars if len(cvr) != 8: cvr = None # Find existing Hub customer by CVR or normalized name existing = None match_method = None if cvr: existing = execute_query( "SELECT id, name, vtiger_id FROM customers WHERE cvr_number = %s", (cvr,) ) if existing: match_method = "CVR" if not existing: # Match by normalized name normalized = normalize_name(name) all_customers = execute_query("SELECT id, name, vtiger_id FROM customers") for customer in all_customers: if normalize_name(customer['name']) == normalized: existing = [customer] match_method = "navn" break if existing: # Link vTiger ID to existing customer current_vtiger_id = existing[0].get('vtiger_id') # Check if this vtiger_id is already assigned to another customer vtiger_owner = execute_query( "SELECT id, name FROM customers WHERE vtiger_id = %s", (vtiger_id,) ) if vtiger_owner and vtiger_owner[0]['id'] != existing[0]['id']: # vTiger ID already belongs to another customer - skip logger.warning(f"⚠️ vTiger #{vtiger_id} allerede tilknyttet '{vtiger_owner[0]['name']}' - springer over '{existing[0]['name']}'") not_found_count += 1 continue if current_vtiger_id is None: # Only set vtiger_id if it's currently NULL execute_query( "UPDATE customers SET vtiger_id = %s, last_synced_at = NOW() WHERE id = %s", (vtiger_id, existing[0]['id']) ) linked_count += 1 logger.info(f"🔗 Linket: {existing[0]['name']} → vTiger #{vtiger_id} (match: {match_method}, CVR: {cvr or 'ingen'})") elif current_vtiger_id != vtiger_id: # SKIP if different vTiger ID - do NOT overwrite existing vtiger_id logger.warning(f"⚠️ Springer over: {existing[0]['name']} har allerede vTiger #{current_vtiger_id}, vil ikke overskrive med #{vtiger_id}") not_found_count += 1 continue else: # Already linked, just update timestamp execute_query("UPDATE customers SET last_synced_at = NOW() WHERE id = %s", (existing[0]['id'],)) else: not_found_count += 1 logger.debug(f"⏭️ Ikke fundet i Hub: '{name}' (CVR: {cvr or 'ingen'}, normalized: '{normalize_name(name)}')") logger.info(f"✅ vTiger link sync fuldført: {linked_count} nye linket, {updated_count} opdateret, {not_found_count} ikke fundet af {len(accounts)} totalt") return { "status": "success", "linked": linked_count, "updated": updated_count, "not_found": not_found_count, "total_processed": len(accounts) } except Exception as e: logger.error(f"❌ vTiger sync error: {e}") raise HTTPException(status_code=500, detail=str(e)) @router.post("/sync/vtiger-contacts") async def sync_vtiger_contacts(current_user: dict = Depends(sync_admin_access)) -> Dict[str, Any]: """ SIMPEL TILGANG - Sync contacts from vTiger and link to customers Step 1: Fetch all contacts from vTiger Step 2: Create/update contact in Hub Step 3: Link to customer IF account_id matches customer.vtiger_id """ try: logger.info("🔄 Starting vTiger contacts sync (SIMPEL VERSION)...") vtiger = get_vtiger_service() # ===== STEP 1: FETCH ALL CONTACTS ===== logger.info("📥 STEP 1: Fetching contacts from vTiger...") all_contacts = [] last_id = None batch_size = 100 while True: if last_id is None: query = f"SELECT id, firstname, lastname, email, phone, mobile, title, department, account_id FROM Contacts ORDER BY id LIMIT {batch_size};" else: query = f"SELECT id, firstname, lastname, email, phone, mobile, title, department, account_id FROM Contacts WHERE id > '{last_id}' ORDER BY id LIMIT {batch_size};" batch = await vtiger.query(query) if not batch or len(batch) == 0: break all_contacts.extend(batch) last_id = batch[-1].get('id') logger.info(f" → Batch: {len(batch)} contacts (ID: {last_id}, total: {len(all_contacts)})") if len(batch) < batch_size: break logger.info(f"✅ STEP 1 DONE: {len(all_contacts)} contacts fetched") # ===== STEP 2 & 3: PROCESS EACH CONTACT ===== logger.info("🔄 STEP 2 & 3: Creating/updating contacts and linking to customers...") created_count = 0 updated_count = 0 skipped_count = 0 linked_count = 0 customers_created_count = 0 debug_count = 0 for i, contact in enumerate(all_contacts, 1): vtiger_id = contact.get('id') first_name = (contact.get('firstname') or '').strip() last_name = (contact.get('lastname') or '').strip() email = contact.get('email') account_id = contact.get('account_id') # This is the vTiger Account ID # Show progress every 100 contacts if i % 100 == 0: logger.info(f" → Progress: {i}/{len(all_contacts)} contacts processed...") # Must have name if not first_name and not last_name: skipped_count += 1 continue # --- STEP 2A: Check if contact exists --- existing = execute_query( "SELECT id FROM contacts WHERE vtiger_id = %s", (vtiger_id,) ) if vtiger_id else None # --- STEP 2B: Create or update contact --- contact_id = None if existing: # UPDATE existing execute_query(""" UPDATE contacts SET first_name = %s, last_name = %s, email = %s, phone = %s, mobile = %s, title = %s, department = %s, updated_at = NOW() WHERE id = %s """, ( first_name, last_name, email, contact.get('phone'), contact.get('mobile'), contact.get('title'), contact.get('department'), existing[0]['id'] )) contact_id = existing[0]['id'] updated_count += 1 else: # CREATE new result = execute_query(""" INSERT INTO contacts (first_name, last_name, email, phone, mobile, title, department, vtiger_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s) RETURNING id """, ( first_name, last_name, email, contact.get('phone'), contact.get('mobile'), contact.get('title'), contact.get('department'), vtiger_id )) if result and len(result) > 0: contact_id = result[0]['id'] created_count += 1 else: skipped_count += 1 continue # --- DEBUG LOGGING FOR FIRST 20 CONTACTS --- if debug_count < 20: logger.warning(f"🔍 DEBUG {debug_count+1}: Contact '{first_name} {last_name}' (vtiger_id={vtiger_id}, contact_id={contact_id}, account_id={account_id})") debug_count += 1 # --- STEP 3: LINK TO CUSTOMER --- if not account_id: # No account_id means contact is not linked to any account in vTiger if debug_count <= 20: logger.warning(f" ↳ No account_id - cannot link") continue # Find Hub customer with matching vtiger_id customer_result = execute_query( "SELECT id, name FROM customers WHERE vtiger_id = %s", (account_id,) ) if not customer_result or len(customer_result) == 0: # FALLBACK: Create customer from vTiger account if not found try: account_query = f"SELECT id, accountname, email1, siccode, cf_accounts_cvr, website, bill_city, bill_code, bill_country FROM Accounts WHERE id = '{account_id}';" account_data = await vtiger.query(account_query) if account_data and len(account_data) > 0: account = account_data[0] account_name = account.get('accountname', '').strip() cvr = account.get('cf_accounts_cvr') or account.get('siccode') if cvr: cvr = re.sub(r'\D', '', str(cvr))[:8] if len(cvr) != 8: cvr = None # Check if customer already exists by CVR (to avoid duplicates) if cvr: existing_by_cvr = execute_query( "SELECT id, name FROM customers WHERE cvr_number = %s", (cvr,) ) if existing_by_cvr: # Found by CVR - update vtiger_id and use this customer execute_query( "UPDATE customers SET vtiger_id = %s, last_synced_at = NOW() WHERE id = %s", (account_id, existing_by_cvr[0]['id']) ) customer_id = existing_by_cvr[0]['id'] customer_name = existing_by_cvr[0]['name'] logger.info(f"✅ Matched by CVR and updated vtiger_id: {customer_name} → {account_id}") else: # Create new customer from vTiger country = (account.get('bill_country') or 'DK').strip().upper()[:2] new_customer = execute_query(""" INSERT INTO customers (name, cvr_number, email, website, city, postal_code, country, vtiger_id, last_synced_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, NOW()) RETURNING id, name """, ( account_name, cvr, account.get('email1'), account.get('website'), account.get('bill_city'), account.get('bill_code'), country, account_id )) if new_customer: customer_id = new_customer[0]['id'] customer_name = new_customer[0]['name'] customers_created_count += 1 logger.info(f"✨ Created customer from vTiger: {customer_name} (vtiger_id={account_id})") else: if debug_count <= 20: logger.warning(f" ↳ Failed to create customer from account_id={account_id}") continue else: # No CVR - create customer anyway (might be foreign company) country = (account.get('bill_country') or 'DK').strip().upper()[:2] new_customer = execute_query(""" INSERT INTO customers (name, email, website, city, postal_code, country, vtiger_id, last_synced_at) VALUES (%s, %s, %s, %s, %s, %s, %s, NOW()) RETURNING id, name """, ( account_name, account.get('email1'), account.get('website'), account.get('bill_city'), account.get('bill_code'), country, account_id )) if new_customer: customer_id = new_customer[0]['id'] customer_name = new_customer[0]['name'] customers_created_count += 1 logger.info(f"✨ Created customer from vTiger (no CVR): {customer_name} (vtiger_id={account_id})") else: if debug_count <= 20: logger.warning(f" ↳ Failed to create customer from account_id={account_id}") continue else: if debug_count <= 20: logger.warning(f" ↳ Account not found in vTiger: {account_id}") continue except Exception as e: logger.error(f"❌ Failed to fetch/create customer from vTiger account {account_id}: {e}") continue else: customer_id = customer_result[0]['id'] customer_name = customer_result[0]['name'] # DELETE existing links for this contact (we replace, not append) # This ensures re-sync updates links to match current vTiger state execute_query( "DELETE FROM contact_companies WHERE contact_id = %s", (contact_id,) ) # CREATE new link from vTiger execute_query( "INSERT INTO contact_companies (contact_id, customer_id, is_primary) VALUES (%s, %s, false)", (contact_id, customer_id) ) linked_count += 1 if linked_count <= 10: # Log first 10 successful links logger.info(f"🔗 LINKED: {first_name} {last_name} → {customer_name}") logger.info(f"✅ SYNC COMPLETE: created={created_count}, updated={updated_count}, linked={linked_count}, customers_created={customers_created_count}, skipped={skipped_count}, total={len(all_contacts)}") return { "status": "success", "created": created_count, "updated": updated_count, "linked": linked_count, "customers_created": customers_created_count, "skipped": skipped_count, "total_processed": len(all_contacts) } except Exception as e: logger.error(f"❌ vTiger contacts sync error: {e}", exc_info=True) raise HTTPException(status_code=500, detail=str(e)) @router.post("/sync/economic") async def sync_from_economic(current_user: dict = Depends(sync_admin_access)) -> Dict[str, Any]: """ Sync customers from e-conomic (PRIMARY SOURCE) Creates/updates Hub customers with e-conomic data """ try: logger.info("🔄 Starting e-conomic customer sync (PRIMARY SOURCE)...") from app.services.economic_service import EconomicService economic = EconomicService() # Get all customers from e-conomic (max 1000 per page) all_customers = [] page = 0 while True: customers = await economic.get_customers(page=page, page_size=1000) if not customers: break all_customers.extend(customers) page += 1 if len(customers) < 1000: # Last page break economic_customers = all_customers logger.info(f"📥 Fetched {len(economic_customers)} customers from e-conomic ({page} pages)") created_count = 0 updated_count = 0 skipped_count = 0 conflict_count = 0 for eco_customer in economic_customers: customer_number_raw = eco_customer.get('customerNumber') customer_number = str(customer_number_raw).strip() if customer_number_raw is not None else None cvr = eco_customer.get('corporateIdentificationNumber') name = eco_customer.get('name', '').strip() address = eco_customer.get('address', '') city = eco_customer.get('city', '') zip_code = eco_customer.get('zip', '') country = eco_customer.get('country', 'DK') email = eco_customer.get('email', '') website = eco_customer.get('website', '') if not customer_number or not name: skipped_count += 1 continue # Clean CVR if cvr: cvr = re.sub(r'\D', '', str(cvr))[:8] if len(cvr) != 8: cvr = None # Clean country code (max 2 chars for ISO codes) if country: country = str(country).strip().upper()[:2] if not country: country = 'DK' else: country = 'DK' # Extract email domain email_domain = email.split('@')[-1] if '@' in email else None # Strict matching: ONLY match by economic_customer_number existing = execute_query( "SELECT id, name FROM customers WHERE economic_customer_number = %s ORDER BY id", (customer_number,) ) # Conflict handling: duplicate local rows for same e-conomic number if len(existing) > 1: conflict_count += 1 skipped_count += 1 duplicate_ids = ", ".join(str(row['id']) for row in existing) logger.error( "❌ Konflikt: e-conomic #%s matcher %s lokale kunder (ids: %s) - springer over", customer_number, len(existing), duplicate_ids ) continue if existing: target_customer_id = existing[0]['id'] # Update existing customer - ONLY update fields e-conomic owns # E-conomic does NOT overwrite: name, cvr_number (set once only) update_query = """ UPDATE customers SET economic_customer_number = %s, email_domain = %s, address = %s, city = %s, postal_code = %s, country = %s, website = %s, last_synced_at = NOW() WHERE id = %s """ execute_query(update_query, ( customer_number, email_domain, address, city, zip_code, country, website, target_customer_id )) updated_count += 1 logger.info( "✏️ Opdateret lokal kunde id=%s: %s (e-conomic #%s, CVR: %s)", target_customer_id, name, customer_number, cvr or 'ingen' ) else: # Create new customer from e-conomic insert_query = """ INSERT INTO customers (name, economic_customer_number, cvr_number, email_domain, address, city, postal_code, country, website, last_synced_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, NOW()) RETURNING id """ result = execute_query(insert_query, ( name, customer_number, cvr, email_domain, address, city, zip_code, country, website )) if result: new_customer_id = result[0]['id'] created_count += 1 logger.info( "✨ Oprettet lokal kunde id=%s: %s (e-conomic #%s, CVR: %s)", new_customer_id, name, customer_number, cvr or 'ingen' ) else: skipped_count += 1 logger.info( "✅ e-conomic sync fuldført: %s oprettet, %s opdateret, %s konflikter, %s sprunget over af %s totalt", created_count, updated_count, conflict_count, skipped_count, len(economic_customers) ) return { "status": "success", "created": created_count, "updated": updated_count, "conflicts": conflict_count, "skipped": skipped_count, "total_processed": len(economic_customers) } except Exception as e: logger.error(f"❌ e-conomic sync error: {e}") raise HTTPException(status_code=500, detail=str(e)) @router.post("/sync/cvr-to-economic") async def sync_cvr_to_economic(current_user: dict = Depends(sync_admin_access)) -> Dict[str, Any]: """ Find customers in Hub with CVR but without e-conomic customer number Search e-conomic for matching CVR and update Hub """ try: logger.info("🔄 Starting CVR to e-conomic sync...") from app.services.economic_service import EconomicService economic = EconomicService() # Find customers with CVR but no economic_customer_number customers = execute_query(""" SELECT id, name, cvr_number FROM customers WHERE cvr_number IS NOT NULL AND cvr_number != '' AND economic_customer_number IS NULL LIMIT 100 """) logger.info(f"📥 Found {len(customers)} customers with CVR but no e-conomic number") found_count = 0 linked_count = 0 for customer in customers: cvr = customer['cvr_number'] # Search e-conomic for this CVR eco_customer = await economic.search_customer_by_cvr(cvr) if eco_customer: customer_number = eco_customer.get('customerNumber') if customer_number: # Update Hub customer execute_query( "UPDATE customers SET economic_customer_number = %s, last_synced_at = NOW() WHERE id = %s", (customer_number, customer['id']) ) found_count += 1 linked_count += 1 logger.info(f"✅ Fundet og linket: {customer['name']} (CVR: {cvr}) → e-conomic kunde #{customer_number}") else: found_count += 1 logger.warning(f"⚠️ Fundet men mangler kundenummer: {customer['name']} (CVR: {cvr})") logger.info(f"✅ CVR søgning fuldført: {found_count} fundet, {linked_count} linket af {len(customers)} kontrolleret") return { "status": "not_implemented", "message": "Requires e-conomic API search by CVR functionality", "found": 0, "candidates": len(customers) } except Exception as e: logger.error(f"❌ CVR to e-conomic sync error: {e}") raise HTTPException(status_code=500, detail=str(e)) @router.get("/sync/diagnostics") async def sync_diagnostics(current_user: dict = Depends(sync_admin_access)) -> Dict[str, Any]: """ Diagnostics: Check contact linking coverage Shows why contacts aren't linking to customers """ try: logger.info("🔍 Running sync diagnostics...") # Check Hub data hub_stats = execute_query(""" SELECT COUNT(*) as total_customers, COUNT(vtiger_id) as customers_with_vtiger FROM customers """) vtiger_ids_in_hub = execute_query( "SELECT vtiger_id FROM customers WHERE vtiger_id IS NOT NULL" ) hub_vtiger_set = set(row['vtiger_id'] for row in vtiger_ids_in_hub) contact_stats = execute_query(""" SELECT COUNT(*) as total_contacts, COUNT(vtiger_id) as contacts_with_vtiger FROM contacts """) link_stats = execute_query(""" SELECT COUNT(*) as total_links FROM contact_companies """) # Fetch sample contacts from vTiger vtiger = get_vtiger_service() query = "SELECT id, firstname, lastname, account_id FROM Contacts ORDER BY id LIMIT 200;" contacts = await vtiger.query(query) # Analyze with_account = [c for c in contacts if c.get('account_id')] account_ids = set(c['account_id'] for c in with_account) matched_accounts = [aid for aid in account_ids if aid in hub_vtiger_set] unmatched_accounts = [aid for aid in account_ids if aid not in hub_vtiger_set] result = { "hub": { "total_customers": hub_stats[0]['total_customers'], "customers_with_vtiger_id": hub_stats[0]['customers_with_vtiger'], "unique_vtiger_ids": len(hub_vtiger_set), "total_contacts": contact_stats[0]['total_contacts'], "contacts_with_vtiger_id": contact_stats[0]['contacts_with_vtiger'], "total_links": link_stats[0]['total_links'] }, "vtiger_sample": { "sample_size": len(contacts), "contacts_with_account_id": len(with_account), "unique_account_ids": len(account_ids), "account_ids_found_in_hub": len(matched_accounts), "account_ids_missing_in_hub": len(unmatched_accounts), "match_rate_percent": round((len(matched_accounts) / len(account_ids) * 100), 1) if account_ids else 0 }, "examples": { "matched_account_ids": matched_accounts[:5], "unmatched_account_ids": unmatched_accounts[:10] }, "recommendation": "" } # Add recommendation match_rate = result['vtiger_sample']['match_rate_percent'] if match_rate < 50: result['recommendation'] = f"⚠️ LOW MATCH RATE ({match_rate}%) - Kør først /sync/vtiger for at linke flere accounts" elif match_rate < 90: result['recommendation'] = f"⚡ MEDIUM MATCH RATE ({match_rate}%) - Nogen accounts mangler stadig at blive linket" else: result['recommendation'] = f"✅ HIGH MATCH RATE ({match_rate}%) - De fleste contacts kan linkes" logger.info(f"✅ Diagnostics complete: {result['recommendation']}") return result except Exception as e: logger.error(f"❌ Diagnostics error: {e}", exc_info=True) raise HTTPException(status_code=500, detail=str(e))