From 7c69cb22e79fab4bc1d48c0f58d4d5af22305b37 Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 19 Dec 2025 13:09:42 +0100 Subject: [PATCH] Feature: Add sync page to settings - Add Sync navigation tab in settings - Sync UI with status cards (total, vTiger, e-conomic) - Action cards for vTiger and e-conomic sync - Sync log with real-time updates - JavaScript functions for sync operations - Backend sync router with vTiger account sync - Backend vTiger contacts sync with customer linking - Placeholder for e-conomic sync (needs get_customers method) - Name normalization for company matching - CVR number matching and validation --- .env.example | 32 ++- app/core/config.py | 28 +++ app/settings/frontend/settings.html | 372 ++++++++++++++++++++++++++++ app/system/backend/sync_router.py | 365 +++++++++++++++++++++++++++ main.py | 2 + 5 files changed, 798 insertions(+), 1 deletion(-) create mode 100644 app/system/backend/sync_router.py diff --git a/.env.example b/.env.example index 7043df1..1c38371 100644 --- a/.env.example +++ b/.env.example @@ -59,4 +59,34 @@ VTIGER_API_KEY=your_vtiger_api_key # Old vTiger installation (if different from cloud) OLD_VTIGER_URL=http://your-old-vtiger-server.com OLD_VTIGER_USERNAME=your_old_username -OLD_VTIGER_API_KEY=your_old_api_key \ No newline at end of file +OLD_VTIGER_API_KEY=your_old_api_key + +# ===================================================== +# EMAIL SYSTEM CONFIGURATION +# ===================================================== +# IMAP Settings (Standard email) +IMAP_SERVER=imap.gmail.com +IMAP_PORT=993 +IMAP_USERNAME=your_email@gmail.com +IMAP_PASSWORD=your_app_password +IMAP_USE_SSL=true +IMAP_FOLDER=INBOX +IMAP_READ_ONLY=true # Safety: READ-ONLY mode + +# Microsoft Graph API (Alternative to IMAP - for Office365/Outlook) +USE_GRAPH_API=false +GRAPH_TENANT_ID=your_tenant_id +GRAPH_CLIENT_ID=your_client_id +GRAPH_CLIENT_SECRET=your_client_secret +GRAPH_USER_EMAIL=your_email@domain.com + +# Email Processing Settings +EMAIL_TO_TICKET_ENABLED=false +EMAIL_RULES_ENABLED=true +EMAIL_RULES_AUTO_PROCESS=false +EMAIL_AI_ENABLED=false +EMAIL_AUTO_CLASSIFY=false +EMAIL_AI_CONFIDENCE_THRESHOLD=0.7 +EMAIL_MAX_FETCH_PER_RUN=50 +EMAIL_PROCESS_INTERVAL_MINUTES=5 +EMAIL_WORKFLOWS_ENABLED=true \ No newline at end of file diff --git a/app/core/config.py b/app/core/config.py index 42d53c5..bea85fc 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -40,6 +40,34 @@ class Settings(BaseSettings): OLLAMA_ENDPOINT: str = "http://localhost:11434" OLLAMA_MODEL: str = "llama3.2:3b" + # Email System Configuration + # IMAP Settings + IMAP_SERVER: str = "" + IMAP_PORT: int = 993 + IMAP_USERNAME: str = "" + IMAP_PASSWORD: str = "" + IMAP_USE_SSL: bool = True + IMAP_FOLDER: str = "INBOX" + IMAP_READ_ONLY: bool = True + + # Microsoft Graph API (alternative to IMAP) + USE_GRAPH_API: bool = False + GRAPH_TENANT_ID: str = "" + GRAPH_CLIENT_ID: str = "" + GRAPH_CLIENT_SECRET: str = "" + GRAPH_USER_EMAIL: str = "" + + # Email Processing Settings + EMAIL_TO_TICKET_ENABLED: bool = False + EMAIL_RULES_ENABLED: bool = True + EMAIL_RULES_AUTO_PROCESS: bool = False + EMAIL_AI_ENABLED: bool = False + EMAIL_AUTO_CLASSIFY: bool = False + EMAIL_AI_CONFIDENCE_THRESHOLD: float = 0.7 + EMAIL_MAX_FETCH_PER_RUN: int = 50 + EMAIL_PROCESS_INTERVAL_MINUTES: int = 5 + EMAIL_WORKFLOWS_ENABLED: bool = True + # vTiger Cloud Integration VTIGER_URL: str = "" VTIGER_USERNAME: str = "" diff --git a/app/settings/frontend/settings.html b/app/settings/frontend/settings.html index 0e1cd01..0c3da56 100644 --- a/app/settings/frontend/settings.html +++ b/app/settings/frontend/settings.html @@ -92,6 +92,9 @@ Tags + + Sync + AI Prompts @@ -295,7 +298,153 @@
+ +
+
+
Data Synkronisering
+

Synkroniser firmaer og kontakter fra vTiger og e-conomic

+ + +
+
+
+
+
+
+
+ +
+
+
+
Firmaer i Hub
+
-
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
Med vTiger ID
+
-
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
Med e-conomic ID
+
-
+
+
+
+
+
+
+ + +
+
+
+
+
+
+ +
+
+
Sync fra vTiger
+

Hent firmaer og kontakter fra vTiger CRM. Matcher på CVR nummer eller firma navn.

+
+
+
+ + +
+
+
+ + Sidst synkroniseret: Aldrig +
+
+
+
+
+ +
+
+
+
+
+ +
+
+
Sync fra e-conomic
+

Hent kundenumre fra e-conomic. Matcher på CVR nummer eller firma navn.

+
+
+
+ + +
+
+
+ + Sidst synkroniseret: Aldrig +
+
+
+
+
+
+ + +
+
+
+
Synkroniserings Log
+ +
+
+
+
+
+ +

Indlæser log...

+
+
+
+
+
+ + @@ -1215,6 +1364,229 @@ if (tagsNavLink) { }); } +// ====== SYNC MANAGEMENT ====== +let syncLog = []; + +async function loadSyncStats() { + try { + const response = await fetch('/api/v1/customers?limit=10000'); + if (!response.ok) throw new Error('Failed to load customers'); + const customers = await response.json(); + + const stats = { + total: customers.length, + withVtiger: customers.filter(c => c.vtiger_id).length, + withEconomic: customers.filter(c => c.economic_customer_number).length + }; + + document.getElementById('syncStatsCustomers').textContent = stats.total; + document.getElementById('syncStatsVtiger').textContent = stats.withVtiger; + document.getElementById('syncStatsEconomic').textContent = stats.withEconomic; + } catch (error) { + console.error('Error loading sync stats:', error); + } +} + +async function loadSyncLog() { + const container = document.getElementById('syncLogContainer'); + + if (syncLog.length === 0) { + container.innerHTML = ` +
+ +

Ingen synkroniseringer endnu

+ Klik på en af sync knapperne ovenfor for at starte +
+ `; + return; + } + + container.innerHTML = syncLog.map(log => ` +
+
+
+
+ + ${log.title} +
+ ${log.message} +
+ ${new Date(log.timestamp).toLocaleString('da-DK')} +
+
+ `).join(''); +} + +function addSyncLogEntry(title, message, status = 'info') { + syncLog.unshift({ + title, + message, + status, + timestamp: new Date().toISOString() + }); + + // Keep last 50 entries + if (syncLog.length > 50) { + syncLog = syncLog.slice(0, 50); + } + + loadSyncLog(); +} + +async function syncFromVtiger() { + const btn = document.getElementById('btnSyncVtiger'); + btn.disabled = true; + btn.innerHTML = 'Synkroniserer...'; + + try { + addSyncLogEntry('vTiger Sync Startet', 'Henter firmaer fra vTiger...', 'info'); + + const response = await fetch('/api/v1/system/sync/vtiger', { + method: 'POST' + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.detail || 'Sync fejlede'); + } + + const result = await response.json(); + addSyncLogEntry( + 'vTiger Sync Fuldført', + `${result.created || 0} nye firmaer, ${result.updated || 0} opdateret`, + 'success' + ); + + document.getElementById('lastSyncVtiger').textContent = new Date().toLocaleString('da-DK'); + await loadSyncStats(); + showNotification('vTiger sync fuldført!', 'success'); + + } catch (error) { + addSyncLogEntry('vTiger Sync Fejl', error.message, 'error'); + showNotification('Fejl: ' + error.message, 'error'); + } finally { + btn.disabled = false; + btn.innerHTML = 'Sync Firmaer fra vTiger'; + } +} + +async function syncVtigerContacts() { + const btn = document.getElementById('btnSyncVtigerContacts'); + btn.disabled = true; + btn.innerHTML = 'Synkroniserer...'; + + try { + addSyncLogEntry('vTiger Kontakt Sync Startet', 'Henter kontakter fra vTiger...', 'info'); + + const response = await fetch('/api/v1/system/sync/vtiger-contacts', { + method: 'POST' + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.detail || 'Sync fejlede'); + } + + const result = await response.json(); + addSyncLogEntry( + 'vTiger Kontakt Sync Fuldført', + `${result.created || 0} nye kontakter, ${result.updated || 0} opdateret`, + 'success' + ); + + showNotification('vTiger kontakt sync fuldført!', 'success'); + + } catch (error) { + addSyncLogEntry('vTiger Kontakt Sync Fejl', error.message, 'error'); + showNotification('Fejl: ' + error.message, 'error'); + } finally { + btn.disabled = false; + btn.innerHTML = 'Sync Kontakter fra vTiger'; + } +} + +async function syncFromEconomic() { + const btn = document.getElementById('btnSyncEconomic'); + btn.disabled = true; + btn.innerHTML = 'Synkroniserer...'; + + try { + addSyncLogEntry('e-conomic Sync Startet', 'Henter kundenumre fra e-conomic...', 'info'); + + const response = await fetch('/api/v1/system/sync/economic', { + method: 'POST' + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.detail || 'Sync fejlede'); + } + + const result = await response.json(); + addSyncLogEntry( + 'e-conomic Sync Fuldført', + `${result.matched || 0} firmaer matchet med e-conomic kundenumre`, + 'success' + ); + + document.getElementById('lastSyncEconomic').textContent = new Date().toLocaleString('da-DK'); + await loadSyncStats(); + showNotification('e-conomic sync fuldført!', 'success'); + + } catch (error) { + addSyncLogEntry('e-conomic Sync Fejl', error.message, 'error'); + showNotification('Fejl: ' + error.message, 'error'); + } finally { + btn.disabled = false; + btn.innerHTML = 'Sync Firmaer fra e-conomic'; + } +} + +async function syncCvrToEconomic() { + const btn = document.getElementById('btnSyncCvrEconomic'); + btn.disabled = true; + btn.innerHTML = 'Søger...'; + + try { + addSyncLogEntry('CVR Søgning Startet', 'Tjekker CVR numre i e-conomic...', 'info'); + + const response = await fetch('/api/v1/system/sync/cvr-to-economic', { + method: 'POST' + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.detail || 'Søgning fejlede'); + } + + const result = await response.json(); + addSyncLogEntry( + 'CVR Søgning Fuldført', + `Fundet ${result.found || 0} firmaer med CVR match i e-conomic`, + 'success' + ); + + await loadSyncStats(); + showNotification('CVR søgning fuldført!', 'success'); + + } catch (error) { + addSyncLogEntry('CVR Søgning Fejl', error.message, 'error'); + showNotification('Fejl: ' + error.message, 'error'); + } finally { + btn.disabled = false; + btn.innerHTML = 'Find Manglende CVR i e-conomic'; + } +} + +// Load sync data when sync tab is activated +const syncNavLink = document.querySelector('a[data-tab="sync"]'); +if (syncNavLink) { + syncNavLink.addEventListener('click', () => { + loadSyncStats(); + loadSyncLog(); + }); +} + // Load on page ready document.addEventListener('DOMContentLoaded', () => { loadSettings(); diff --git a/app/system/backend/sync_router.py b/app/system/backend/sync_router.py new file mode 100644 index 0000000..2d60e07 --- /dev/null +++ b/app/system/backend/sync_router.py @@ -0,0 +1,365 @@ +""" +System Sync Router +API endpoints for syncing data between vTiger, e-conomic and Hub +""" + +import logging +from fastapi import APIRouter, HTTPException +from typing import Dict, Any +from app.core.database import execute_query +from app.services.vtiger_service import get_vtiger_service +import re + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +def normalize_name(name: str) -> str: + """Normalize company name for matching""" + if not name: + return "" + # Remove common suffixes and punctuation + name = re.sub(r'\b(A/S|ApS|I/S|IVS|v/)\b', '', name, flags=re.IGNORECASE) + name = re.sub(r'[^\w\s]', '', name) # Remove punctuation + return name.lower().strip() + + +@router.post("/sync/vtiger") +async def sync_from_vtiger() -> Dict[str, Any]: + """ + Sync companies from vTiger Accounts module + Matches by CVR number or normalized company name + """ + try: + logger.info("🔄 Starting vTiger accounts sync...") + + vtiger = get_vtiger_service() + + # Query vTiger for all accounts with CVR or name + query = """ + SELECT id, accountname, email1, siccode, cf_accounts_cvr, + cf_854, website1, bill_city, bill_code, bill_country + FROM Accounts + WHERE accountname != '' + LIMIT 1000 + """ + + accounts = await vtiger.query(query) + logger.info(f"📥 Fetched {len(accounts)} accounts from vTiger") + + created_count = 0 + updated_count = 0 + skipped_count = 0 + + for account in accounts: + vtiger_id = account.get('id') + name = account.get('accountname', '').strip() + cvr = account.get('cf_accounts_cvr') or account.get('siccode') + economic_customer_number = account.get('cf_854') # Custom field for e-conomic number + + if not name: + skipped_count += 1 + continue + + # Clean CVR number + if cvr: + cvr = re.sub(r'\D', '', str(cvr))[:8] # Remove non-digits, max 8 chars + if len(cvr) != 8: + cvr = None + + # Try to find existing customer by vTiger ID or CVR + existing = None + if vtiger_id: + existing = execute_query( + "SELECT id FROM customers WHERE vtiger_id = %s", + (vtiger_id,) + ) + + if not existing and cvr: + existing = execute_query( + "SELECT id FROM customers WHERE cvr_number = %s", + (cvr,) + ) + + if not existing: + # Match by normalized name + normalized = normalize_name(name) + all_customers = execute_query("SELECT id, name FROM customers") + for customer in all_customers: + if normalize_name(customer['name']) == normalized: + existing = [customer] + break + + if existing: + # Update existing customer + update_fields = [] + params = [] + + if vtiger_id: + update_fields.append("vtiger_id = %s") + params.append(vtiger_id) + + if cvr: + update_fields.append("cvr_number = %s") + params.append(cvr) + + if economic_customer_number: + update_fields.append("economic_customer_number = %s") + params.append(int(economic_customer_number)) + + update_fields.append("last_synced_at = NOW()") + + if update_fields: + params.append(existing[0]['id']) + query = f"UPDATE customers SET {', '.join(update_fields)} WHERE id = %s" + execute_query(query, tuple(params)) + updated_count += 1 + logger.debug(f"✅ Updated: {name}") + else: + # Create new customer + insert_query = """ + INSERT INTO customers + (name, vtiger_id, cvr_number, economic_customer_number, + email_domain, city, postal_code, country, website, last_synced_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, NOW()) + RETURNING id + """ + + email_domain = account.get('email1', '').split('@')[-1] if '@' in account.get('email1', '') else None + city = account.get('bill_city') + postal_code = account.get('bill_code') + country = account.get('bill_country') or 'DK' + website = account.get('website1') + + result = execute_query(insert_query, ( + name, + vtiger_id, + cvr, + int(economic_customer_number) if economic_customer_number else None, + email_domain, + city, + postal_code, + country, + website + )) + + if result: + created_count += 1 + logger.debug(f"✨ Created: {name}") + + logger.info(f"✅ vTiger sync complete: {created_count} created, {updated_count} updated, {skipped_count} skipped") + + return { + "status": "success", + "created": created_count, + "updated": updated_count, + "skipped": skipped_count, + "total_processed": len(accounts) + } + + except Exception as e: + logger.error(f"❌ vTiger sync error: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/sync/vtiger-contacts") +async def sync_vtiger_contacts() -> Dict[str, Any]: + """ + Sync contacts from vTiger Contacts module + Links to existing Hub customers by Account ID + """ + try: + logger.info("🔄 Starting vTiger contacts sync...") + + vtiger = get_vtiger_service() + + # Query vTiger for contacts + query = """ + SELECT id, firstname, lastname, email, phone, mobile, title, + department, account_id + FROM Contacts + WHERE firstname != '' OR lastname != '' + LIMIT 1000 + """ + + contacts = await vtiger.query(query) + logger.info(f"📥 Fetched {len(contacts)} contacts from vTiger") + + created_count = 0 + updated_count = 0 + skipped_count = 0 + + for contact in contacts: + vtiger_contact_id = contact.get('id') + first_name = contact.get('firstname', '').strip() + last_name = contact.get('lastname', '').strip() + + if not (first_name or last_name): + skipped_count += 1 + continue + + # Find existing contact by vTiger ID + existing = None + if vtiger_contact_id: + existing = execute_query( + "SELECT id FROM contacts WHERE vtiger_id = %s", + (vtiger_contact_id,) + ) + + contact_data = { + 'first_name': first_name, + 'last_name': last_name, + 'email': contact.get('email'), + 'phone': contact.get('phone'), + 'mobile': contact.get('mobile'), + 'title': contact.get('title'), + 'department': contact.get('department'), + 'vtiger_id': vtiger_contact_id + } + + if existing: + # Update existing contact + update_query = """ + UPDATE contacts + SET first_name = %s, last_name = %s, email = %s, phone = %s, + mobile = %s, title = %s, department = %s, updated_at = NOW() + WHERE id = %s + """ + execute_query(update_query, ( + contact_data['first_name'], + contact_data['last_name'], + contact_data['email'], + contact_data['phone'], + contact_data['mobile'], + contact_data['title'], + contact_data['department'], + existing[0]['id'] + )) + updated_count += 1 + contact_id = existing[0]['id'] + else: + # Create new contact + insert_query = """ + INSERT INTO contacts + (first_name, last_name, email, phone, mobile, title, department, vtiger_id) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + RETURNING id + """ + result = execute_query(insert_query, ( + contact_data['first_name'], + contact_data['last_name'], + contact_data['email'], + contact_data['phone'], + contact_data['mobile'], + contact_data['title'], + contact_data['department'], + contact_data['vtiger_id'] + )) + + if result: + created_count += 1 + contact_id = result[0]['id'] + else: + skipped_count += 1 + continue + + # Link contact to customer if account_id exists + account_id = contact.get('account_id') + if account_id and contact_id: + # Find customer by vTiger account ID + customer = execute_query( + "SELECT id FROM customers WHERE vtiger_id = %s", + (account_id,) + ) + + if customer: + # Check if relationship exists + existing_rel = execute_query( + "SELECT id FROM contact_companies WHERE contact_id = %s AND customer_id = %s", + (contact_id, customer[0]['id']) + ) + + if not existing_rel: + # Create relationship + execute_query( + "INSERT INTO contact_companies (contact_id, customer_id, is_primary) VALUES (%s, %s, false)", + (contact_id, customer[0]['id']) + ) + + logger.info(f"✅ vTiger contacts sync complete: {created_count} created, {updated_count} updated, {skipped_count} skipped") + + return { + "status": "success", + "created": created_count, + "updated": updated_count, + "skipped": skipped_count, + "total_processed": len(contacts) + } + + except Exception as e: + logger.error(f"❌ vTiger contacts sync error: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/sync/economic") +async def sync_from_economic() -> Dict[str, Any]: + """ + Sync customer numbers from e-conomic + Matches Hub customers to e-conomic by CVR number or normalized name + """ + try: + logger.info("🔄 Starting e-conomic sync...") + + # Note: This requires adding get_customers() method to economic_service.py + # For now, return a placeholder response + + logger.warning("⚠️ e-conomic sync not fully implemented yet") + + return { + "status": "not_implemented", + "message": "e-conomic customer sync requires get_customers() method in economic_service.py", + "matched": 0 + } + + except Exception as e: + logger.error(f"❌ e-conomic sync error: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/sync/cvr-to-economic") +async def sync_cvr_to_economic() -> Dict[str, Any]: + """ + Find customers in Hub with CVR but without e-conomic customer number + Search e-conomic for matching CVR and update Hub + """ + try: + logger.info("🔄 Starting CVR to e-conomic sync...") + + # Find customers with CVR but no economic_customer_number + customers = execute_query(""" + SELECT id, name, cvr_number + FROM customers + WHERE cvr_number IS NOT NULL + AND cvr_number != '' + AND economic_customer_number IS NULL + LIMIT 100 + """) + + logger.info(f"📥 Found {len(customers)} customers with CVR but no e-conomic number") + + # Note: This requires e-conomic API search functionality + # For now, return placeholder + + logger.warning("⚠️ CVR to e-conomic sync not fully implemented yet") + + return { + "status": "not_implemented", + "message": "Requires e-conomic API search by CVR functionality", + "found": 0, + "candidates": len(customers) + } + + except Exception as e: + logger.error(f"❌ CVR to e-conomic sync error: {e}") + raise HTTPException(status_code=500, detail=str(e)) diff --git a/main.py b/main.py index 95e93c9..c9e56ef 100644 --- a/main.py +++ b/main.py @@ -20,6 +20,7 @@ from app.hardware.backend import router as hardware_api from app.billing.backend import router as billing_api from app.billing.frontend import views as billing_views from app.system.backend import router as system_api +from app.system.backend import sync_router from app.dashboard.backend import views as dashboard_views from app.prepaid.backend import router as prepaid_api from app.prepaid.backend import views as prepaid_views @@ -98,6 +99,7 @@ app.include_router(customers_api.router, prefix="/api/v1", tags=["Customers"]) app.include_router(hardware_api.router, prefix="/api/v1", tags=["Hardware"]) app.include_router(billing_api.router, prefix="/api/v1", tags=["Billing"]) app.include_router(system_api.router, prefix="/api/v1", tags=["System"]) +app.include_router(sync_router.router, prefix="/api/v1/system", tags=["System Sync"]) app.include_router(prepaid_api.router, prefix="/api/v1", tags=["Prepaid Cards"]) app.include_router(ticket_api.router, prefix="/api/v1/ticket", tags=["Tickets"]) app.include_router(vendors_api.router, prefix="/api/v1", tags=["Vendors"])