bmc_hub/scripts/import_from_omnisync.py

"""
Import customers and contacts from OmniSync database with better error handling
"""

import psycopg2
from psycopg2.extras import RealDictCursor
import sqlite3
import os
import logging

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

OMNISYNC_DB = '/omnisync_data/fakturering.db'


def get_postgres_connection():
    """Get PostgreSQL connection"""
    database_url = os.getenv('DATABASE_URL', 'postgresql://bmc_hub:bmc_hub@postgres:5432/bmc_hub')
    return psycopg2.connect(database_url, cursor_factory=RealDictCursor)


def get_sqlite_connection():
    """Get SQLite connection to OmniSync database"""
    return sqlite3.connect(OMNISYNC_DB)


def import_customers():
    """Import customers from OmniSync"""
    sqlite_conn = get_sqlite_connection()
    sqlite_cursor = sqlite_conn.cursor()
    
    imported = 0
    skipped = 0
    errors = []
    
    try:
        # Get active customers from OmniSync
        sqlite_cursor.execute("""
            SELECT 
                name, cvr_number, email, phone, address, city, postal_code,
                country, website, vtiger_id, economic_customer_number,
                email_domain, created_at
            FROM customers
            WHERE active = 1 AND deleted_at IS NULL
            ORDER BY name
        """)
        
        customers = sqlite_cursor.fetchall()
        logger.info(f"📥 Found {len(customers)} active customers in OmniSync")
        
        for row in customers:
            name, cvr, email, phone, address, city, postal_code, country, website, vtiger_id, economic_no, email_domain, created_at = row
            
            # Skip if no name
            if not name or name.strip() == '':
                skipped += 1
                continue
            
            # Insert each customer individually
            individual_conn = get_postgres_connection()
            individual_cursor = individual_conn.cursor()
            
            try:
                individual_cursor.execute("""
                    INSERT INTO customers (
                        name, cvr_number, email, phone, address, postal_code, city,
                        country, website, vtiger_id, economic_customer_number,
                        email_domain, is_active, created_at
                    ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                    ON CONFLICT (cvr_number) DO UPDATE SET
                        name = EXCLUDED.name,
                        email = EXCLUDED.email,
                        phone = EXCLUDED.phone,
                        address = EXCLUDED.address,
                        postal_code = EXCLUDED.postal_code,
                        city = EXCLUDED.city,
                        website = EXCLUDED.website,
                        vtiger_id = EXCLUDED.vtiger_id,
                        economic_customer_number = EXCLUDED.economic_customer_number
                    RETURNING id
                """, (
                    name, cvr, email, phone, address, postal_code, city,
                    country or 'Danmark', website, vtiger_id, economic_no,
                    email_domain, True, created_at
                ))
                
                result = individual_cursor.fetchone()
                individual_conn.commit()
                
                if result:
                    imported += 1
                    if imported % 50 == 0:
                        logger.info(f"   Imported {imported} customers...")
                        
            except Exception as e:
                error_msg = str(e)[:100]
                if imported + skipped < 10:  # Only log first 10 errors in detail
                    logger.warning(f"   ⚠️  Could not import '{name}': {error_msg}")
                errors.append((name, error_msg))
                skipped += 1
            finally:
                individual_cursor.close()
                individual_conn.close()
        
        logger.info(f"✅ Customers: {imported} imported, {skipped} skipped")
        if len(errors) > 10:
            logger.info(f"   (Suppressed {len(errors)-10} error messages)")
        
        return imported
        
    except Exception as e:
        logger.error(f"❌ Customer import failed: {e}")
        raise
    finally:
        sqlite_cursor.close()
        sqlite_conn.close()


def import_contacts():
    """Import contacts from OmniSync"""
    sqlite_conn = get_sqlite_connection()
    sqlite_cursor = sqlite_conn.cursor()
    
    imported = 0
    skipped = 0
    errors = []
    
    try:
        # Get contacts
        sqlite_cursor.execute("""
            SELECT 
                c.id, c.first_name, c.last_name, c.email, c.phone, c.mobile,
                c.title, c.department, c.active, c.vtiger_id, c.created_at
            FROM contacts c
            WHERE c.active = 1 AND c.deleted_at IS NULL
            ORDER BY c.last_name, c.first_name
        """)
        
        contacts = sqlite_cursor.fetchall()
        logger.info(f"📥 Found {len(contacts)} active contacts in OmniSync")
        
        for row in contacts:
            omnisync_id, first_name, last_name, email, phone, mobile, title, department, active, vtiger_id, created_at = row
            
            # Skip if no name
            if not first_name and not last_name:
                skipped += 1
                continue
            
            # Individual connection per contact
            postgres_conn = get_postgres_connection()
            postgres_cursor = postgres_conn.cursor()
            
            try:
                # Insert contact
                postgres_cursor.execute("""
                    INSERT INTO contacts (
                        first_name, last_name, email, phone, mobile,
                        title, department, is_active, vtiger_id, created_at
                    ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                    RETURNING id
                """, (
                    first_name or '', last_name or '', email, phone, mobile,
                    title, department, bool(active), vtiger_id, created_at
                ))
                
                result = postgres_cursor.fetchone()
                contact_id = result['id']
                
                # Get company relationships from OmniSync
                sqlite_cursor.execute("""
                    SELECT customer_id, is_primary, role, notes
                    FROM contact_companies
                    WHERE contact_id = ?
                """, (omnisync_id,))
                
                companies = sqlite_cursor.fetchall()
                
                # Link to companies in BMC Hub
                for company_row in companies:
                    customer_id_omnisync, is_primary, role, notes = company_row
                    
                    # Find matching customer in BMC Hub by vtiger_id or name
                    # First get the OmniSync customer
                    sqlite_cursor.execute("SELECT vtiger_id, name FROM customers WHERE id = ?", (customer_id_omnisync,))
                    customer_data = sqlite_cursor.fetchone()
                    
                    if customer_data:
                        vtiger_id_customer, customer_name = customer_data
                        
                        # Find in BMC Hub
                        if vtiger_id_customer:
                            postgres_cursor.execute("SELECT id FROM customers WHERE vtiger_id = %s", (vtiger_id_customer,))
                        else:
                            postgres_cursor.execute("SELECT id FROM customers WHERE name = %s LIMIT 1", (customer_name,))
                        
                        bmc_customer = postgres_cursor.fetchone()
                        
                        if bmc_customer:
                            postgres_cursor.execute("""
                                INSERT INTO contact_companies (contact_id, customer_id, is_primary, role, notes)
                                VALUES (%s, %s, %s, %s, %s)
                                ON CONFLICT DO NOTHING
                            """, (contact_id, bmc_customer['id'], bool(is_primary), role, notes))
                
                postgres_conn.commit()
                imported += 1
                if imported % 50 == 0:
                    logger.info(f"   Imported {imported} contacts...")
                    
            except Exception as e:
                error_msg = str(e)[:100]
                if imported + skipped < 10:
                    logger.warning(f"   ⚠️  Could not import '{first_name} {last_name}': {error_msg}")
                errors.append((f"{first_name} {last_name}", error_msg))
                skipped += 1
            finally:
                postgres_cursor.close()
                postgres_conn.close()
        
        logger.info(f"✅ Contacts: {imported} imported, {skipped} skipped")
        if len(errors) > 10:
            logger.info(f"   (Suppressed {len(errors)-10} error messages)")
        
        return imported
        
    except Exception as e:
        logger.error(f"❌ Contact import failed: {e}")
        raise
    finally:
        sqlite_cursor.close()
        sqlite_conn.close()


if __name__ == "__main__":
    logger.info("🚀 Starting OmniSync import...")
    logger.info(f"📂 Source: {OMNISYNC_DB}")
    
    customer_count = import_customers()
    contact_count = import_contacts()
    
    logger.info(f"\n🎉 Import completed!")
    logger.info(f"   Customers: {customer_count}")
    logger.info(f"   Contacts: {contact_count}")