From 8e99453deaf32ef6154be323c64419607a1cc0ff Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 3 Jul 2026 20:20:22 +0200 Subject: [PATCH] chore(release): bump version to 2.3.23 --- VERSION | 2 +- app/contacts/backend/router.py | 59 +- app/core/config.py | 4 + app/customers/backend/router.py | 221 ++- app/customers/frontend/customer_detail.html | 471 ++++- app/economy/frontend/also_cloud.html | 1215 ++++++++++++ app/economy/frontend/views.py | 8 + app/emails/backend/router.py | 2 +- app/emails/frontend/emails.html | 17 + app/jobs/archived_vtiger_sync.py | 29 + app/modules/also/backend/router.py | 80 +- app/modules/also/backend/service.py | 1728 ++++++++++++++++- app/modules/also/models/schemas.py | 15 +- .../bottom_bar/backend/public_router.py | 25 +- app/modules/bottom_bar/backend/router.py | 105 +- app/modules/bottom_bar/backend/service.py | 210 +- app/modules/drift/backend/router.py | 150 +- app/modules/drift/templates/drift.html | 27 +- app/modules/search/backend/router.py | 29 + app/services/customer_consistency.py | 316 ++- app/services/email_workflow_service.py | 50 +- app/services/simplycrm_service.py | 2 +- app/settings/backend/router.py | 6 +- app/settings/frontend/settings.html | 11 +- app/shared/frontend/base.html | 241 ++- app/ticket/backend/router.py | 367 +++- .../frontend/archived_ticket_detail.html | 834 ++++++-- app/ticket/frontend/archived_ticket_list.html | 1388 +++++++++++-- app/ticket/frontend/views.py | 1 + main.py | 16 + migrations/194_also_cloud_email_workflow.sql | 71 + static/js/bottom-bar.js | 786 +++++++- tests/test_drift_uisp_support.py | 47 + 33 files changed, 7843 insertions(+), 690 deletions(-) create mode 100644 app/economy/frontend/also_cloud.html create mode 100644 app/jobs/archived_vtiger_sync.py create mode 100644 migrations/194_also_cloud_email_workflow.sql diff --git a/VERSION b/VERSION index a4c8060..951d17f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.22 +2.3.23 diff --git a/app/contacts/backend/router.py b/app/contacts/backend/router.py index 14909a9..93527c8 100644 --- a/app/contacts/backend/router.py +++ b/app/contacts/backend/router.py @@ -5,7 +5,7 @@ Handles contact CRUD operations with multi-company support from fastapi import APIRouter, HTTPException, Query from typing import Optional, List -from app.core.database import execute_query, execute_insert, execute_update +from app.core.database import execute_query, execute_insert, execute_update, execute_query_single from app.core.contact_utils import get_contact_customer_ids, get_primary_customer_id from app.customers.backend.router import ( get_customer_subscriptions, @@ -119,36 +119,55 @@ async def get_contacts( where_sql = "WHERE " + " AND ".join(where_clauses) if where_clauses else "" - # Count total + # Count total matching contacts before pagination. count_query = f""" - SELECT COUNT(DISTINCT c.id) + SELECT COUNT(DISTINCT c.id) AS count FROM contacts c {where_sql} """ count_result = execute_query_single(count_query, tuple(params)) - total = count_result['count'] if count_result else 0 - - # Get contacts with company count - query = f""" - SELECT + total = int((count_result or {}).get('count') or 0) + + # Fetch the page of contacts first, then enrich with aggregated company data. + # This avoids grouped-pagination mismatches across PostgreSQL plans/versions. + page_query = f""" + SELECT c.id, c.first_name, c.last_name, c.email, c.phone, c.mobile, c.title, c.department, c.is_active, c.vtiger_id, - c.created_at, c.updated_at, - COUNT(DISTINCT cc.customer_id) as company_count, - ARRAY_AGG(DISTINCT cu.name ORDER BY cu.name) FILTER (WHERE cu.name IS NOT NULL) as company_names + c.created_at, c.updated_at FROM contacts c - LEFT JOIN contact_companies cc ON c.id = cc.contact_id - LEFT JOIN customers cu ON cc.customer_id = cu.id {where_sql} - GROUP BY c.id, c.first_name, c.last_name, c.email, c.phone, c.mobile, - c.title, c.department, c.is_active, c.vtiger_id, c.created_at, c.updated_at - ORDER BY c.last_name, c.first_name + ORDER BY c.last_name, c.first_name, c.id LIMIT %s OFFSET %s """ - params.extend([limit, offset]) - - contacts = execute_query(query, tuple(params)) # Returns all rows - + page_params = list(params) + page_params.extend([limit, offset]) + contacts = execute_query(page_query, tuple(page_params)) or [] + + if contacts: + contact_ids = [row["id"] for row in contacts if row.get("id") is not None] + placeholders = ",".join(["%s"] * len(contact_ids)) + company_rows = execute_query( + f""" + SELECT + cc.contact_id, + COUNT(DISTINCT cc.customer_id) AS company_count, + ARRAY_AGG(DISTINCT cu.name ORDER BY cu.name) + FILTER (WHERE cu.name IS NOT NULL) AS company_names + FROM contact_companies cc + LEFT JOIN customers cu ON cc.customer_id = cu.id + WHERE cc.contact_id IN ({placeholders}) + GROUP BY cc.contact_id + """, + tuple(contact_ids), + ) or [] + company_map = {row["contact_id"]: row for row in company_rows} + + for contact in contacts: + company_info = company_map.get(contact["id"]) or {} + contact["company_count"] = int(company_info.get("company_count") or 0) + contact["company_names"] = company_info.get("company_names") or [] + return { "contacts": contacts or [], "total": total, diff --git a/app/core/config.py b/app/core/config.py index cd5c27b..d718306 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -199,6 +199,10 @@ class Settings(BaseSettings): SIMPLYCRM_TICKET_EMAIL_MODULE: str = "Emails" SIMPLYCRM_TICKET_EMAIL_RELATION_FIELD: str = "parent_id" SIMPLYCRM_TICKET_EMAIL_FALLBACK_RELATION_FIELD: str = "related_to" + ARCHIVED_VTIGER_SYNC_ENABLED: bool = True + ARCHIVED_VTIGER_SYNC_INTERVAL_MINUTES: int = 30 + ARCHIVED_VTIGER_SYNC_LIMIT: int = 5000 + ARCHIVED_VTIGER_SYNC_INCLUDE_MESSAGES: bool = False # Backup System Configuration BACKUP_ENABLED: bool = True diff --git a/app/customers/backend/router.py b/app/customers/backend/router.py index 67f2018..7435435 100644 --- a/app/customers/backend/router.py +++ b/app/customers/backend/router.py @@ -59,6 +59,42 @@ def _ensure_customer_supplier_tag(customer_id: int) -> None: logger.warning("⚠️ Could not ensure supplier tag for customer %s: %s", customer_id, tag_error) +def _ensure_customer_tag(customer_id: int, tag_name: str, description: str = "") -> None: + try: + tag = execute_query_single( + "SELECT id FROM tags WHERE LOWER(name) = LOWER(%s) AND type = 'category' LIMIT 1", + (tag_name,), + ) + if tag and tag.get("id") is not None: + tag_id = int(tag["id"]) + else: + created = execute_query_single( + """ + INSERT INTO tags (name, type, description, color, is_active) + VALUES (%s, %s, %s, %s, %s) + ON CONFLICT (name, type) + DO UPDATE SET is_active = TRUE, updated_at = CURRENT_TIMESTAMP + RETURNING id + """, + (tag_name, "category", description or tag_name, "#198754", True), + ) + tag_id = int(created["id"]) if created and created.get("id") is not None else None + + if not tag_id: + return + + execute_query( + """ + INSERT INTO entity_tags (entity_type, entity_id, tag_id) + VALUES (%s, %s, %s) + ON CONFLICT (entity_type, entity_id, tag_id) DO NOTHING + """, + ("customer", customer_id, tag_id), + ) + except Exception as tag_error: + logger.warning("⚠️ Could not ensure tag %s for customer %s: %s", tag_name, customer_id, tag_error) + + # Pydantic Models class CustomerBase(BaseModel): name: str @@ -111,6 +147,11 @@ class ContactCreate(BaseModel): role: Optional[str] = None +class ContactSyncRequest(BaseModel): + selected_match_keys: List[str] + add_sync_ok_tag: Optional[bool] = False + + @router.get("/customers") async def list_customers( limit: int = Query(default=50, ge=1, le=1000), @@ -747,12 +788,21 @@ async def check_customer_data_consistency(customer_id: int): 1 for field_data in discrepancies.values() if field_data['discrepancy'] ) + contact_discrepancies = consistency_service.compare_contacts(all_data) + actionable_contact_discrepancies = [ + item for item in contact_discrepancies + if item.get("action") != "matched" + ] return { "enabled": True, "customer_id": customer_id, - "discrepancy_count": discrepancy_count, + "discrepancy_count": discrepancy_count + len(actionable_contact_discrepancies), + "field_discrepancy_count": discrepancy_count, + "contact_discrepancy_count": len(actionable_contact_discrepancies), + "contact_total_count": len(contact_discrepancies), "discrepancies": discrepancies, + "contact_discrepancies": contact_discrepancies, "systems_available": { "hub": True, "vtiger": all_data.get('vtiger') is not None, @@ -813,6 +863,70 @@ async def sync_customer_field( raise HTTPException(status_code=500, detail=str(e)) +@router.post("/customers/{customer_id}/sync-contacts") +async def sync_customer_contacts(customer_id: int, request: ContactSyncRequest): + """Sync selected vTiger contacts into Hub and optionally tag the customer.""" + try: + consistency_service = CustomerConsistencyService() + stats = await consistency_service.sync_vtiger_contacts_to_hub( + customer_id=customer_id, + selected_match_keys=request.selected_match_keys or [], + ) + + if request.add_sync_ok_tag and stats.get("selected", 0) > 0 and stats.get("skipped", 0) == 0: + _ensure_customer_tag(customer_id, "Sync OK", "Kundedata og kontakter er manuelt verificeret") + + return { + "success": True, + "customer_id": customer_id, + "stats": stats, + "sync_ok_tag_added": bool(request.add_sync_ok_tag and stats.get("selected", 0) > 0 and stats.get("skipped", 0) == 0), + } + except Exception as e: + logger.error(f"❌ Failed to sync contacts for customer {customer_id}: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/customers/{customer_id}/mark-sync-ok") +async def mark_customer_sync_ok(customer_id: int): + """Ensure the customer has the Sync OK tag.""" + try: + _ensure_customer_tag(customer_id, "Sync OK", "Kundedata og kontakter er manuelt verificeret") + return {"success": True, "customer_id": customer_id, "tag": "Sync OK"} + except Exception as e: + logger.error(f"❌ Failed to add Sync OK tag for customer {customer_id}: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.get("/customers/{customer_id}/data-consistency-debug") +async def debug_customer_data_consistency(customer_id: int): + """Debug payload for customer consistency, including contact sync source data.""" + try: + consistency_service = CustomerConsistencyService() + all_data = await consistency_service.fetch_all_data(customer_id) + contact_discrepancies = consistency_service.compare_contacts(all_data) + + return { + "customer_id": customer_id, + "hub_customer": { + "id": all_data.get("hub", {}).get("id"), + "name": all_data.get("hub", {}).get("name"), + "vtiger_id": all_data.get("hub", {}).get("vtiger_id"), + }, + "counts": { + "hub_contacts": len(all_data.get("hub_contacts") or []), + "vtiger_contacts": len(all_data.get("vtiger_contacts") or []), + "contact_rows": len(contact_discrepancies), + }, + "hub_contacts": all_data.get("hub_contacts") or [], + "vtiger_contacts": all_data.get("vtiger_contacts") or [], + "contact_discrepancies": contact_discrepancies, + } + except Exception as e: + logger.error(f"❌ Failed to debug consistency for customer {customer_id}: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.post("/customers/sync-economic-from-simplycrm") async def sync_economic_numbers_from_simplycrm(): """ @@ -1850,26 +1964,108 @@ async def get_customer_acmp_overview( products = execute_query( """ + WITH base AS ( + SELECT + COALESCE(material_number, 'N/A') AS material_number, + COALESCE(vendor, 'N/A') AS vendor, + COALESCE(product_name, 'Ukendt produkt') AS product_name, + COALESCE(billable_parameters, 1) AS qty, + COALESCE(total_price, sales_price, 0) AS revenue, + COALESCE(cost_amount, 0) AS cost, + billing_start, + created_at + FROM also_import_lines + WHERE matched_customer_id = %s + ) SELECT - COALESCE(material_number, 'N/A') AS material_number, - COALESCE(vendor, 'N/A') AS vendor, - COALESCE(product_name, 'Ukendt produkt') AS product_name, + material_number, + vendor, + product_name, COUNT(*)::INTEGER AS line_count, - COALESCE(SUM(COALESCE(billable_parameters, 1)), 0) AS quantity_total, - COALESCE(SUM(COALESCE(total_price, sales_price, 0)), 0) AS revenue_total, - COALESCE(SUM(COALESCE(cost_amount, 0)), 0) AS cost_total, - COALESCE(SUM(COALESCE(total_price, sales_price, 0) - COALESCE(cost_amount, 0)), 0) AS margin_total, + COALESCE(SUM(qty), 0) AS quantity_total, + COALESCE(SUM(revenue), 0) AS revenue_total, + COALESCE(SUM(cost), 0) AS cost_total, + COALESCE(SUM(revenue - cost), 0) AS margin_total, + COALESCE(SUM(qty) FILTER ( + WHERE date_trunc('month', COALESCE(billing_start::timestamp, created_at)) = date_trunc('month', CURRENT_DATE) + ), 0) AS current_month_qty, + COALESCE(SUM(qty) FILTER ( + WHERE date_trunc('month', COALESCE(billing_start::timestamp, created_at)) = date_trunc('month', CURRENT_DATE - INTERVAL '1 month') + ), 0) AS previous_month_qty, MAX(billing_start) AS last_billing_start, MAX(created_at) AS last_seen_at - FROM also_import_lines - WHERE matched_customer_id = %s - GROUP BY COALESCE(material_number, 'N/A'), COALESCE(vendor, 'N/A'), COALESCE(product_name, 'Ukendt produkt') + FROM base + GROUP BY material_number, vendor, product_name ORDER BY revenue_total DESC, quantity_total DESC LIMIT 200 """, (customer_id,), ) or [] + changes = execute_query( + """ + WITH current_month AS ( + SELECT + COALESCE(material_number, 'N/A') AS material_number, + COALESCE(vendor, 'N/A') AS vendor, + COALESCE(product_name, 'Ukendt produkt') AS product_name, + SUM(COALESCE(billable_parameters, 1)) AS qty + FROM also_import_lines + WHERE matched_customer_id = %s + AND date_trunc('month', COALESCE(billing_start::timestamp, created_at)) = date_trunc('month', CURRENT_DATE) + GROUP BY 1, 2, 3 + ), + previous_month AS ( + SELECT + COALESCE(material_number, 'N/A') AS material_number, + COALESCE(vendor, 'N/A') AS vendor, + COALESCE(product_name, 'Ukendt produkt') AS product_name, + SUM(COALESCE(billable_parameters, 1)) AS qty + FROM also_import_lines + WHERE matched_customer_id = %s + AND date_trunc('month', COALESCE(billing_start::timestamp, created_at)) = date_trunc('month', CURRENT_DATE - INTERVAL '1 month') + GROUP BY 1, 2, 3 + ) + SELECT + COALESCE(c.material_number, p.material_number) AS material_number, + COALESCE(c.vendor, p.vendor) AS vendor, + COALESCE(c.product_name, p.product_name) AS product_name, + COALESCE(p.qty, 0) AS previous_month_qty, + COALESCE(c.qty, 0) AS current_month_qty, + COALESCE(c.qty, 0) - COALESCE(p.qty, 0) AS quantity_delta + FROM current_month c + FULL OUTER JOIN previous_month p + ON c.material_number = p.material_number + AND c.vendor = p.vendor + AND c.product_name = p.product_name + WHERE COALESCE(c.qty, 0) <> COALESCE(p.qty, 0) + ORDER BY ABS(COALESCE(c.qty, 0) - COALESCE(p.qty, 0)) DESC, COALESCE(c.qty, 0) DESC + LIMIT 100 + """, + (customer_id, customer_id), + ) or [] + + recent_lines = execute_query( + """ + SELECT + COALESCE(product_name, 'Ukendt produkt') AS product_name, + COALESCE(material_number, 'N/A') AS material_number, + COALESCE(vendor, 'N/A') AS vendor, + COALESCE(billable_parameters, 1) AS quantity, + COALESCE(total_price, sales_price, 0) AS revenue, + COALESCE(total_price, sales_price, 0) - COALESCE(cost_amount, 0) AS margin, + queue_status, + order_draft_id, + billing_start, + created_at + FROM also_import_lines + WHERE matched_customer_id = %s + ORDER BY COALESCE(billing_start::timestamp, created_at) DESC, id DESC + LIMIT 50 + """, + (customer_id,), + ) or [] + monthly = execute_query( """ SELECT @@ -1907,6 +2103,8 @@ async def get_customer_acmp_overview( "status_breakdown": status_breakdown, "products": products, "monthly": monthly, + "changes": changes, + "recent_lines": recent_lines, } return response @@ -1915,4 +2113,3 @@ async def get_customer_acmp_overview( except Exception as e: logger.error("❌ Error fetching ACMP overview for customer %s: %s", customer_id, e) raise HTTPException(status_code=500, detail=str(e)) - diff --git a/app/customers/frontend/customer_detail.html b/app/customers/frontend/customer_detail.html index b8c5182..a740816 100644 --- a/app/customers/frontend/customer_detail.html +++ b/app/customers/frontend/customer_detail.html @@ -345,6 +345,80 @@ font-weight: 700; } + .consistency-contact-comparison { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 0.75rem; + margin-top: 0.75rem; + } + + .consistency-contact-box { + border: 1px solid rgba(15, 76, 117, 0.14); + border-radius: 10px; + padding: 0.75rem; + background: rgba(15, 76, 117, 0.03); + } + + .consistency-contact-box--new { + border-color: rgba(25, 135, 84, 0.24); + background: rgba(25, 135, 84, 0.05); + } + + .consistency-contact-box-title { + font-size: 0.74rem; + font-weight: 700; + letter-spacing: 0.05em; + text-transform: uppercase; + color: var(--text-secondary); + margin-bottom: 0.45rem; + } + + .consistency-contact-line { + font-size: 0.9rem; + margin-bottom: 0.3rem; + } + + .consistency-contact-line:last-child { + margin-bottom: 0; + } + + .consistency-contact-line-label { + color: var(--text-secondary); + font-weight: 600; + margin-right: 0.35rem; + } + + .consistency-diff-list { + margin-top: 0.6rem; + padding-top: 0.6rem; + border-top: 1px dashed rgba(15, 76, 117, 0.14); + } + + .consistency-diff-row { + display: grid; + grid-template-columns: 140px 1fr 1fr; + gap: 0.6rem; + font-size: 0.86rem; + margin-bottom: 0.35rem; + } + + .consistency-diff-row:last-child { + margin-bottom: 0; + } + + .consistency-diff-field { + font-weight: 600; + color: var(--text-secondary); + } + + .consistency-diff-old { + color: #8b1e3f; + } + + .consistency-diff-new { + color: #146c43; + } + @media (max-width: 992px) { .contacts-toolbar { padding: 0.7rem; @@ -359,6 +433,11 @@ #contactsContainer { min-width: 920px; } + + .consistency-contact-comparison, + .consistency-diff-row { + grid-template-columns: 1fr; + } } {% endblock %} @@ -1029,6 +1108,9 @@ Produkt Materiale Vendor + Nu + Sidste md. + Ændring Antal Omsætning DB @@ -1038,6 +1120,52 @@ + +
+
+
+
+
Ændringer siden sidste måned
+ Pr. produkt +
+
+ + + + + + + + + + +
ProduktSidste md.NuDelta
+
+
+
+
+
+
+
Seneste ACMP-linjer
+ Seneste 50 linjer +
+
+ + + + + + + + + + + +
DatoProduktAntalOms.Status
+
+
+
+
@@ -2934,6 +3062,20 @@ function getContactMobileValue(contact) { return normalizePhoneValue(contact.mobile || contact.mobile_phone); } +function renderContactActionButtons(number, displayName, contactId) { + const safeNumber = escapeHtml(number); + const safeDisplayName = escapeHtml(displayName || ''); + const safeContactId = contactId || 'null'; + return ` +
+ + +
+ `; +} + function buildContactsRows(contacts) { return contacts.map(contact => { const displayName = getContactDisplayName(contact); @@ -2947,10 +3089,10 @@ function buildContactsRows(contacts) { const email = contact.email ? `${escapeHtml(contact.email)}` : '—'; const phone = phoneValue - ? `
${escapeHtml(phoneValue)}
` + ? `
${escapeHtml(phoneValue)}${renderContactActionButtons(phoneValue, displayName, contactId)}
` : '—'; const mobile = mobileValue - ? `
${escapeHtml(mobileValue)}
` + ? `
${escapeHtml(mobileValue)}${renderContactActionButtons(mobileValue, displayName, contactId)}
` : '—'; const title = titleValue ? escapeHtml(titleValue) : '—'; const primaryBadge = contact.is_primary ? 'Primær' : '—'; @@ -4570,6 +4712,96 @@ async function saveCustomerEdit() { // Data Consistency Functions let consistencyData = null; +function escapeAttribute(value) { + return String(value ?? '') + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(//g, '>'); +} + +function renderConsistencyValue(value) { + if (value === null || value === undefined || String(value).trim() === '') { + return 'Tom'; + } + return escapeHtml(String(value)); +} + +function getContactFieldLabel(field) { + const labels = { + first_name: 'Fornavn', + last_name: 'Efternavn', + email: 'Email', + phone: 'Telefon', + mobile: 'Mobil', + title: 'Titel', + department: 'Afdeling' + }; + return labels[field] || field; +} + +function buildContactComparisonLines(contact) { + return [ + ['Navn', contact?.display_name], + ['Email', contact?.email], + ['Telefon', contact?.phone], + ['Mobil', contact?.mobile], + ['Titel', contact?.title], + ['Afdeling', contact?.department] + ].map(([label, value]) => ` +
+ ${escapeHtml(label)}: + ${renderConsistencyValue(value)} +
+ `).join(''); +} + +function buildChangedFieldsDetails(hub, vtiger, changedFields) { + if (!Array.isArray(changedFields) || !changedFields.length) { + return ''; + } + + return ` +
+ ${changedFields.map((field) => ` +
+
${escapeHtml(getContactFieldLabel(field))}
+
Hub nu: ${renderConsistencyValue(hub?.[field])}
+
Ny fra vTiger: ${renderConsistencyValue(vtiger?.[field])}
+
+ `).join('')} +
+ `; +} + +function buildFieldDifferenceSummary(fieldData) { + const comparisons = []; + const hubValue = String(fieldData?.hub ?? '').trim(); + const vtigerValue = String(fieldData?.vtiger ?? '').trim(); + const economicValue = String(fieldData?.economic ?? '').trim(); + + if (fieldData && fieldData.vtiger !== undefined && hubValue !== vtigerValue) { + comparisons.push('Hub/vTiger'); + } + if (fieldData && fieldData.economic !== undefined && hubValue !== economicValue) { + comparisons.push('Hub/e-conomic'); + } + if ( + fieldData && + fieldData.vtiger !== undefined && + fieldData.economic !== undefined && + vtigerValue !== economicValue + ) { + comparisons.push('vTiger/e-conomic'); + } + + if (!comparisons.length) { + return 'Ingen'; + } + + return comparisons.map(item => `${escapeHtml(item)}`).join(''); +} + async function checkDataConsistency() { try { const response = await fetch(`/api/v1/customers/${customerId}/data-consistency`); @@ -4602,7 +4834,16 @@ function showConsistencyModal() { } const tbody = document.getElementById('consistencyTableBody'); + const contactsContainer = document.getElementById('consistencyContactsContainer'); + const contactsSection = document.getElementById('consistencyContactsSection'); + const syncOkCheckbox = document.getElementById('consistencyAddSyncOkTag'); tbody.innerHTML = ''; + if (contactsContainer) { + contactsContainer.innerHTML = ''; + } + if (syncOkCheckbox) { + syncOkCheckbox.checked = false; + } // Field labels in Danish const fieldLabels = { @@ -4628,7 +4869,10 @@ function showConsistencyModal() { // Field name const fieldCell = document.createElement('td'); - fieldCell.innerHTML = `${fieldLabels[fieldName] || fieldName}`; + fieldCell.innerHTML = ` + ${fieldLabels[fieldName] || fieldName} +
Forskellig værdi fundet på dette felt
+ `; row.appendChild(fieldCell); // Hub value @@ -4636,9 +4880,9 @@ function showConsistencyModal() { hubCell.innerHTML = `
+ id="hub_${fieldName}" value="hub" data-value="${escapeAttribute(fieldData.hub || '')}">
`; @@ -4650,9 +4894,9 @@ function showConsistencyModal() { vtigerCell.innerHTML = `
+ id="vtiger_${fieldName}" value="vtiger" data-value="${escapeAttribute(fieldData.vtiger || '')}">
`; @@ -4667,9 +4911,9 @@ function showConsistencyModal() { economicCell.innerHTML = `
+ id="economic_${fieldName}" value="economic" data-value="${escapeAttribute(fieldData.economic || '')}">
`; @@ -4680,11 +4924,69 @@ function showConsistencyModal() { // Action cell (which system to use) const actionCell = document.createElement('td'); - actionCell.innerHTML = '← Vælg'; + actionCell.innerHTML = ` +
Forskelle: ${buildFieldDifferenceSummary(fieldData)}
+ Vælg hvilken værdi der skal bruges som ny korrekt værdi + `; row.appendChild(actionCell); tbody.appendChild(row); } + + const contactDiscrepancies = Array.isArray(consistencyData.contact_discrepancies) + ? consistencyData.contact_discrepancies + : []; + + if (contactsSection) { + contactsSection.classList.toggle('d-none', contactDiscrepancies.length === 0); + } + + if (contactsContainer && contactDiscrepancies.length > 0) { + contactsContainer.innerHTML = contactDiscrepancies.map((item, index) => { + const vtiger = item.vtiger || {}; + const hub = item.hub || null; + const selectable = item.selectable !== false; + const checkedAttr = selectable ? '' : 'disabled'; + const statusClass = item.action === 'matched' ? 'bg-success-subtle text-success border-success-subtle' : 'bg-light text-dark border'; + const changedFields = buildChangedFieldsDetails(hub, vtiger, item.changed_fields || []); + const hubSummary = hub + ? ` +
+
Nuværende i Hub
+ ${buildContactComparisonLines(hub)} +
+ ` + : ` +
+
Nuværende i Hub
+
Ingen kontakt tilknyttet denne kunde endnu
+
+ `; + const vtigerSummary = ` +
+
Ny værdi fra vTiger
+ ${buildContactComparisonLines(vtiger)} +
+ `; + + return ` + + `; + }).join(''); + } const modal = new bootstrap.Modal(document.getElementById('consistencyModal')); modal.show(); @@ -4692,12 +4994,15 @@ function showConsistencyModal() { async function syncSelectedFields() { const selections = []; + const selectedContactKeys = []; // Gather all selected values const radioButtons = document.querySelectorAll('#consistencyTableBody input[type="radio"]:checked'); + const contactCheckboxes = document.querySelectorAll('.consistency-contact-check:checked'); + const addSyncOkTag = Boolean(document.getElementById('consistencyAddSyncOkTag')?.checked); - if (radioButtons.length === 0) { - alert('Vælg venligst mindst ét felt at synkronisere'); + if (radioButtons.length === 0 && contactCheckboxes.length === 0) { + alert('Vælg venligst mindst ét felt eller én kontakt at synkronisere'); return; } @@ -4712,15 +5017,21 @@ async function syncSelectedFields() { source_value: sourceValue }); }); + + contactCheckboxes.forEach((checkbox) => { + if (checkbox.value) { + selectedContactKeys.push(checkbox.value); + } + }); // Confirm action - if (!confirm(`Du er ved at synkronisere ${selections.length} felt(er) på tværs af alle systemer. Fortsæt?`)) { + if (!confirm(`Du er ved at synkronisere ${selections.length} felt(er) og ${selectedContactKeys.length} kontakt(er). Fortsæt?`)) { return; } - // Sync each field - let successCount = 0; + let fieldSuccessCount = 0; let failCount = 0; + let contactSummary = null; for (const selection of selections) { try { @@ -4731,7 +5042,7 @@ async function syncSelectedFields() { ); if (response.ok) { - successCount++; + fieldSuccessCount++; } else { failCount++; console.error(`Failed to sync ${selection.field_name}`); @@ -4741,6 +5052,39 @@ async function syncSelectedFields() { console.error(`Error syncing ${selection.field_name}:`, error); } } + + if (selectedContactKeys.length > 0) { + try { + const response = await fetch(`/api/v1/customers/${customerId}/sync-contacts`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + selected_match_keys: selectedContactKeys, + add_sync_ok_tag: false + }) + }); + + if (response.ok) { + const payload = await response.json(); + contactSummary = payload.stats || null; + } else { + failCount++; + console.error('Failed to sync selected contacts'); + } + } catch (error) { + failCount++; + console.error('Error syncing selected contacts:', error); + } + } + + if (addSyncOkTag && failCount === 0 && (fieldSuccessCount > 0 || (contactSummary && contactSummary.selected > 0))) { + try { + await addSyncOkTagToCustomer(); + } catch (error) { + failCount++; + console.error('Failed to add Sync OK tag:', error); + } + } // Close modal const modal = bootstrap.Modal.getInstance(document.getElementById('consistencyModal')); @@ -4748,9 +5092,19 @@ async function syncSelectedFields() { // Show result if (failCount === 0) { - alert(`✓ ${successCount} felt(er) synkroniseret succesfuldt!`); + const parts = []; + if (fieldSuccessCount > 0) { + parts.push(`${fieldSuccessCount} felt`); + } + if (contactSummary && contactSummary.selected > 0) { + parts.push(`${contactSummary.selected} kontakt(er)`); + } + if (addSyncOkTag) { + parts.push('tag "Sync OK"'); + } + alert(`✓ Synkroniseret: ${parts.join(', ')}`); } else { - alert(`⚠️ ${successCount} felt(er) synkroniseret, ${failCount} fejlede`); + alert(`⚠️ ${fieldSuccessCount} felt synkroniseret, ${selectedContactKeys.length} kontakt(er) forsøgt, ${failCount} fejl`); } // Reload customer data and recheck consistency @@ -4758,6 +5112,17 @@ async function syncSelectedFields() { await checkDataConsistency(); } +async function addSyncOkTagToCustomer() { + const response = await fetch(`/api/v1/customers/${customerId}/mark-sync-ok`, { + method: 'POST' + }); + + if (!response.ok) { + const error = await response.json().catch(() => ({})); + throw new Error(error.detail || 'Kunne ikke tilføje Sync OK tag'); + } +} + function showAddContactModal() { // TODO: Open add contact modal console.log('Add contact for customer:', customerId); @@ -5220,12 +5585,61 @@ function renderAcmpOverview(payload) { ${escapeHtml(row.product_name || '-')} ${escapeHtml(row.material_number || '-')} ${escapeHtml(row.vendor || '-')} + ${Number(row.current_month_qty || 0).toLocaleString('da-DK')} + ${Number(row.previous_month_qty || 0).toLocaleString('da-DK')} + ${formatSignedNumber(Number(row.current_month_qty || 0) - Number(row.previous_month_qty || 0))} ${Number(row.quantity_total || 0).toLocaleString('da-DK')} ${formatDKK(Number(row.revenue_total || 0))} ${formatDKK(Number(row.margin_total || 0))} `).join('') - : 'Ingen produkter fundet'; + : 'Ingen produkter fundet'; + + const changesRows = document.getElementById('acmpChangesRows'); + const changes = payload.changes || []; + changesRows.innerHTML = changes.length > 0 + ? changes.map(row => ` + + +
${escapeHtml(row.product_name || '-')}
+
${escapeHtml(row.material_number || '-')}
+ + ${Number(row.previous_month_qty || 0).toLocaleString('da-DK')} + ${Number(row.current_month_qty || 0).toLocaleString('da-DK')} + ${formatSignedNumber(Number(row.quantity_delta || 0))} + + `).join('') + : 'Ingen ændringer fundet'; + + const recentRows = document.getElementById('acmpRecentRows'); + const recent = payload.recent_lines || []; + recentRows.innerHTML = recent.length > 0 + ? recent.map(row => ` + + ${formatShortDate(row.billing_start || row.created_at)} + +
${escapeHtml(row.product_name || '-')}
+
${escapeHtml(row.material_number || '-')}
+ + ${Number(row.quantity || 0).toLocaleString('da-DK')} + ${formatDKK(Number(row.revenue || 0))} + ${escapeHtml(row.queue_status || '-')} + + `).join('') + : 'Ingen linjer fundet'; +} + +function formatSignedNumber(value) { + const number = Number(value || 0); + const prefix = number > 0 ? '+' : ''; + return `${prefix}${number.toLocaleString('da-DK')}`; +} + +function formatShortDate(value) { + if (!value) return '-'; + const date = new Date(value); + if (Number.isNaN(date.getTime())) return escapeHtml(String(value)); + return date.toLocaleDateString('da-DK'); } function editInternalComment() { @@ -5470,7 +5884,7 @@ document.addEventListener('DOMContentLoaded', () => {
Vejledning: Vælg den korrekte værdi for hvert felt med uoverensstemmelser. - Når du klikker "Synkroniser Valgte", vil de valgte værdier blive opdateret i alle systemer. + vTiger bruges kun som læsekilde. Synkronisering skriver kun til BMC Hub og evt. e-conomic.
@@ -5489,6 +5903,23 @@ document.addEventListener('DOMContentLoaded', () => {
+ +
+
+ Kontaktpersoner fra vTiger +
+
+ Alle fundne vTiger-kontakter vises her. Marker kun de kontakter der skal oprettes, opdateres eller linkes til denne kunde i Hub. +
+
+
+ +
+ + +
+
+ + +
+
+
+
Omsætning
+
0 kr.
+
+
+
+
+
Kost
+
0 kr.
+
+
+
+
+
Margin
+
0 kr.
+
+
+
+
+
Afventer godkendelse
+
0
+
+
+
+
+
Manglende kunder
+
0
+
+
+
+
+
Manglende produkter
+
0
+
+
+
+ +
+
+
+
+
Upload Billing-Grundlag
+
+
+
+
+ + +
+
+ + +
+
Workflow: upload af ALSO Excel, matching, validering og automatisk oprettelse af ordrekladder for klare linjer. ZIP kan stadig bruges som fallback.
+
+
Ingen upload kørt endnu.
+
+
+
+ +
+
+
+
Statusfordeling
+
+
+
+
Linjer
+
+
+
+
Importjobs
+
+
+
+
+
+
+ +
+
+
+
+
Månedshistorik
+ +
+
+
+ + + + + + + + + + + + + + + +
MånedJobsLinjerKunderOmsætningKostMargin
Indlæser...
+
+
+
+
+ +
+
+
+
Månedlige Afvigelser
+
+
+
+ + + + + + + + + + + + + +
KundeProduktSidsteNuÆndring
Indlæser...
+
+
+
+
+
+ +
+
+
+
+
Importhistorik
+ +
+
+
+ + + + + + + + + + + + + +
JobStatusFilImporteretLinjer
Indlæser...
+
+
+
+ +
+
+
Queue Snapshot
+
+ + +
+
+
+
+ + + + + + + + + + + + + + + + +
IDStatusKundeProduktBeløbKostMarginDraft
Indlæser...
+
+
+
+
+ +
+
+
+
Job Detaljer
+
+ Vælg et job + + +
+
+
+
Ingen job valgt endnu.
+
+ + + + + + + + + + + + +
LinjeStatusKunde / ProduktBeløb
Vælg et job for at se linjer.
+
+
+
+
+
+{% endblock %} + +{% block extra_js %} + +{% endblock %} diff --git a/app/economy/frontend/views.py b/app/economy/frontend/views.py index b3d8581..1bb0fb6 100644 --- a/app/economy/frontend/views.py +++ b/app/economy/frontend/views.py @@ -12,3 +12,11 @@ async def economy_time_queue_page(request: Request): "economy/frontend/time_queue.html", {"request": request, "title": "Economy Time Queue"}, ) + + +@router.get("/economy/also-cloud", response_class=HTMLResponse) +async def economy_also_cloud_page(request: Request): + return templates.TemplateResponse( + "economy/frontend/also_cloud.html", + {"request": request, "title": "ALSO Cloud Marketplace"}, + ) diff --git a/app/emails/backend/router.py b/app/emails/backend/router.py index d6aec6a..e912e8d 100644 --- a/app/emails/backend/router.py +++ b/app/emails/backend/router.py @@ -550,7 +550,7 @@ def _compute_workflow_preview(email_data: Dict[str, Any]) -> Dict[str, Any]: reasons = [] matches = True - if trigger != classification: + if trigger not in {classification, 'any'}: matches = False reasons.append(f"classification_mismatch ({trigger} != {classification or 'none'})") diff --git a/app/emails/frontend/emails.html b/app/emails/frontend/emails.html index 2472835..3fd7340 100644 --- a/app/emails/frontend/emails.html +++ b/app/emails/frontend/emails.html @@ -4870,6 +4870,23 @@ const WORKFLOW_TEMPLATES = { workflow_steps: [ { action: 'flag_for_review', params: { reason: 'low_confidence' } } ] + }, + 'also_cloud_billing': { + name: 'ALSO Cloud Billing Import', + description: 'Downloader ALSO Cloud Marketplace ZIP, importerer ACMP-linjer og opretter ordrekladder', + classification_trigger: 'any', + confidence_threshold: 0.00, + priority: 15, + workflow_steps: [ + { + action: 'process_also_cloud_billing', + params: { + sender_pattern: 'marketplace\\.also\\.', + require_sender_match: true, + source_label: 'ALSO Cloud Marketplace email' + } + } + ] } }; diff --git a/app/jobs/archived_vtiger_sync.py b/app/jobs/archived_vtiger_sync.py new file mode 100644 index 0000000..e101c7f --- /dev/null +++ b/app/jobs/archived_vtiger_sync.py @@ -0,0 +1,29 @@ +""" +Scheduled archived vTiger sync job. +""" + +import logging + +from app.core.config import settings +from app.ticket.backend.router import _run_vtiger_archived_import + +logger = logging.getLogger(__name__) + + +async def run_archived_vtiger_sync() -> None: + """Run incremental archived vTiger sync in the background scheduler.""" + if not settings.ARCHIVED_VTIGER_SYNC_ENABLED: + logger.info("⏭️ Archived vTiger sync skipped (ARCHIVED_VTIGER_SYNC_ENABLED=false)") + return + + logger.info("🔄 Archived vTiger sync job started") + try: + result = await _run_vtiger_archived_import( + limit=settings.ARCHIVED_VTIGER_SYNC_LIMIT, + include_messages=settings.ARCHIVED_VTIGER_SYNC_INCLUDE_MESSAGES, + force=False, + incremental=True, + ) + logger.info("✅ Archived vTiger sync job completed: %s", result) + except Exception as exc: + logger.error("❌ Archived vTiger sync job failed: %s", exc, exc_info=True) diff --git a/app/modules/also/backend/router.py b/app/modules/also/backend/router.py index 4b844c1..c52caf6 100644 --- a/app/modules/also/backend/router.py +++ b/app/modules/also/backend/router.py @@ -1,6 +1,6 @@ from typing import Optional -from fastapi import APIRouter, Query, Request +from fastapi import APIRouter, File, Query, Request, UploadFile from app.modules.also.backend.service import also_service from app.modules.also.models.schemas import ( @@ -11,6 +11,8 @@ from app.modules.also.models.schemas import ( AlsoImportJobCreate, AlsoImportJobResponse, AlsoImportLinesRequest, + AlsoManualCompanyMapRequest, + AlsoManualProductMapRequest, AlsoProductMappingUpsert, AlsoQueueApproveRequest, AlsoQueueLineResponse, @@ -49,16 +51,80 @@ async def list_import_jobs( return also_service.list_import_jobs(status=status, limit=limit) -@router.get("/also/import-jobs/{job_id}", response_model=AlsoImportJobResponse) +@router.get("/also/import-jobs/{job_id}") async def get_import_job(job_id: int): return also_service.get_import_job(job_id) +@router.delete("/also/import-jobs/{job_id}") +async def delete_import_job( + job_id: int, + delete_order_drafts: bool = Query(default=False), +): + return also_service.delete_import_job(job_id=job_id, delete_order_drafts=delete_order_drafts) + + +@router.post("/also/import-jobs/{job_id}/auto-map") +async def auto_map_import_job(job_id: int): + return also_service.auto_map_import_job(job_id=job_id) + + +@router.post("/also/import-lines/{line_id}/map-company") +async def manual_map_company_for_line( + line_id: int, + payload: AlsoManualCompanyMapRequest, + request: Request, +): + return also_service.manual_map_company_for_line( + line_id=line_id, + customer_id=payload.customer_id, + notes=payload.notes, + updated_by_user_id=_user_id_from_request(request), + ) + + +@router.post("/also/import-lines/{line_id}/map-product") +async def manual_map_product_for_line( + line_id: int, + payload: AlsoManualProductMapRequest, + request: Request, +): + return also_service.manual_map_product_for_line( + line_id=line_id, + product_id=payload.product_id, + notes=payload.notes, + updated_by_user_id=_user_id_from_request(request), + ) + + +@router.post("/also/import-upload") +async def upload_import_file( + request: Request, + file: UploadFile = File(...), +): + payload = await file.read() + return also_service.import_billing_upload( + file_name=file.filename or "also-billing.xlsx", + file_bytes=payload, + imported_by_user_id=_user_id_from_request(request), + source_label="Manual upload", + ) + + @router.post("/also/import-jobs/{job_id}/lines") async def import_lines(job_id: int, payload: AlsoImportLinesRequest): return also_service.import_lines(job_id=job_id, payload=payload) +@router.get("/also/import-jobs/{job_id}/lines") +async def get_import_job_lines( + job_id: int, + status: Optional[str] = Query(default=None), + limit: int = Query(default=500, ge=1, le=2000), +): + return also_service.get_import_job_lines(job_id=job_id, status=status, limit=limit) + + @router.get("/also/queue", response_model=list[AlsoQueueLineResponse]) async def get_queue( status: Optional[str] = Query(default=None), @@ -77,6 +143,16 @@ async def get_dashboard_differences(limit: int = Query(default=50, ge=1, le=200) return also_service.get_monthly_differences(limit=limit) +@router.get("/also/dashboard/status-breakdown") +async def get_status_breakdown(): + return also_service.get_status_breakdown() + + +@router.get("/also/dashboard/monthly-history") +async def get_monthly_history(months: int = Query(default=6, ge=1, le=24)): + return also_service.get_monthly_history(months=months) + + @router.post("/also/queue/run-matching", response_model=AlsoQueueProcessResult) async def run_matching(payload: AlsoQueueProcessRequest): return also_service.run_matching( diff --git a/app/modules/also/backend/service.py b/app/modules/also/backend/service.py index fde13d7..b889c6f 100644 --- a/app/modules/also/backend/service.py +++ b/app/modules/also/backend/service.py @@ -1,15 +1,23 @@ import hashlib import json import logging -from datetime import datetime +import csv +import io +import unicodedata +import zipfile +from datetime import datetime, timedelta from decimal import Decimal +from pathlib import Path import re +import xml.etree.ElementTree as ET from typing import Any, Dict, List, Optional from fastapi import HTTPException +import httpx from app.core.config import settings -from app.core.database import execute_query, execute_query_single, table_has_column +from app.core.database import execute_query, execute_query_single, execute_update, get_db_connection, release_db_connection, table_has_column +from psycopg2.extras import RealDictCursor from app.modules.also.models.schemas import ( AlsoCompanyMappingUpsert, AlsoImportJobCreate, @@ -20,6 +28,18 @@ from app.modules.also.models.schemas import ( logger = logging.getLogger(__name__) +ALSO_EFFECTIVE_COST_SQL = ( + "COALESCE(" + "l.cost_amount, " + "CASE " + "WHEN l.matched_product_id IS NOT NULL THEN " + "COALESCE(p.supplier_price, 0) * COALESCE(NULLIF(l.billable_parameters, 0), 1) " + "ELSE 0 " + "END, " + "0)" +) + + def _json_default(value: Any) -> Any: if isinstance(value, Decimal): return float(value) @@ -36,6 +56,16 @@ def _normalized_text(value: Optional[Any]) -> str: return str(value or "").strip() +def _normalize_match_key(value: Optional[Any]) -> str: + text = _normalized_text(value) + if not text: + return "" + text = unicodedata.normalize("NFKD", text) + text = "".join(ch for ch in text if not unicodedata.combining(ch)) + text = re.sub(r"[^a-zA-Z0-9]+", "", text).lower() + return text + + def _to_decimal(value: Any, default: Decimal = Decimal("0")) -> Decimal: if value is None or value == "": return default @@ -50,6 +80,210 @@ def _normalize_cvr(vat_value: Optional[str]) -> str: return digits +def _normalize_download_url(download_url: str) -> str: + raw = _normalized_text(download_url) + if not raw: + return raw + + match = re.match(r"^(https://marketplace\.also\.[^/]+)/#/Redirect/(.+)$", raw, re.IGNORECASE) + if match: + return f"{match.group(1)}/{match.group(2)}" + return raw + + +def _normalize_header(value: Optional[str]) -> str: + return re.sub(r"[^a-z0-9]+", "_", _normalized_text(value).strip().lower()).strip("_") + + +def _parse_decimal_candidate(value: Any) -> Optional[Decimal]: + if value is None: + return None + if isinstance(value, Decimal): + return value + if isinstance(value, (int, float)): + return Decimal(str(value)) + + raw = str(value).strip() + if not raw: + return None + + raw = raw.replace("\u00a0", "").replace(" ", "") + raw = raw.replace("DKK", "").replace("EUR", "").replace("USD", "") + raw = raw.replace("%", "") + if raw.count(",") == 1 and raw.count(".") > 1: + raw = raw.replace(".", "").replace(",", ".") + elif raw.count(",") == 1 and raw.count(".") == 0: + raw = raw.replace(",", ".") + elif raw.count(",") > 1 and raw.count(".") == 0: + raw = raw.replace(",", "") + elif raw.count(".") > 1 and raw.count(",") == 0: + raw = raw.replace(".", "") + elif "," in raw and "." in raw: + if raw.rfind(",") > raw.rfind("."): + raw = raw.replace(".", "").replace(",", ".") + else: + raw = raw.replace(",", "") + try: + return Decimal(raw) + except Exception: + return None + + +def _parse_date_candidate(value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, datetime): + return value.date().isoformat() + if hasattr(value, "isoformat"): + try: + iso = value.isoformat() + if isinstance(iso, str) and iso: + return iso[:10] + except Exception: + pass + + raw = str(value).strip() + if not raw: + return None + + numeric_candidate = _parse_decimal_candidate(raw) + if numeric_candidate is not None: + try: + numeric_float = float(numeric_candidate) + if 20000 <= numeric_float <= 80000: + base = datetime(1899, 12, 30) + return (base + timedelta(days=numeric_float)).date().isoformat() + except Exception: + pass + + for fmt in ("%Y-%m-%d", "%d-%m-%Y", "%d/%m/%Y", "%Y/%m/%d", "%m/%d/%Y", "%d.%m.%Y"): + try: + return datetime.strptime(raw, fmt).date().isoformat() + except ValueError: + continue + try: + return datetime.fromisoformat(raw.replace("Z", "+00:00")).date().isoformat() + except Exception: + return None + + +def _flatten_json_object(value: Dict[str, Any], prefix: str = "") -> Dict[str, Any]: + flattened: Dict[str, Any] = {} + for raw_key, raw_val in value.items(): + key = _normalize_header(raw_key) + if not key: + continue + + target_key = f"{prefix}_{key}" if prefix else key + if isinstance(raw_val, dict): + flattened.update(_flatten_json_object(raw_val, target_key)) + elif isinstance(raw_val, list): + continue + else: + flattened[target_key] = raw_val + if key not in flattened: + flattened[key] = raw_val + return flattened + + +def _derive_file_context(file_name: str) -> Dict[str, Any]: + stem = Path(file_name or "").stem + context: Dict[str, Any] = {} + + date_match = re.search(r"(.+?)_(\d{2}-\d{2}-\d{4})_(\d{2}-\d{2}-\d{4})$", stem) + if date_match: + company = date_match.group(1).replace("_", " ").strip() + if company: + context["company"] = company + start = _parse_date_candidate(date_match.group(2)) + end = _parse_date_candidate(date_match.group(3)) + if start: + context["billing_start"] = start + if end: + context["billing_end"] = end + return context + + if stem: + context["company"] = stem.replace("_", " ").strip() + return context + + +def _has_candidate_fields(row: Dict[str, Any]) -> bool: + candidate_keys = ( + "company", + "customer", + "customer_name", + "tenant", + "companydisplayname", + "product", + "product_name", + "productdisplayname", + "productname", + "subscription", + "description", + "material", + "material_number", + "sku", + "amount", + "total", + "charge", + "price", + "sales_price", + "cost_amount", + "billing_start", + "period_start", + "startdate", + ) + return any(_normalized_text(row.get(key)) for key in candidate_keys) + + +def _is_probable_leaf_json_row(row: Dict[str, Any], has_nested_children: bool) -> bool: + product_keys = ("product_name", "productdisplayname", "productname", "material_number", "sku", "description") + financial_keys = ("charge", "amount", "total", "total_price", "sales_price", "price") + period_keys = ("billing_start", "period_start", "startdate") + + has_product = any(_normalized_text(row.get(key)) for key in product_keys) + has_financial = any(_normalized_text(row.get(key)) for key in financial_keys) + has_period = any(_normalized_text(row.get(key)) for key in period_keys) + + if has_product: + return True + if has_financial and has_period: + return True + if not has_nested_children and _has_candidate_fields(row): + return True + return False + + +def _extract_quantity_from_field_values(value: Any) -> Optional[Decimal]: + text = _normalized_text(value) + if not text: + return None + match = re.search(r"Quantity\s*=\s*([0-9]+(?:[.,][0-9]+)?)", text, re.IGNORECASE) + if not match: + return None + return _parse_decimal_candidate(match.group(1)) + + +def _extract_period_start(value: Any) -> Optional[str]: + text = _normalized_text(value) + if not text: + return None + match = re.match(r"\s*(\d{2}[./-]\d{2}[./-]\d{4})\s*-\s*(\d{2}[./-]\d{2}[./-]\d{4})\s*$", text) + if not match: + return None + return _parse_date_candidate(match.group(1)) + + +def _is_zero_value_tenant_line(line: Dict[str, Any]) -> bool: + product_name = _normalize_match_key(line.get("product_name")) + if "microsoftorganizationtenant" not in product_name: + return False + total_price = _to_decimal(line.get("total_price"), default=Decimal("0")) + unit_price = _to_decimal(line.get("unit_price"), default=Decimal("0")) + return total_price == 0 and unit_price == 0 + + def _line_hash(job_id: int, line_payload: Dict[str, Any]) -> str: source_ref = _normalized_text(line_payload.get("source_line_ref")) if source_ref: @@ -70,6 +304,28 @@ def _line_hash(job_id: int, line_payload: Dict[str, Any]) -> str: class AlsoService: + HEADER_ALIASES = { + "company": ["company", "customer", "customer_name", "firma", "company_name", "kunde", "tenant", "billing_customer", "name", "tenant_name", "customer_company_name", "account_name", "company_display_name", "companydisplayname"], + "customer_id": ["customer_id", "customerid", "kunde_id", "account_customer_id", "external_customer_id"], + "account_id": ["account_id", "accountid", "tenant_id", "subscription_id", "company_account_id", "companyaccountid"], + "vat": ["vat", "vat_number", "cvr", "cvr_number", "orgnr", "organization_number", "customer_vat_id", "department_vat_id"], + "also_company_id": ["also_company_id", "company_id", "cloud_company_id", "reseller_customer_id", "company_account_id", "companyaccountid"], + "material_number": ["material_number", "material", "material_no", "sku", "item_number", "product_code", "varenummer", "part_number", "article_number", "article_no", "service_code"], + "product_name": ["product_name", "product", "description", "service_name", "item_description", "subscription_name", "product_description", "service", "subscription", "offer_name", "license_name", "item_name", "product_display_name", "productdisplayname", "productname"], + "vendor": ["vendor", "manufacturer", "brand", "leverandor", "publisher", "supplier"], + "cost_amount": ["cost_amount", "cost", "costs", "purchase_price", "buy_price", "costprice", "indkob", "cost_total"], + "sales_price": ["sales_price", "sales", "sell_price", "list_price", "omsaetning", "revenue", "sales_total"], + "unit_price": ["unit_price", "price", "unit_cost", "price_per_unit", "monthly_price", "sales_price_per_unit", "unit_sales_price", "sales_price_of_unit", "charge"], + "total_price": ["total_price", "amount", "line_total", "net_amount", "subtotal", "extended_price", "total", "sales_price_total", "sales_price", "total_amount", "charge"], + "currency": ["currency", "valuta"], + "billing_start": ["billing_start", "period_start", "start_date", "billing_start_date", "invoice_date", "service_period_start", "billing_month", "period_from", "billing_from", "valid_from", "from_date", "start_date", "startdate"], + "charge_interval": ["charge_interval", "actual_charge_interval", "actualchargeinterval", "term", "commitment", "period_type", "contract_term"], + "billing_interval": ["billing_interval", "interval", "billing_cycle", "frequency", "charge_frequency"], + "billable_parameters": ["billable_parameters", "billableparameters", "quantity", "qty", "udrc_value", "licenses", "seats", "users", "units", "antal", "license_count", "unit_count", "count"], + "source_line_ref": ["source_line_ref", "line_id", "line_ref", "id", "reference"], + "line_no": ["line_no", "line_number", "lineno", "row_number"], + } + @property def enabled(self) -> bool: return bool(settings.ALSO_ENABLED) @@ -95,6 +351,72 @@ class AlsoService: "preferred_import_order": ["api", "xml", "json_export", "xml_export", "csv"], } + def _mark_import_job_failed(self, job_id: int, error_message: str) -> None: + execute_update( + """ + UPDATE also_import_jobs + SET status = 'failed', + finished_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP, + log_json = COALESCE(log_json, '[]'::jsonb) || %s::jsonb + WHERE id = %s + """, + ( + _json_dumps( + [ + { + "timestamp": datetime.utcnow().isoformat(), + "level": "error", + "message": error_message[:1000], + } + ] + ), + job_id, + ), + ) + + def _complete_import_job( + self, + job_id: int, + *, + status_hint: Optional[str] = None, + ) -> Dict[str, Any]: + queue_totals = execute_query_single( + """ + SELECT + COUNT(*)::INTEGER AS total_lines, + COUNT(*) FILTER (WHERE queue_status = 'approved')::INTEGER AS approved_lines, + COUNT(*) FILTER (WHERE queue_status = 'ready_for_approval')::INTEGER AS ready_lines, + COUNT(*) FILTER (WHERE queue_status = 'error')::INTEGER AS error_lines, + COUNT(*) FILTER (WHERE matched_customer_id IS NULL)::INTEGER AS unmatched_customers, + COUNT(*) FILTER (WHERE matched_product_id IS NULL)::INTEGER AS unmatched_products + FROM also_import_lines + WHERE import_job_id = %s + """, + (job_id,), + ) or {} + + final_status = status_hint or "completed" + if ( + int(queue_totals.get("error_lines") or 0) > 0 + or int(queue_totals.get("unmatched_customers") or 0) > 0 + or int(queue_totals.get("unmatched_products") or 0) > 0 + ): + final_status = "partial" + + execute_update( + """ + UPDATE also_import_jobs + SET status = %s, + finished_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + (final_status, job_id), + ) + + return {"status": final_status, "queue_totals": queue_totals} + def _resolve_customer_match(self, line: Dict[str, Any]) -> Optional[int]: also_company_id = _normalized_text(line.get("also_company_id")) if also_company_id: @@ -139,6 +461,47 @@ class AlsoService: if row and row.get("id"): return int(row["id"]) + normalized_company = _normalize_match_key(company_name) + if normalized_company: + candidate_rows = execute_query( + """ + SELECT id, name + FROM customers + WHERE name ILIKE %s + ORDER BY CHAR_LENGTH(name) ASC, id ASC + LIMIT 50 + """, + (f"%{company_name}%",), + ) or [] + + for candidate in candidate_rows: + candidate_name = candidate.get("name") + if _normalize_match_key(candidate_name) == normalized_company: + return int(candidate["id"]) + + for candidate in candidate_rows: + candidate_key = _normalize_match_key(candidate.get("name")) + if candidate_key.startswith(normalized_company) or normalized_company.startswith(candidate_key): + return int(candidate["id"]) + + fallback_candidates = execute_query( + """ + SELECT id, name + FROM customers + ORDER BY id ASC + LIMIT 5000 + """, + (), + ) or [] + for candidate in fallback_candidates: + candidate_key = _normalize_match_key(candidate.get("name")) + if candidate_key == normalized_company: + return int(candidate["id"]) + for candidate in fallback_candidates: + candidate_key = _normalize_match_key(candidate.get("name")) + if normalized_company and (normalized_company in candidate_key or candidate_key in normalized_company): + return int(candidate["id"]) + return None def _resolve_product_match(self, line: Dict[str, Any]) -> Optional[int]: @@ -160,8 +523,46 @@ class AlsoService: ) if mapped and mapped.get("hub_product_id"): return int(mapped["hub_product_id"]) + elif material_number: + mapped_without_vendor = execute_query_single( + """ + SELECT hub_product_id + FROM also_product_mapping + WHERE material_number = %s + AND is_active = true + ORDER BY id ASC + LIMIT 1 + """, + (material_number,), + ) + if mapped_without_vendor and mapped_without_vendor.get("hub_product_id"): + return int(mapped_without_vendor["hub_product_id"]) if material_number: + direct_product = execute_query_single( + """ + SELECT id + FROM products + WHERE COALESCE(deleted_at, NULL) IS NULL + AND ( + sku_internal = %s + OR supplier_sku = %s + OR LOWER(REGEXP_REPLACE(COALESCE(sku_internal, ''), '[^a-zA-Z0-9]', '', 'g')) = %s + OR LOWER(REGEXP_REPLACE(COALESCE(supplier_sku, ''), '[^a-zA-Z0-9]', '', 'g')) = %s + ) + ORDER BY id ASC + LIMIT 1 + """, + ( + material_number, + material_number, + _normalize_match_key(material_number), + _normalize_match_key(material_number), + ), + ) + if direct_product and direct_product.get("id"): + return int(direct_product["id"]) + supplier = execute_query_single( """ SELECT product_id @@ -178,6 +579,19 @@ class AlsoService: if supplier and supplier.get("product_id"): return int(supplier["product_id"]) + supplier_without_vendor = execute_query_single( + """ + SELECT product_id + FROM product_suppliers + WHERE supplier_sku = %s + ORDER BY id ASC + LIMIT 1 + """, + (material_number,), + ) + if supplier_without_vendor and supplier_without_vendor.get("product_id"): + return int(supplier_without_vendor["product_id"]) + if product_name: row = execute_query_single( "SELECT id FROM products WHERE LOWER(name) = LOWER(%s) LIMIT 1", @@ -186,8 +600,315 @@ class AlsoService: if row and row.get("id"): return int(row["id"]) + fuzzy = execute_query( + """ + SELECT id, name + FROM products + WHERE name ILIKE %s + ORDER BY CHAR_LENGTH(name) ASC, id ASC + LIMIT 25 + """, + (f"%{product_name}%",), + ) or [] + normalized_product = _normalize_match_key(product_name) + for candidate in fuzzy: + candidate_key = _normalize_match_key(candidate.get("name")) + if candidate_key == normalized_product: + return int(candidate["id"]) + for candidate in fuzzy: + candidate_key = _normalize_match_key(candidate.get("name")) + if normalized_product and (normalized_product in candidate_key or candidate_key in normalized_product): + return int(candidate["id"]) + return None + def _auto_map_customer_for_line(self, line: Dict[str, Any]) -> bool: + also_company_id = _normalized_text(line.get("also_company_id")) + if not also_company_id: + return False + + existing = execute_query_single( + """ + SELECT id + FROM also_company_mapping + WHERE also_company_id = %s + AND is_active = true + LIMIT 1 + """, + (also_company_id,), + ) + if existing: + return False + + candidate_customer_id = self._resolve_customer_match(line) + if not candidate_customer_id: + return False + + execute_query( + """ + INSERT INTO also_company_mapping ( + also_company_id, + also_customer_id, + customer_id, + match_confidence, + notes, + is_active + ) VALUES (%s, %s, %s, %s, %s, true) + ON CONFLICT (also_company_id) + DO UPDATE SET + customer_id = EXCLUDED.customer_id, + also_customer_id = EXCLUDED.also_customer_id, + match_confidence = EXCLUDED.match_confidence, + notes = EXCLUDED.notes, + is_active = true, + updated_at = CURRENT_TIMESTAMP + """, + ( + also_company_id, + _normalized_text(line.get("customer_id")) or _normalized_text(line.get("account_id")) or None, + candidate_customer_id, + Decimal("0.85"), + "Auto-created from ALSO import", + ), + ) + return True + + def _auto_map_product_for_line(self, line: Dict[str, Any]) -> bool: + material_number = _normalized_text(line.get("material_number")) + if not material_number or _is_zero_value_tenant_line(line): + return False + + vendor = _normalized_text(line.get("vendor")) or "ALSO" + existing = execute_query_single( + """ + SELECT id + FROM also_product_mapping + WHERE material_number = %s + AND LOWER(vendor) = LOWER(%s) + AND is_active = true + LIMIT 1 + """, + (material_number, vendor), + ) + if existing: + return False + + candidate_product_id = self._resolve_product_match( + { + **line, + "vendor": line.get("vendor") or vendor, + } + ) + if not candidate_product_id: + return False + + execute_query( + """ + INSERT INTO also_product_mapping ( + material_number, + vendor, + hub_product_id, + product_name_snapshot, + is_active + ) VALUES (%s, %s, %s, %s, true) + ON CONFLICT (material_number, vendor) + DO UPDATE SET + hub_product_id = EXCLUDED.hub_product_id, + product_name_snapshot = EXCLUDED.product_name_snapshot, + is_active = true, + updated_at = CURRENT_TIMESTAMP + """, + ( + material_number, + vendor, + candidate_product_id, + _normalized_text(line.get("product_name")) or None, + ), + ) + return True + + def _create_local_product_for_line(self, line: Dict[str, Any], vendor: str) -> Optional[int]: + material_number = _normalized_text(line.get("material_number")) + product_name = _normalized_text(line.get("product_name")) + if not material_number or not product_name: + return None + + existing = execute_query_single( + """ + SELECT id + FROM products + WHERE deleted_at IS NULL + AND ( + sku_internal = %s + OR supplier_sku = %s + ) + ORDER BY id ASC + LIMIT 1 + """, + (material_number, material_number), + ) + if existing and existing.get("id"): + return int(existing["id"]) + + manufacturer = vendor + if not manufacturer or manufacturer == "ALSO": + if "microsoft" in _normalize_match_key(product_name): + manufacturer = "Microsoft" + else: + manufacturer = "ALSO" + + sales_price = _to_decimal(line.get("unit_price"), default=Decimal("0")) + created = execute_query( + """ + INSERT INTO products ( + name, + short_description, + type, + status, + sku_internal, + manufacturer, + supplier_name, + supplier_sku, + supplier_price, + supplier_currency, + sales_price, + vat_rate, + billable + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s + ) + RETURNING id + """, + ( + product_name, + "Auto-oprettet fra ALSO Cloud Marketplace", + "subscription", + "active", + material_number, + manufacturer, + "ALSO Cloud Marketplace", + material_number, + sales_price, + _normalized_text(line.get("currency")) or "DKK", + sales_price, + Decimal("25.00"), + True, + ), + ) or [] + if not created: + return None + + product_id = int(created[0]["id"]) + execute_query( + """ + INSERT INTO product_suppliers ( + product_id, + supplier_name, + supplier_code, + supplier_sku, + supplier_price, + supplier_currency, + source, + last_updated_at + ) VALUES (%s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP) + RETURNING id + """, + ( + product_id, + "ALSO Cloud Marketplace", + vendor or "ALSO", + material_number, + sales_price, + _normalized_text(line.get("currency")) or "DKK", + "also_cloud", + ), + ) + return product_id + + def auto_map_import_job(self, job_id: int) -> Dict[str, Any]: + self._assert_enabled() + lines = self.get_import_job_lines(job_id=job_id, status=None, limit=5000) + if not lines: + raise HTTPException(status_code=404, detail="No lines found for import job") + + customer_mappings_created = 0 + product_mappings_created = 0 + products_created = 0 + + for line in lines: + if self._auto_map_customer_for_line(line): + customer_mappings_created += 1 + if self._auto_map_product_for_line(line): + product_mappings_created += 1 + continue + + if _is_zero_value_tenant_line(line): + continue + + if line.get("matched_product_id"): + continue + + material_number = _normalized_text(line.get("material_number")) + if not material_number: + continue + + vendor = _normalized_text(line.get("vendor")) or "ALSO" + created_product_id = self._create_local_product_for_line(line, vendor=vendor) + if not created_product_id: + continue + + products_created += 1 + execute_query( + """ + INSERT INTO also_product_mapping ( + material_number, + vendor, + hub_product_id, + product_name_snapshot, + is_active + ) VALUES (%s, %s, %s, %s, true) + ON CONFLICT (material_number, vendor) + DO UPDATE SET + hub_product_id = EXCLUDED.hub_product_id, + product_name_snapshot = EXCLUDED.product_name_snapshot, + is_active = true, + updated_at = CURRENT_TIMESTAMP + """, + ( + material_number, + vendor, + created_product_id, + _normalized_text(line.get("product_name")) or None, + ), + ) + product_mappings_created += 1 + + matching_result = self.run_matching(import_job_id=job_id, line_ids=[], limit=5000) + validation_result = self.run_validation(import_job_id=job_id, line_ids=[], limit=5000) + + approval_result: Dict[str, Any] = {"approved_lines": 0, "created_drafts": 0, "draft_ids": []} + ready_count = int(validation_result.get("ready_for_approval") or 0) + if ready_count > 0: + approval_result = self.approve_lines_to_drafts( + import_job_id=job_id, + line_ids=[], + approved_by_user_id=None, + ) + + completion = self._complete_import_job(job_id) + + return { + "job_id": job_id, + "customer_mappings_created": customer_mappings_created, + "product_mappings_created": product_mappings_created, + "products_created": products_created, + "matching_result": matching_result, + "validation_result": validation_result, + "approval_result": approval_result, + "queue_totals": completion["queue_totals"], + "status": completion["status"], + } + def _derive_queue_status(self, matched_customer_id: Optional[int], matched_product_id: Optional[int], has_errors: bool) -> str: if has_errors: return "error" @@ -200,6 +921,9 @@ class AlsoService: return "new" def _build_validation_errors(self, line: Dict[str, Any], duplicate_ids: set[int]) -> List[Dict[str, Any]]: + if _is_zero_value_tenant_line(line): + return [] + errors: List[Dict[str, Any]] = [] if not line.get("matched_customer_id"): @@ -301,6 +1025,500 @@ class AlsoService: ) return rows[0] + def _pick_value(self, row: Dict[str, Any], field_name: str) -> Any: + for alias in self.HEADER_ALIASES.get(field_name, []): + if alias in row and row[alias] not in (None, ""): + return row[alias] + return None + + def _normalize_import_row( + self, + row: Dict[str, Any], + source_line_ref: str, + file_context: Optional[Dict[str, Any]] = None, + ) -> Optional[Dict[str, Any]]: + merged_row = dict(file_context or {}) + merged_row.update(row) + normalized = {_normalize_header(key): value for key, value in merged_row.items()} + customer_id_value = self._pick_value(normalized, "customer_id") + account_id_value = self._pick_value(normalized, "account_id") + also_company_id_value = self._pick_value(normalized, "also_company_id") + + product_name = ( + normalized.get("productdisplayname") + or normalized.get("product_display_name") + or self._pick_value(normalized, "product_name") + ) + material_number = self._pick_value(normalized, "material_number") or normalized.get("productname") + company = ( + normalized.get("companydisplayname") + or normalized.get("company_display_name") + or self._pick_value(normalized, "company") + ) + explicit_charge = _parse_decimal_candidate(normalized.get("charge")) + total_price = explicit_charge if explicit_charge is not None else _parse_decimal_candidate(self._pick_value(normalized, "total_price")) + sales_price = _parse_decimal_candidate(self._pick_value(normalized, "sales_price")) + cost_amount = _parse_decimal_candidate(self._pick_value(normalized, "cost_amount")) + unit_price = explicit_charge if explicit_charge is not None else _parse_decimal_candidate(self._pick_value(normalized, "unit_price")) + quantity = ( + _extract_quantity_from_field_values(normalized.get("fieldvalues")) + or _extract_quantity_from_field_values(normalized.get("billableparameters")) + or _parse_decimal_candidate(self._pick_value(normalized, "billable_parameters")) + ) + charge_interval_value = self._pick_value(normalized, "charge_interval") + billing_start_value = ( + _extract_period_start(charge_interval_value) + or normalized.get("startdate") + or normalized.get("start_date") + or self._pick_value(normalized, "billing_start") + ) + + if total_price is None and sales_price is not None: + total_price = sales_price + + if not any([product_name, material_number, company, total_price, sales_price, cost_amount]): + return None + + return { + "line_no": self._pick_value(normalized, "line_no"), + "source_line_ref": self._pick_value(normalized, "source_line_ref") or source_line_ref, + "company": company, + "customer_id": str(customer_id_value) if customer_id_value not in (None, "") else None, + "account_id": str(account_id_value) if account_id_value not in (None, "") else None, + "vat": self._pick_value(normalized, "vat"), + "also_company_id": str(also_company_id_value) if also_company_id_value not in (None, "") else None, + "material_number": material_number, + "product_name": product_name, + "vendor": self._pick_value(normalized, "vendor"), + "cost_amount": str(cost_amount) if cost_amount is not None else None, + "sales_price": str(sales_price) if sales_price is not None else None, + "unit_price": str(unit_price) if unit_price is not None else None, + "total_price": str(total_price) if total_price is not None else None, + "currency": self._pick_value(normalized, "currency") or "DKK", + "billing_start": _parse_date_candidate(billing_start_value) or _parse_date_candidate((file_context or {}).get("billing_start")), + "charge_interval": charge_interval_value, + "billing_interval": self._pick_value(normalized, "billing_interval"), + "billable_parameters": str(quantity) if quantity is not None else None, + "raw_line": merged_row, + } + + def _parse_delimited_file(self, payload: bytes) -> List[Dict[str, Any]]: + text = payload.decode("utf-8-sig", errors="ignore") + if not text.strip(): + return [] + + delimiter = "," + sample = text[:5000] + try: + delimiter = csv.Sniffer().sniff(sample, delimiters=",;\t|").delimiter + except Exception: + if ";" in sample: + delimiter = ";" + elif "\t" in sample: + delimiter = "\t" + + reader = csv.DictReader(io.StringIO(text), delimiter=delimiter) + return [dict(row) for row in reader if any(_normalized_text(value) for value in row.values())] + + def _parse_json_file(self, payload: bytes, file_name: Optional[str] = None) -> List[Dict[str, Any]]: + decoded = json.loads(payload.decode("utf-8", errors="ignore")) + file_context = _derive_file_context(file_name or "") + rows: List[Dict[str, Any]] = [] + seen: set[str] = set() + + def add_row(candidate: Dict[str, Any], inherited: Optional[Dict[str, Any]] = None) -> None: + merged = dict(file_context) + if inherited: + merged.update(inherited) + merged.update(candidate) + if not _has_candidate_fields(merged): + return + key = json.dumps(merged, ensure_ascii=False, default=str, sort_keys=True) + if key in seen: + return + seen.add(key) + rows.append(merged) + + def walk(node: Any, inherited: Optional[Dict[str, Any]] = None) -> None: + if isinstance(node, list): + for item in node: + walk(item, inherited) + return + + if not isinstance(node, dict): + return + + flat = _flatten_json_object(node) + merged_inherited = dict(inherited or {}) + for key, value in flat.items(): + if value in (None, "", [], {}): + continue + if key not in merged_inherited: + merged_inherited[key] = value + + has_nested_children = any(isinstance(value, (dict, list)) for value in node.values()) + if _is_probable_leaf_json_row(flat, has_nested_children=has_nested_children): + add_row(flat, inherited) + + for value in node.values(): + if isinstance(value, dict): + walk(value, merged_inherited) + elif isinstance(value, list): + for item in value: + walk(item, merged_inherited) + + walk(decoded, {}) + return rows + + def _parse_xml_file(self, payload: bytes) -> List[Dict[str, Any]]: + root = ET.fromstring(payload) + rows: List[Dict[str, Any]] = [] + for candidate in root.findall(".//row") + root.findall(".//line") + root.findall(".//item") + root.findall(".//record"): + row: Dict[str, Any] = {} + for child in list(candidate): + row[_normalize_header(child.tag)] = child.text + if row: + rows.append(row) + return rows + + def _parse_excel_file(self, payload: bytes) -> List[Dict[str, Any]]: + if not zipfile.is_zipfile(io.BytesIO(payload)): + raise HTTPException(status_code=422, detail="Excel-filen kunne ikke læses som XLSX") + + ns = { + "a": "http://schemas.openxmlformats.org/spreadsheetml/2006/main", + "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships", + "pr": "http://schemas.openxmlformats.org/package/2006/relationships", + } + + def _column_index(ref: str) -> int: + letters = "".join(ch for ch in ref if ch.isalpha()).upper() + value = 0 + for ch in letters: + value = (value * 26) + (ord(ch) - 64) + return max(value - 1, 0) + + with zipfile.ZipFile(io.BytesIO(payload)) as workbook_zip: + shared_strings: List[str] = [] + if "xl/sharedStrings.xml" in workbook_zip.namelist(): + shared_root = ET.fromstring(workbook_zip.read("xl/sharedStrings.xml")) + for si in shared_root.findall("a:si", ns): + fragments = [node.text or "" for node in si.iterfind(".//a:t", ns)] + shared_strings.append("".join(fragments)) + + workbook_root = ET.fromstring(workbook_zip.read("xl/workbook.xml")) + rel_root = ET.fromstring(workbook_zip.read("xl/_rels/workbook.xml.rels")) + relationship_map = { + rel.attrib["Id"]: rel.attrib["Target"] + for rel in rel_root.findall("pr:Relationship", ns) + } + + rows: List[Dict[str, Any]] = [] + + for sheet in workbook_root.find("a:sheets", ns) or []: + relation_id = sheet.attrib.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id") + target = relationship_map.get(relation_id or "") + if not target: + continue + + sheet_path = target if target.startswith("xl/") else f"xl/{target}" + sheet_root = ET.fromstring(workbook_zip.read(sheet_path)) + header: Optional[List[str]] = None + + for row_node in sheet_root.findall(".//a:sheetData/a:row", ns): + cell_map: Dict[int, Any] = {} + for cell in row_node.findall("a:c", ns): + ref = cell.attrib.get("r", "") + idx = _column_index(ref) + value_node = cell.find("a:v", ns) + cell_type = cell.attrib.get("t") + + if cell_type == "inlineStr": + inline_node = cell.find("a:is", ns) + value = "".join(node.text or "" for node in inline_node.iterfind(".//a:t", ns)) if inline_node is not None else "" + elif value_node is None: + value = "" + else: + raw_value = value_node.text or "" + if cell_type == "s": + try: + value = shared_strings[int(raw_value)] + except Exception: + value = raw_value + else: + value = raw_value + + cell_map[idx] = value + + if not cell_map: + continue + + max_idx = max(cell_map) + values = [cell_map.get(i, "") for i in range(max_idx + 1)] + + if header is None: + candidate_header = [_normalized_text(v) for v in values] + if not any(candidate_header): + continue + header = candidate_header + continue + + if not any(_normalized_text(v) for v in values): + continue + + row: Dict[str, Any] = {} + for idx, column_name in enumerate(header): + key = column_name.strip() + if not key: + continue + row[key] = values[idx] if idx < len(values) else "" + if any(_normalized_text(v) for v in row.values()): + rows.append(row) + + return rows + + def _extract_zip_rows(self, zip_bytes: bytes) -> Dict[str, Any]: + lines: List[Dict[str, Any]] = [] + extracted_files: List[str] = [] + source_type = "csv" + + with zipfile.ZipFile(io.BytesIO(zip_bytes)) as archive: + for member in archive.infolist(): + if member.is_dir(): + continue + + name = member.filename + ext = Path(name).suffix.lower() + if ext not in {".csv", ".tsv", ".txt", ".json", ".xml", ".xlsx", ".xls"}: + continue + + extracted_files.append(name) + payload = archive.read(member) + source_rows: List[Dict[str, Any]] = [] + + if ext in {".csv", ".tsv", ".txt"}: + source_rows = self._parse_delimited_file(payload) + source_type = "csv" + elif ext == ".json": + source_rows = self._parse_json_file(payload, file_name=name) + source_type = "json_export" + elif ext == ".xml": + source_rows = self._parse_xml_file(payload) + source_type = "xml_export" + elif ext in {".xlsx", ".xls"}: + source_rows = self._parse_excel_file(payload) + source_type = "csv" + + file_context = _derive_file_context(name) + for index, row in enumerate(source_rows, start=1): + normalized = self._normalize_import_row( + row, + source_line_ref=f"{name}:{index}", + file_context=file_context, + ) + if normalized: + lines.append(normalized) + + return { + "source_type": source_type, + "lines": lines, + "files": extracted_files, + } + + async def import_billing_zip_from_url( + self, + download_url: str, + imported_by_user_id: Optional[int] = None, + email_id: Optional[int] = None, + source_label: Optional[str] = None, + ) -> Dict[str, Any]: + self._assert_enabled() + normalized_url = _normalize_download_url(download_url) + + timeout = httpx.Timeout( + connect=min(float(settings.ALSO_TIMEOUT_SECONDS or 20), 20.0), + read=max(float(settings.ALSO_TIMEOUT_SECONDS or 20), 20.0), + write=20.0, + pool=20.0, + ) + + async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client: + response = await client.get(normalized_url) + if response.headers.get("content-type", "").lower().startswith("application/json"): + try: + payload = response.json() + except Exception: + payload = {} + if payload.get("error") and "authenticate header" in _normalized_text(payload.get("text")).lower(): + raise HTTPException( + status_code=502, + detail="ALSO download kræver gyldig Authenticate-header/session. Linket alene er ikke nok fra server-side workflow.", + ) + response.raise_for_status() + zip_bytes = response.content + + if not zipfile.is_zipfile(io.BytesIO(zip_bytes)): + raise HTTPException(status_code=422, detail="ALSO download returnerede ikke en ZIP-fil") + + parsed = self._extract_zip_rows(zip_bytes) + normalized_lines = parsed.get("lines") or [] + if not normalized_lines: + raise HTTPException(status_code=422, detail="ALSO billing ZIP contained no recognizable billing rows") + + file_name = normalized_url.rstrip("/").split("/")[-1] or "also-billing.zip" + job = self.create_import_job( + AlsoImportJobCreate( + source_type=parsed.get("source_type") or "csv", + source_label=source_label or "Email workflow", + file_name=file_name, + import_version="email_workflow_v1", + raw_payload={ + "email_id": email_id, + "download_url": normalized_url, + "zip_entries": parsed.get("files") or [], + "line_count": len(normalized_lines), + }, + ), + imported_by_user_id=imported_by_user_id, + ) + job_id = int(job["id"]) + try: + import_result = self.import_lines( + job_id=job_id, + payload=AlsoImportLinesRequest(lines=normalized_lines), + ) + matching_result = self.run_matching(import_job_id=job_id, line_ids=[], limit=5000) + validation_result = self.run_validation(import_job_id=job_id, line_ids=[], limit=5000) + + approval_result: Dict[str, Any] = {"approved_lines": 0, "created_drafts": 0, "draft_ids": []} + ready_count = int(validation_result.get("ready_for_approval") or 0) + if ready_count > 0: + approval_result = self.approve_lines_to_drafts( + import_job_id=job_id, + line_ids=[], + approved_by_user_id=imported_by_user_id, + ) + + completion = self._complete_import_job(job_id) + final_status = completion["status"] + queue_totals = completion["queue_totals"] + except Exception as e: + self._mark_import_job_failed(job_id, str(e)) + raise + + if email_id: + execute_update( + """ + UPDATE email_messages + SET status = 'processed', + folder = 'Processed', + processed_at = CURRENT_TIMESTAMP, + auto_processed = true, + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + (email_id,), + ) + + return { + "job_id": int(job["id"]), + "status": final_status, + "download_url": normalized_url, + "zip_entries": parsed.get("files") or [], + "import_result": import_result, + "matching_result": matching_result, + "validation_result": validation_result, + "approval_result": approval_result, + "queue_totals": queue_totals, + } + + def import_billing_upload( + self, + *, + file_name: str, + file_bytes: bytes, + imported_by_user_id: Optional[int] = None, + source_label: Optional[str] = None, + ) -> Dict[str, Any]: + self._assert_enabled() + + safe_file_name = Path(file_name or "also-billing.xlsx").name + ext = Path(safe_file_name).suffix.lower() + + if ext == ".xlsx": + source_rows = self._parse_excel_file(file_bytes) + parsed = { + "source_type": "csv", + "lines": [ + self._normalize_import_row( + row, + source_line_ref=f"{safe_file_name}:{index}", + file_context=None, + ) + for index, row in enumerate(source_rows, start=1) + ], + "files": [safe_file_name], + } + parsed["lines"] = [row for row in (parsed.get("lines") or []) if row] + elif zipfile.is_zipfile(io.BytesIO(file_bytes)): + parsed = self._extract_zip_rows(file_bytes) + else: + raise HTTPException(status_code=422, detail="Upload skal være en XLSX-fil eller en gyldig ZIP-fil") + + normalized_lines = parsed.get("lines") or [] + if not normalized_lines: + raise HTTPException(status_code=422, detail="Filen indeholdt ingen genkendelige billing-linjer") + + job = self.create_import_job( + AlsoImportJobCreate( + source_type=parsed.get("source_type") or "csv", + source_label=source_label or "Manual upload", + file_name=safe_file_name, + import_version="manual_upload_v1", + raw_payload={ + "upload_filename": safe_file_name, + "uploaded_entries": parsed.get("files") or [], + "upload_format": ext.lstrip(".") or "zip", + "line_count": len(normalized_lines), + }, + ), + imported_by_user_id=imported_by_user_id, + ) + + job_id = int(job["id"]) + try: + import_result = self.import_lines( + job_id=job_id, + payload=AlsoImportLinesRequest(lines=normalized_lines), + ) + matching_result = self.run_matching(import_job_id=job_id, line_ids=[], limit=5000) + validation_result = self.run_validation(import_job_id=job_id, line_ids=[], limit=5000) + + approval_result: Dict[str, Any] = {"approved_lines": 0, "created_drafts": 0, "draft_ids": []} + ready_count = int(validation_result.get("ready_for_approval") or 0) + if ready_count > 0: + approval_result = self.approve_lines_to_drafts( + import_job_id=job_id, + line_ids=[], + approved_by_user_id=imported_by_user_id, + ) + + completion = self._complete_import_job(job_id) + + return { + "job_id": job_id, + "status": completion["status"], + "zip_entries": parsed.get("files") or [], + "import_result": import_result, + "matching_result": matching_result, + "validation_result": validation_result, + "approval_result": approval_result, + "queue_totals": completion["queue_totals"], + } + except Exception as e: + self._mark_import_job_failed(job_id, str(e)) + raise + def list_import_jobs(self, status: Optional[str], limit: int) -> List[Dict[str, Any]]: self._assert_enabled() if status: @@ -466,26 +1684,171 @@ class AlsoService: self._assert_enabled() if status: return execute_query( - """ - SELECT * - FROM also_import_lines - WHERE queue_status = %s - ORDER BY id DESC + f""" + SELECT + l.*, + {ALSO_EFFECTIVE_COST_SQL} AS effective_cost_amount, + COALESCE(l.total_price, l.sales_price, 0) - ({ALSO_EFFECTIVE_COST_SQL}) AS effective_margin_amount + FROM also_import_lines l + LEFT JOIN products p ON p.id = l.matched_product_id + WHERE l.queue_status = %s + ORDER BY l.id DESC LIMIT %s """, (status, max(1, min(limit, 1000))), ) or [] return execute_query( - """ - SELECT * - FROM also_import_lines - ORDER BY id DESC + f""" + SELECT + l.*, + {ALSO_EFFECTIVE_COST_SQL} AS effective_cost_amount, + COALESCE(l.total_price, l.sales_price, 0) - ({ALSO_EFFECTIVE_COST_SQL}) AS effective_margin_amount + FROM also_import_lines l + LEFT JOIN products p ON p.id = l.matched_product_id + ORDER BY l.id DESC LIMIT %s """, (max(1, min(limit, 1000)),), ) or [] + def get_import_job_lines(self, job_id: int, status: Optional[str], limit: int) -> List[Dict[str, Any]]: + self._assert_enabled() + params: List[Any] = [job_id] + where = ["l.import_job_id = %s"] + + if status: + where.append("l.queue_status = %s") + params.append(status) + + params.append(max(1, min(limit, 2000))) + return execute_query( + f""" + SELECT + l.*, + c.name AS matched_customer_name, + p.name AS matched_product_name, + {ALSO_EFFECTIVE_COST_SQL} AS effective_cost_amount, + COALESCE(l.total_price, l.sales_price, 0) - ({ALSO_EFFECTIVE_COST_SQL}) AS effective_margin_amount, + d.title AS order_draft_title, + d.sync_status AS order_draft_sync_status, + d.economic_order_number, + d.economic_invoice_number + FROM also_import_lines l + LEFT JOIN customers c ON c.id = l.matched_customer_id + LEFT JOIN products p ON p.id = l.matched_product_id + LEFT JOIN ordre_drafts d ON d.id = l.order_draft_id + WHERE {' AND '.join(where)} + ORDER BY l.line_no ASC NULLS LAST, l.id ASC + LIMIT %s + """, + tuple(params), + ) or [] + + def delete_import_job(self, job_id: int, *, delete_order_drafts: bool = False) -> Dict[str, Any]: + self._assert_enabled() + conn = get_db_connection() + try: + with conn.cursor(cursor_factory=RealDictCursor) as cursor: + cursor.execute( + """ + SELECT + j.*, + COALESCE(l.line_count, 0) AS line_count + FROM also_import_jobs j + LEFT JOIN ( + SELECT import_job_id, COUNT(*) AS line_count + FROM also_import_lines + WHERE import_job_id = %s + GROUP BY import_job_id + ) l ON l.import_job_id = j.id + WHERE j.id = %s + """, + (job_id, job_id), + ) + job = cursor.fetchone() + if not job: + raise HTTPException(status_code=404, detail="Import job not found") + + cursor.execute( + """ + SELECT + d.id, + d.title, + d.sync_status, + d.economic_order_number, + d.economic_invoice_number + FROM ordre_drafts d + JOIN ( + SELECT DISTINCT order_draft_id + FROM also_import_lines + WHERE import_job_id = %s + AND order_draft_id IS NOT NULL + ) x ON x.order_draft_id = d.id + ORDER BY d.id ASC + """, + (job_id,), + ) + drafts = cursor.fetchall() or [] + + blocked_statuses = {"exported", "posted", "paid"} + blocked_drafts = [dict(row) for row in drafts if str(row.get("sync_status") or "").strip().lower() in blocked_statuses] + + if drafts and not delete_order_drafts: + raise HTTPException( + status_code=409, + detail={ + "message": "Import job has linked ordre drafts. Confirm deletion with delete_order_drafts=true if drafts should be removed too.", + "draft_count": len(drafts), + "blocked_draft_count": len(blocked_drafts), + "drafts": [dict(row) for row in drafts], + }, + ) + + if blocked_drafts: + raise HTTPException( + status_code=409, + detail={ + "message": "One or more linked ordre drafts are already exported/posted/paid and cannot be auto-deleted.", + "blocked_drafts": blocked_drafts, + }, + ) + + deleted_draft_ids: List[int] = [] + if drafts and delete_order_drafts: + draft_ids = [int(row["id"]) for row in drafts if row.get("id") is not None] + if draft_ids: + placeholders = ",".join(["%s"] * len(draft_ids)) + cursor.execute( + f"DELETE FROM ordre_drafts WHERE id IN ({placeholders})", + tuple(draft_ids), + ) + deleted_draft_ids = draft_ids + + cursor.execute( + "DELETE FROM also_import_jobs WHERE id = %s RETURNING id", + (job_id,), + ) + deleted = cursor.fetchone() + if not deleted: + raise HTTPException(status_code=404, detail="Import job not found") + + conn.commit() + return { + "success": True, + "deleted_job_id": job_id, + "deleted_line_count": int(job.get("line_count") or 0), + "deleted_order_draft_ids": deleted_draft_ids, + } + except HTTPException: + conn.rollback() + raise + except Exception: + conn.rollback() + raise + finally: + release_db_connection(conn) + def run_matching(self, import_job_id: Optional[int], line_ids: List[int], limit: int) -> Dict[str, Any]: self._assert_enabled() lines = self._fetch_process_lines(import_job_id=import_job_id, line_ids=line_ids, limit=limit) @@ -500,7 +1863,10 @@ class AlsoService: current_errors = line.get("validation_errors_json") or [] has_errors = bool(current_errors) - new_status = self._derive_queue_status(matched_customer_id, matched_product_id, has_errors) + if _is_zero_value_tenant_line(line): + new_status = "approved" + else: + new_status = self._derive_queue_status(matched_customer_id, matched_product_id, has_errors) execute_query( """ @@ -556,7 +1922,11 @@ class AlsoService: for line in lines: errors = self._build_validation_errors(line, duplicate_ids=duplicate_ids) - new_status = self._derive_queue_status(line.get("matched_customer_id"), line.get("matched_product_id"), bool(errors)) + if _is_zero_value_tenant_line(line): + errors = [] + new_status = "approved" + else: + new_status = self._derive_queue_status(line.get("matched_customer_id"), line.get("matched_product_id"), bool(errors)) execute_query( """ @@ -725,16 +2095,19 @@ class AlsoService: def get_dashboard_summary(self) -> Dict[str, Any]: self._assert_enabled() row = execute_query_single( - """ + f""" WITH month_lines AS ( - SELECT * - FROM also_import_lines - WHERE date_trunc('month', COALESCE(billing_start::timestamp, created_at)) = date_trunc('month', CURRENT_DATE) + SELECT + l.*, + {ALSO_EFFECTIVE_COST_SQL} AS effective_cost_amount + FROM also_import_lines l + LEFT JOIN products p ON p.id = l.matched_product_id + WHERE date_trunc('month', COALESCE(l.billing_start::timestamp, l.created_at)) = date_trunc('month', CURRENT_DATE) ) SELECT COALESCE(SUM(COALESCE(total_price, sales_price, 0)), 0) AS monthly_revenue, - COALESCE(SUM(COALESCE(cost_amount, 0)), 0) AS monthly_cost, - COALESCE(SUM(COALESCE(total_price, sales_price, 0) - COALESCE(cost_amount, 0)), 0) AS monthly_margin, + COALESCE(SUM(COALESCE(effective_cost_amount, 0)), 0) AS monthly_cost, + COALESCE(SUM(COALESCE(total_price, sales_price, 0) - COALESCE(effective_cost_amount, 0)), 0) AS monthly_margin, COUNT(*) FILTER (WHERE matched_product_id IS NULL) AS unmatched_products, COUNT(*) FILTER (WHERE matched_customer_id IS NULL) AS unmatched_customers, COUNT(*) FILTER (WHERE queue_status = 'ready_for_approval') AS pending_approvals, @@ -843,6 +2216,97 @@ class AlsoService: return results + def get_status_breakdown(self) -> Dict[str, Any]: + self._assert_enabled() + line_rows = execute_query( + """ + SELECT + queue_status, + COUNT(*)::INTEGER AS total_count, + COUNT(*) FILTER ( + WHERE date_trunc('month', COALESCE(billing_start::timestamp, created_at)) = date_trunc('month', CURRENT_DATE) + )::INTEGER AS current_month_count + FROM also_import_lines + GROUP BY queue_status + ORDER BY total_count DESC, queue_status ASC + """, + (), + ) or [] + + job_rows = execute_query( + """ + SELECT + status, + COUNT(*)::INTEGER AS total_count + FROM also_import_jobs + GROUP BY status + ORDER BY total_count DESC, status ASC + """, + (), + ) or [] + + return { + "line_statuses": line_rows, + "job_statuses": job_rows, + } + + def get_monthly_history(self, months: int = 6) -> List[Dict[str, Any]]: + self._assert_enabled() + rows = execute_query( + """ + WITH month_series AS ( + SELECT generate_series( + date_trunc('month', CURRENT_DATE) - (%s::INTEGER - 1) * INTERVAL '1 month', + date_trunc('month', CURRENT_DATE), + INTERVAL '1 month' + ) AS month_start + ), + line_agg AS ( + SELECT + date_trunc('month', COALESCE(l.billing_start::timestamp, l.created_at)) AS month_start, + COALESCE(SUM(COALESCE(l.total_price, l.sales_price, 0)), 0) AS revenue, + COALESCE(SUM( + COALESCE( + l.cost_amount, + CASE + WHEN l.matched_product_id IS NOT NULL THEN + COALESCE(p.supplier_price, 0) * COALESCE(NULLIF(l.billable_parameters, 0), 1) + ELSE 0 + END, + 0 + ) + ), 0) AS cost, + COUNT(*)::INTEGER AS line_count, + COUNT(DISTINCT l.matched_customer_id)::INTEGER AS customer_count + FROM also_import_lines l + LEFT JOIN products p ON p.id = l.matched_product_id + GROUP BY 1 + ), + job_agg AS ( + SELECT + date_trunc('month', imported_at) AS month_start, + COUNT(*)::INTEGER AS job_count + FROM also_import_jobs + GROUP BY 1 + ) + SELECT + ms.month_start::date AS month_start, + COALESCE(la.revenue, 0) AS revenue, + COALESCE(la.cost, 0) AS cost, + COALESCE(la.revenue, 0) - COALESCE(la.cost, 0) AS margin, + COALESCE(la.line_count, 0) AS line_count, + COALESCE(la.customer_count, 0) AS customer_count, + COALESCE(ja.job_count, 0) AS job_count + FROM month_series ms + LEFT JOIN line_agg la ON la.month_start = ms.month_start + LEFT JOIN job_agg ja ON ja.month_start = ms.month_start + ORDER BY ms.month_start DESC + """, + (max(1, min(months, 24)),), + ) or [] + + return rows + def upsert_company_mapping(self, payload: AlsoCompanyMappingUpsert) -> Dict[str, Any]: self._assert_enabled() rows = execute_query( @@ -875,6 +2339,232 @@ class AlsoService: ) return rows[0] + def manual_map_company_for_line( + self, + *, + line_id: int, + customer_id: int, + notes: Optional[str], + updated_by_user_id: Optional[int], + ) -> Dict[str, Any]: + self._assert_enabled() + + line = execute_query_single( + """ + SELECT * + FROM also_import_lines + WHERE id = %s + """, + (line_id,), + ) + if not line: + raise HTTPException(status_code=404, detail="ALSO import line not found") + + also_company_id = _normalized_text(line.get("also_company_id")) + if not also_company_id: + raise HTTPException(status_code=400, detail="Line is missing also_company_id and cannot be mapped manually") + + customer = execute_query_single( + """ + SELECT id, name + FROM customers + WHERE id = %s + AND deleted_at IS NULL + AND is_active = true + """, + (customer_id,), + ) + if not customer: + raise HTTPException(status_code=404, detail="Customer not found or inactive") + + mapping = self.upsert_company_mapping( + AlsoCompanyMappingUpsert( + also_company_id=also_company_id, + also_customer_id=_normalized_text(line.get("customer_id")) or _normalized_text(line.get("account_id")) or None, + customer_id=customer_id, + match_confidence=Decimal("1.00"), + notes=notes or "Manual mapping from ALSO Cloud Marketplace", + ) + ) + + related_rows = execute_query( + """ + SELECT id + FROM also_import_lines + WHERE import_job_id = %s + AND also_company_id = %s + AND order_draft_id IS NULL + AND queue_status <> 'invoiced' + ORDER BY id ASC + """, + (line.get("import_job_id"), also_company_id), + ) or [] + affected_line_ids = [int(row["id"]) for row in related_rows if row.get("id") is not None] + if not affected_line_ids: + affected_line_ids = [line_id] + + line_limit = max(len(affected_line_ids), 1) + matching_result = self.run_matching(import_job_id=None, line_ids=affected_line_ids, limit=line_limit) + validation_result = self.run_validation(import_job_id=None, line_ids=affected_line_ids, limit=line_limit) + + try: + approval_result = self.approve_lines_to_drafts( + import_job_id=None, + line_ids=affected_line_ids, + approved_by_user_id=updated_by_user_id, + ) + except HTTPException as exc: + if exc.status_code == 400 and exc.detail == "No ready-for-approval lines found": + approval_result = { + "approved_lines": 0, + "created_drafts": 0, + "draft_ids": [], + } + else: + raise + + refreshed_line = execute_query_single( + """ + SELECT + l.*, + c.name AS matched_customer_name + FROM also_import_lines l + LEFT JOIN customers c ON c.id = l.matched_customer_id + WHERE l.id = %s + """, + (line_id,), + ) + + return { + "success": True, + "line_id": line_id, + "import_job_id": int(line["import_job_id"]), + "also_company_id": also_company_id, + "customer_id": int(customer["id"]), + "customer_name": customer.get("name"), + "affected_line_ids": affected_line_ids, + "affected_line_count": len(affected_line_ids), + "mapping": mapping, + "matching_result": matching_result, + "validation_result": validation_result, + "approval_result": approval_result, + "line": refreshed_line, + } + + def manual_map_product_for_line( + self, + *, + line_id: int, + product_id: int, + notes: Optional[str], + updated_by_user_id: Optional[int], + ) -> Dict[str, Any]: + self._assert_enabled() + + line = execute_query_single( + """ + SELECT * + FROM also_import_lines + WHERE id = %s + """, + (line_id,), + ) + if not line: + raise HTTPException(status_code=404, detail="ALSO import line not found") + + material_number = _normalized_text(line.get("material_number")) + if not material_number: + raise HTTPException(status_code=400, detail="Line is missing material_number and cannot be mapped manually") + + vendor = _normalized_text(line.get("vendor")) or "ALSO" + product = execute_query_single( + """ + SELECT id, name + FROM products + WHERE id = %s + AND deleted_at IS NULL + """, + (product_id,), + ) + if not product: + raise HTTPException(status_code=404, detail="Product not found") + + mapping = self.upsert_product_mapping( + AlsoProductMappingUpsert( + material_number=material_number, + vendor=vendor, + hub_product_id=product_id, + product_name_snapshot=_normalized_text(line.get("product_name")) or None, + ) + ) + + related_rows = execute_query( + """ + SELECT id + FROM also_import_lines + WHERE import_job_id = %s + AND material_number = %s + AND LOWER(COALESCE(vendor, 'ALSO')) = LOWER(%s) + AND order_draft_id IS NULL + AND queue_status <> 'invoiced' + ORDER BY id ASC + """, + (line.get("import_job_id"), material_number, vendor), + ) or [] + affected_line_ids = [int(row["id"]) for row in related_rows if row.get("id") is not None] + if not affected_line_ids: + affected_line_ids = [line_id] + + line_limit = max(len(affected_line_ids), 1) + matching_result = self.run_matching(import_job_id=None, line_ids=affected_line_ids, limit=line_limit) + validation_result = self.run_validation(import_job_id=None, line_ids=affected_line_ids, limit=line_limit) + + try: + approval_result = self.approve_lines_to_drafts( + import_job_id=None, + line_ids=affected_line_ids, + approved_by_user_id=updated_by_user_id, + ) + except HTTPException as exc: + if exc.status_code == 400 and exc.detail == "No ready-for-approval lines found": + approval_result = { + "approved_lines": 0, + "created_drafts": 0, + "draft_ids": [], + } + else: + raise + + refreshed_line = execute_query_single( + """ + SELECT + l.*, + p.name AS matched_product_name + FROM also_import_lines l + LEFT JOIN products p ON p.id = l.matched_product_id + WHERE l.id = %s + """, + (line_id,), + ) + + return { + "success": True, + "line_id": line_id, + "import_job_id": int(line["import_job_id"]), + "material_number": material_number, + "vendor": vendor, + "product_id": int(product["id"]), + "product_name": product.get("name"), + "affected_line_ids": affected_line_ids, + "affected_line_count": len(affected_line_ids), + "mapping": mapping, + "notes": notes, + "matching_result": matching_result, + "validation_result": validation_result, + "approval_result": approval_result, + "line": refreshed_line, + } + def upsert_product_mapping(self, payload: AlsoProductMappingUpsert) -> Dict[str, Any]: self._assert_enabled() rows = execute_query( diff --git a/app/modules/also/models/schemas.py b/app/modules/also/models/schemas.py index 09d0fcd..b348f29 100644 --- a/app/modules/also/models/schemas.py +++ b/app/modules/also/models/schemas.py @@ -5,7 +5,7 @@ from typing import Any, Dict, List, Literal, Optional from pydantic import BaseModel, Field -ALSO_SOURCE_TYPES = Literal["api", "xml", "json_export", "xml_export", "csv"] +ALSO_SOURCE_TYPES = Literal["api", "xml", "json_export", "xml_export", "csv", "xlsx_export"] ALSO_QUEUE_STATUSES = Literal[ "new", "matching_products", @@ -97,6 +97,16 @@ class AlsoProductMappingUpsert(BaseModel): product_name_snapshot: Optional[str] = None +class AlsoManualCompanyMapRequest(BaseModel): + customer_id: int = Field(gt=0) + notes: Optional[str] = None + + +class AlsoManualProductMapRequest(BaseModel): + product_id: int = Field(gt=0) + notes: Optional[str] = None + + class AlsoImportJobResponse(BaseModel): id: int source_type: str @@ -119,6 +129,9 @@ class AlsoQueueLineResponse(BaseModel): product_name: Optional[str] = None vendor: Optional[str] = None total_price: Optional[Decimal] = None + cost_amount: Optional[Decimal] = None + effective_cost_amount: Optional[Decimal] = None + effective_margin_amount: Optional[Decimal] = None currency: Optional[str] = None matched_customer_id: Optional[int] = None matched_product_id: Optional[int] = None diff --git a/app/modules/bottom_bar/backend/public_router.py b/app/modules/bottom_bar/backend/public_router.py index aced919..493f75e 100644 --- a/app/modules/bottom_bar/backend/public_router.py +++ b/app/modules/bottom_bar/backend/public_router.py @@ -6,7 +6,7 @@ from typing import Optional from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect from app.core.auth_service import AuthService -from .service import get_active_timer, get_dashboard_status, get_notifications +from .service import get_active_timer, get_dashboard_status, get_notifications, get_user_messages_summary logger = logging.getLogger(__name__) @@ -78,11 +78,19 @@ async def bottom_bar_ws(websocket: WebSocket): initial_status = get_dashboard_status() initial_notifications = get_notifications(user_id, limit=20) + initial_messages = get_user_messages_summary(user_id, limit=20) await websocket.send_json({"event": "status_delta", "data": initial_status}) - await websocket.send_json({"event": "notification_delta", "data": initial_notifications}) + await websocket.send_json({ + "event": "notification_delta", + "data": { + "notifications": initial_notifications, + "messages": initial_messages, + }, + }) last_status_json = json.dumps(initial_status, sort_keys=True, default=str) last_notifications_json = json.dumps(initial_notifications, sort_keys=True, default=str) + last_messages_json = json.dumps(initial_messages, sort_keys=True, default=str) last_timer_elapsed = -1 status_tick = 0 @@ -98,6 +106,7 @@ async def bottom_bar_ws(websocket: WebSocket): if status_tick >= 5: status = get_dashboard_status() notifications = get_notifications(user_id, limit=20) + messages = get_user_messages_summary(user_id, limit=20) status_json = json.dumps(status, sort_keys=True, default=str) if status_json != last_status_json: @@ -105,9 +114,17 @@ async def bottom_bar_ws(websocket: WebSocket): last_status_json = status_json notifications_json = json.dumps(notifications, sort_keys=True, default=str) - if notifications_json != last_notifications_json: - await websocket.send_json({"event": "notification_delta", "data": notifications}) + messages_json = json.dumps(messages, sort_keys=True, default=str) + if notifications_json != last_notifications_json or messages_json != last_messages_json: + await websocket.send_json({ + "event": "notification_delta", + "data": { + "notifications": notifications, + "messages": messages, + }, + }) last_notifications_json = notifications_json + last_messages_json = messages_json status_tick = 0 diff --git a/app/modules/bottom_bar/backend/router.py b/app/modules/bottom_bar/backend/router.py index 3bfc2b3..d52aed8 100644 --- a/app/modules/bottom_bar/backend/router.py +++ b/app/modules/bottom_bar/backend/router.py @@ -8,7 +8,14 @@ from app.core.auth_service import AuthService from app.core.auth_dependencies import get_current_user from app.core.database import execute_query, execute_query_single, execute_update -from .service import build_bottom_bar_state, get_own_timer_snapshot, get_unassigned_open_cases +from .service import ( + acknowledge_message, + build_bottom_bar_state, + ensure_bottom_bar_messages_schema, + get_own_timer_snapshot, + get_unassigned_open_cases, + mark_user_messages_read, +) router = APIRouter() logger = logging.getLogger(__name__) @@ -58,6 +65,20 @@ class NoteToCustomerPayload(BaseModel): mode: str = "append" +class BottomBarMessageCreatePayload(BaseModel): + message: str + recipient_user_id: Optional[int] = None + requires_manual_ack: bool = False + + +class BottomBarMessageReadPayload(BaseModel): + partner_user_id: Optional[int] = None + + +class BottomBarMessageAcknowledgePayload(BaseModel): + message_id: int + + def _ensure_user_notes_schema() -> None: global _USER_NOTES_SCHEMA_READY if _USER_NOTES_SCHEMA_READY: @@ -521,6 +542,88 @@ async def get_own_timers( return get_own_timer_snapshot(int(current_user_id), paused_limit=paused_limit) +@router.post("/messages") +async def send_bottom_bar_message( + payload: BottomBarMessageCreatePayload, + current_user: dict = Depends(get_current_user), +): + current_user_id = current_user.get("id") + if current_user_id is None: + raise HTTPException(status_code=401, detail="Not authenticated") + + ensure_bottom_bar_messages_schema() + + message_text = str(payload.message or "").strip() + if not message_text: + raise HTTPException(status_code=400, detail="Besked må ikke være tom") + if len(message_text) > 2000: + raise HTTPException(status_code=400, detail="Besked er for lang") + + recipient_user_id = payload.recipient_user_id + if recipient_user_id is not None: + recipient_user_id = int(recipient_user_id) + _ensure_user_exists(recipient_user_id) + if recipient_user_id == int(current_user_id): + raise HTTPException(status_code=400, detail="Du kan ikke sende en besked til dig selv") + + row = execute_query_single( + """ + INSERT INTO bottom_bar_messages (sender_user_id, recipient_user_id, message_text, requires_manual_ack) + VALUES (%s, %s, %s, %s) + RETURNING id, sender_user_id, recipient_user_id, message_text, requires_manual_ack, created_at + """, + (int(current_user_id), recipient_user_id, message_text, bool(payload.requires_manual_ack)), + ) or {} + + return { + "message": "Besked sendt", + "item": { + "id": row.get("id"), + "from": _resolve_current_user_display_name(current_user), + "to": "Alle på vagt" if recipient_user_id is None else f"Bruger #{recipient_user_id}", + "text": row.get("message_text") or message_text, + "requires_manual_ack": bool(row.get("requires_manual_ack")), + "created_at": row.get("created_at"), + "is_own": True, + "is_unread": False, + "is_acknowledged": False, + }, + } + + +@router.post("/messages/read") +async def mark_bottom_bar_messages_read( + payload: Optional[BottomBarMessageReadPayload] = None, + current_user: dict = Depends(get_current_user), +): + current_user_id = current_user.get("id") + if current_user_id is None: + raise HTTPException(status_code=401, detail="Not authenticated") + + return { + "ok": True, + "updated": mark_user_messages_read( + int(current_user_id), + partner_user_id=payload.partner_user_id if payload else None, + ), + } + + +@router.post("/messages/acknowledge") +async def acknowledge_bottom_bar_message( + payload: BottomBarMessageAcknowledgePayload, + current_user: dict = Depends(get_current_user), +): + current_user_id = current_user.get("id") + if current_user_id is None: + raise HTTPException(status_code=401, detail="Not authenticated") + + if not acknowledge_message(int(current_user_id), int(payload.message_id)): + raise HTTPException(status_code=404, detail="Besked ikke fundet eller allerede bekræftet") + + return {"ok": True, "message_id": int(payload.message_id)} + + @router.get("/boss/unassigned-cases") async def list_unassigned_open_cases( limit: int = Query(default=25, ge=1, le=100), diff --git a/app/modules/bottom_bar/backend/service.py b/app/modules/bottom_bar/backend/service.py index 0c70137..2d9b98a 100644 --- a/app/modules/bottom_bar/backend/service.py +++ b/app/modules/bottom_bar/backend/service.py @@ -9,6 +9,7 @@ logger = logging.getLogger(__name__) CLOSED_CASE_STATUSES = ("lukket", "løst", "closed", "resolved") URGENT_PRIORITIES = ("urgent", "high", "kritisk", "critical") +_BOTTOM_BAR_MESSAGES_SCHEMA_READY = False def _safe_count(row: Optional[dict], key: str = "count") -> int: @@ -65,6 +66,203 @@ def _table_columns(table_name: str) -> List[str]: return [str(r.get("column_name") or "").strip().lower() for r in rows if r.get("column_name")] +def ensure_bottom_bar_messages_schema() -> None: + global _BOTTOM_BAR_MESSAGES_SCHEMA_READY + if _BOTTOM_BAR_MESSAGES_SCHEMA_READY: + return + + exists = execute_query_single("SELECT to_regclass('public.bottom_bar_messages') AS table_name") or {} + table_exists = bool(exists.get("table_name")) + + if not table_exists: + execute_query( + """ + CREATE TABLE IF NOT EXISTS bottom_bar_messages ( + id SERIAL PRIMARY KEY, + sender_user_id INTEGER NOT NULL REFERENCES users(user_id) ON DELETE CASCADE, + recipient_user_id INTEGER NULL REFERENCES users(user_id) ON DELETE CASCADE, + message_text TEXT NOT NULL, + requires_manual_ack BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + read_at TIMESTAMP NULL + ) + """ + ) + logger.warning("⚠️ bottom_bar_messages table was missing and has been created automatically") + else: + columns = set(_table_columns("bottom_bar_messages")) + if "requires_manual_ack" not in columns: + execute_query( + """ + ALTER TABLE bottom_bar_messages + ADD COLUMN requires_manual_ack BOOLEAN NOT NULL DEFAULT FALSE + """ + ) + logger.warning("⚠️ Added requires_manual_ack to bottom_bar_messages") + + execute_query( + """ + CREATE INDEX IF NOT EXISTS idx_bottom_bar_messages_recipient_created + ON bottom_bar_messages (recipient_user_id, created_at DESC) + """ + ) + execute_query( + """ + CREATE INDEX IF NOT EXISTS idx_bottom_bar_messages_sender_created + ON bottom_bar_messages (sender_user_id, created_at DESC) + """ + ) + execute_query( + """ + CREATE INDEX IF NOT EXISTS idx_bottom_bar_messages_unread + ON bottom_bar_messages (recipient_user_id, read_at, created_at DESC) + """ + ) + execute_query( + """ + CREATE INDEX IF NOT EXISTS idx_bottom_bar_messages_manual_ack + ON bottom_bar_messages (recipient_user_id, requires_manual_ack, read_at, created_at DESC) + """ + ) + + _BOTTOM_BAR_MESSAGES_SCHEMA_READY = True + + +def get_user_messages_summary(user_id: Optional[int], limit: int = 20) -> Dict[str, Any]: + if user_id is None: + return {"count": 0, "list": []} + + ensure_bottom_bar_messages_schema() + safe_limit = max(1, min(int(limit or 20), 100)) + + rows = execute_query( + """ + SELECT + m.id, + m.sender_user_id, + m.recipient_user_id, + m.message_text, + m.requires_manual_ack, + m.created_at, + m.read_at, + COALESCE(NULLIF(sender.full_name, ''), sender.username, ('Bruger #' || sender.user_id::text)) AS sender_name, + COALESCE(NULLIF(recipient.full_name, ''), recipient.username, ('Bruger #' || recipient.user_id::text)) AS recipient_name + FROM bottom_bar_messages m + JOIN users sender ON sender.user_id = m.sender_user_id + LEFT JOIN users recipient ON recipient.user_id = m.recipient_user_id + WHERE m.sender_user_id = %s + OR m.recipient_user_id = %s + OR (m.recipient_user_id IS NULL AND EXISTS ( + SELECT 1 + FROM users u + WHERE u.user_id = %s + AND COALESCE(u.is_active, TRUE) = TRUE + )) + ORDER BY m.created_at DESC, m.id DESC + LIMIT %s + """, + (int(user_id), int(user_id), int(user_id), safe_limit), + ) or [] + + unread_row = execute_query_single( + """ + SELECT COUNT(*) AS count + FROM bottom_bar_messages + WHERE read_at IS NULL + AND sender_user_id <> %s + AND ( + recipient_user_id = %s + OR recipient_user_id IS NULL + ) + """, + (int(user_id), int(user_id)), + ) or {} + + unread_count = _safe_count(unread_row) + + items = [] + for row in reversed(rows): + recipient_user_id = row.get("recipient_user_id") + recipient_name = "Alle på vagt" if recipient_user_id is None else (row.get("recipient_name") or f"Bruger #{recipient_user_id}") + items.append( + { + "id": row.get("id"), + "sender_user_id": row.get("sender_user_id"), + "recipient_user_id": row.get("recipient_user_id"), + "from": row.get("sender_name") or "Ukendt", + "to": recipient_name, + "text": row.get("message_text") or "", + "requires_manual_ack": bool(row.get("requires_manual_ack")), + "created_at": row.get("created_at").isoformat() if row.get("created_at") else None, + "is_own": int(row.get("sender_user_id") or 0) == int(user_id), + "is_unread": row.get("read_at") is None and int(row.get("sender_user_id") or 0) != int(user_id), + "is_acknowledged": row.get("read_at") is not None, + } + ) + + return { + "count": unread_count, + "list": items, + } + + +def mark_user_messages_read(user_id: Optional[int], partner_user_id: Optional[int] = None) -> int: + if user_id is None: + return 0 + + ensure_bottom_bar_messages_schema() + params: List[Any] = [int(user_id)] + if partner_user_id is not None and int(partner_user_id) > 0: + where_clause = """ + recipient_user_id = %s + AND read_at IS NULL + AND COALESCE(requires_manual_ack, FALSE) = FALSE + AND sender_user_id <> %s + """ + params.append(int(user_id)) + where_clause += " AND sender_user_id = %s" + params.append(int(partner_user_id)) + else: + where_clause = """ + (recipient_user_id = %s OR recipient_user_id IS NULL) + AND read_at IS NULL + AND COALESCE(requires_manual_ack, FALSE) = FALSE + AND sender_user_id <> %s + """ + params.append(int(user_id)) + + row = execute_query_single( + f""" + UPDATE bottom_bar_messages + SET read_at = CURRENT_TIMESTAMP + WHERE {where_clause} + RETURNING COUNT(*) OVER() AS affected_count + """, + tuple(params), + ) or {} + + return int(row.get("affected_count") or 0) + + +def acknowledge_message(user_id: Optional[int], message_id: int) -> bool: + if user_id is None or int(message_id or 0) <= 0: + return False + + ensure_bottom_bar_messages_schema() + row = execute_query_single( + """ + UPDATE bottom_bar_messages + SET read_at = CURRENT_TIMESTAMP + WHERE id = %s + AND recipient_user_id = %s + AND read_at IS NULL + RETURNING id + """, + (int(message_id), int(user_id)), + ) or {} + return bool(row.get("id")) + + def get_drift_status(limit: int = 5) -> Dict[str, Any]: rel_row = execute_query_single( """ @@ -731,6 +929,7 @@ def build_bottom_bar_state( timer = get_active_timer(user_id) own_timers = get_own_timer_snapshot(user_id, paused_limit=10) notifications = get_notifications(user_id, limit=10) + messages_summary = get_user_messages_summary(user_id, limit=20) drift_status = get_drift_status(limit=5) unassigned_open_cases = get_unassigned_open_cases(limit=8) recent_cases = _get_recent_cases(user_id, limit=10) @@ -771,13 +970,6 @@ def build_bottom_bar_state( } ) - messages = [ - { - "from": "System", - "text": f"{notifications.get('count', 0)} aktive notifikationer", - } - ] - tasks = [] for n in (notifications.get("items") or [])[:5]: tasks.append( @@ -956,8 +1148,8 @@ def build_bottom_bar_state( "list": [], }, "messages": { - "count": len(messages), - "list": messages, + "count": int(messages_summary.get("count") or 0), + "list": messages_summary.get("list") or [], }, "tasks": { "count": len(tasks), diff --git a/app/modules/drift/backend/router.py b/app/modules/drift/backend/router.py index 27e0b92..6e82187 100644 --- a/app/modules/drift/backend/router.py +++ b/app/modules/drift/backend/router.py @@ -32,6 +32,7 @@ class DriftCustomerMappingPayload(BaseModel): monitor_key: Optional[str] = None monitor_name: Optional[str] = None customer_id: int + source: Optional[str] = None class DriftBlacklistPayload(BaseModel): @@ -163,10 +164,19 @@ def _row_to_event(row: Dict[str, Any]) -> Dict[str, Any]: started = row.get("started_at") updated = row.get("updated_at") resolved = row.get("resolved_at") - raw = row.get("raw_json") if isinstance(row.get("raw_json"), dict) else {} + raw_payload = row.get("raw_json") + if isinstance(raw_payload, str): + try: + raw_payload = json.loads(raw_payload) + except (TypeError, ValueError): + raw_payload = {} + raw = raw_payload if isinstance(raw_payload, dict) else {} raw_item = raw.get("raw_item") if isinstance(raw.get("raw_item"), dict) else {} raw_overview = raw_item.get("overview") if isinstance(raw_item.get("overview"), dict) else {} nested_overview = raw.get("overview") if isinstance(raw.get("overview"), dict) else {} + identification = raw_item.get("identification") if isinstance(raw_item.get("identification"), dict) else {} + if not isinstance(identification, dict): + identification = raw.get("identification") if isinstance(raw.get("identification"), dict) else {} last_seen = ( raw.get("last_seen") or raw_overview.get("lastSeen") @@ -185,6 +195,83 @@ def _row_to_event(row: Dict[str, Any]) -> Dict[str, Any]: source_event_id = row.get("source_event_id") or "" monitor_key = source_event_id[5:] if source_event_id.startswith("kuma-") else source_event_id + ip_value = None + for candidate in ( + raw.get("ip"), + raw.get("ip_address"), + raw.get("ipAddress"), + raw_item.get("ip"), + raw_item.get("ip_address"), + raw_item.get("ipAddress"), + raw_item.get("overview", {}).get("ip") if isinstance(raw_item.get("overview"), dict) else None, + raw_item.get("overview", {}).get("ip_address") if isinstance(raw_item.get("overview"), dict) else None, + raw_item.get("overview", {}).get("ipAddress") if isinstance(raw_item.get("overview"), dict) else None, + identification.get("ip"), + identification.get("ip_address"), + identification.get("ipAddress"), + identification.get("address"), + raw_item.get("address"), + raw_item.get("addresses"), + raw_item.get("network"), + ): + if candidate is None: + continue + value = str(candidate).strip() + if value: + ip_value = value + break + + site_client_value = None + def _normalize_text_value(value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, dict): + for key in ("name", "title", "client_name", "clientName", "site_name", "siteName", "value"): + nested = value.get(key) + if isinstance(nested, str) and nested.strip(): + return nested.strip() + return None + if isinstance(value, list): + for item in value: + normalized = _normalize_text_value(item) + if normalized: + return normalized + return None + if isinstance(value, (str, int, float, bool)): + text = str(value).strip() + return text or None + return None + + for candidate in ( + raw.get("site_client_name"), + raw.get("client_name"), + raw.get("clientName"), + raw.get("client"), + raw_item.get("client_name"), + raw_item.get("clientName"), + raw_item.get("client"), + raw_item.get("site_client_name"), + raw_item.get("site_name"), + raw_item.get("site"), + raw_item.get("siteName"), + raw.get("site"), + raw.get("site_name"), + raw.get("siteName"), + identification.get("site_client_name"), + identification.get("client_name"), + identification.get("clientName"), + identification.get("client"), + identification.get("site"), + identification.get("site_name"), + identification.get("siteName"), + row.get("site_name"), + row.get("customer_name"), + ): + normalized = _normalize_text_value(candidate) + if normalized: + site_client_value = normalized + break + raw_source_link = raw.get("source_link") or raw.get("device_link") if not raw_source_link and isinstance(raw.get("raw_item"), dict): raw_source_link = _extract_device_link(raw.get("raw_item"), None, source_event_id[5:] if source_event_id.startswith(("uisp-", "kuma-")) else None) @@ -199,8 +286,10 @@ def _row_to_event(row: Dict[str, Any]) -> Dict[str, Any]: "customer": row.get("customer_name"), "customer_id": raw.get("customer_id"), "site": row.get("site_name"), + "site_client_name": site_client_value or row.get("site_name") or row.get("customer_name"), "device": row.get("device_name"), "service": row.get("service_name"), + "ip": ip_value, "source_link": raw_source_link, "device_link": raw_source_link, "message": row.get("message"), @@ -1007,6 +1096,47 @@ def _build_events_from_uisp_payload(payload: Any, source_id: Optional[int], base or str(external_id) ) site = site_obj.get("name") or item.get("site_name") or item.get("site") or item.get("siteName") or "UISP" + ip_value = None + for candidate in ( + item.get("ip"), + item.get("ip_address"), + item.get("ipAddress"), + identification.get("ip"), + identification.get("ip_address"), + identification.get("ipAddress"), + overview.get("ip"), + overview.get("ip_address"), + overview.get("ipAddress"), + item.get("overview", {}).get("ip") if isinstance(item.get("overview"), dict) else None, + item.get("overview", {}).get("ip_address") if isinstance(item.get("overview"), dict) else None, + item.get("overview", {}).get("ipAddress") if isinstance(item.get("overview"), dict) else None, + ): + if candidate is None: + continue + value = str(candidate).strip() + if value: + ip_value = value + break + + site_client_name = None + for candidate in ( + item.get("site_client_name"), + item.get("client_name"), + item.get("clientName"), + item.get("client"), + identification.get("site_client_name"), + identification.get("client_name"), + identification.get("clientName"), + identification.get("client"), + site_obj.get("name"), + site, + ): + if candidate is None: + continue + value = str(candidate).strip() + if value: + site_client_name = value + break state_value = ( overview.get("status") or identification.get("status") @@ -1061,6 +1191,9 @@ def _build_events_from_uisp_payload(payload: Any, source_id: Optional[int], base "overview_status": overview.get("status"), "last_seen": last_seen, "site": site, + "site_client_name": site_client_name or site, + "ip": ip_value, + "ip_address": ip_value, "raw_item": item, }, } @@ -1505,14 +1638,21 @@ async def list_customer_drift_events(customer_id: int, limit: int = Query(defaul @router.put("/drift/customer-mappings") async def upsert_customer_mapping(payload: DriftCustomerMappingPayload): _ensure_schema() - source = _ensure_source() + monitor_key = str(payload.monitor_key or "").strip() + monitor_name = str(payload.monitor_name or "").strip() + requested_source = str(payload.source or "").strip().lower() + if requested_source in {"uisp", "uptime-kuma", "uptime_kuma", "kuma"}: + connector_type = "uisp" if requested_source == "uisp" else "uptime-kuma" + elif monitor_key.startswith("uisp-") or monitor_name.startswith("uisp-"): + connector_type = "uisp" + else: + connector_type = "uptime-kuma" + + source = _ensure_source(connector_type) source_id = source.get("id") if not source_id: raise HTTPException(status_code=500, detail="Drift source kunne ikke initialiseres") - - monitor_key = str(payload.monitor_key or "").strip() - monitor_name = str(payload.monitor_name or "").strip() if not monitor_key and not monitor_name: raise HTTPException(status_code=400, detail="monitor_key eller monitor_name er paakraevet") diff --git a/app/modules/drift/templates/drift.html b/app/modules/drift/templates/drift.html index f871f69..a0b1cad 100644 --- a/app/modules/drift/templates/drift.html +++ b/app/modules/drift/templates/drift.html @@ -98,6 +98,7 @@ + @@ -353,7 +357,7 @@ async function loadDriftEvents() { const visibleEvents = (Array.isArray(events) ? events : []).filter(event => !isDriftEventBlacklisted(event, blacklist)); syncBottomDriftCountFromVisibleEvents(visibleEvents); if (!visibleEvents.length) { - body.innerHTML = 'Ingen hændelser fundet.'; + body.innerHTML = 'Ingen hændelser fundet.'; return; } body.innerHTML = visibleEvents.map(event => ` @@ -369,7 +373,7 @@ async function loadDriftEvents() {
${event.customer || '-'} - ${event.monitor_key ? `` : ''} + ${event.monitor_key ? `` : ''}
@@ -378,6 +382,8 @@ async function loadDriftEvents() { ${(event.device_link || event.source_link) ? `` : ''} + ${event.ip || '-'} + ${event.site_client_name || event.site || event.customer || '-'} ${event.started ? new Date(event.started).toLocaleString('da-DK') : '-'} ${event.last_seen ? new Date(event.last_seen).toLocaleString('da-DK') : '-'} ${event.duration_minutes !== null && event.duration_minutes !== undefined ? `${event.duration_minutes} min` : '-'} @@ -395,7 +401,7 @@ async function loadDriftEvents() { } catch (e) { console.error(e); syncBottomDriftCountFromVisibleEvents([]); - body.innerHTML = 'Kunne ikke hente drift-data.'; + body.innerHTML = 'Kunne ikke hente drift-data.'; } } @@ -422,6 +428,11 @@ async function acknowledgeDriftEvent(eventId) { return; } + const statusFilter = document.getElementById('drift-status-filter'); + if (statusFilter && (statusFilter.value === '' || statusFilter.value === 'active')) { + statusFilter.value = 'acknowledged'; + } + await loadDriftSummary(); await loadDriftEvents(); } @@ -451,6 +462,7 @@ async function blacklistDriftEvent(eventId, deviceName = '') { async function mapDriftCustomer(monitorKey, monitorName) { const customerId = Number(document.getElementById('driftMapCustomerSelect')?.value || 0); + const source = document.getElementById('driftMapSource')?.value || 'uptime-kuma'; if (!Number.isInteger(customerId) || customerId <= 0) { alert('Vælg en kunde'); return false; @@ -460,7 +472,7 @@ async function mapDriftCustomer(monitorKey, monitorName) { method: 'PUT', credentials: 'include', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ monitor_key: monitorKey, monitor_name: monitorName, customer_id: customerId }) + body: JSON.stringify({ monitor_key: monitorKey, monitor_name: monitorName, customer_id: customerId, source }) }); const data = await res.json().catch(() => ({})); if (!res.ok) { @@ -473,10 +485,11 @@ async function mapDriftCustomer(monitorKey, monitorName) { return true; } -async function openMapCustomerModal(monitorKey, monitorName) { +async function openMapCustomerModal(monitorKey, monitorName, source = 'uptime-kuma') { await loadDriftCustomers(); document.getElementById('driftMapMonitorKey').value = monitorKey || ''; document.getElementById('driftMapMonitorName').value = monitorName || ''; + document.getElementById('driftMapSource').value = source || 'uptime-kuma'; document.getElementById('driftMapModalMonitorText').textContent = `Monitor: ${monitorName || monitorKey}`; renderCustomerSelectOptions(document.getElementById('driftMapCustomerSelect')); const modalEl = document.getElementById('driftMapModal'); @@ -555,7 +568,7 @@ async function saveBulkMapping(monitorKey, monitorName, selectId) { method: 'PUT', credentials: 'include', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ monitor_key: monitorKey, monitor_name: monitorName, customer_id: customerId }) + body: JSON.stringify({ monitor_key: monitorKey, monitor_name: monitorName, customer_id: customerId, source: 'uptime-kuma' }) }); const data = await res.json().catch(() => ({})); if (!res.ok) { diff --git a/app/modules/search/backend/router.py b/app/modules/search/backend/router.py index 9f07392..e898d7e 100644 --- a/app/modules/search/backend/router.py +++ b/app/modules/search/backend/router.py @@ -110,3 +110,32 @@ async def search_locations(q: str = Query(..., min_length=2)): term = f"%{q}%" results = execute_query(sql, (term, term, term)) return results + + +@router.get("/search/products") +async def search_products(q: str = Query(..., min_length=2)): + """ + Autocomplete search for products. + Returns list of {id, name, sku_internal, supplier_sku, manufacturer} + """ + sql = """ + SELECT + id, + name, + sku_internal, + supplier_sku, + manufacturer + FROM products + WHERE ( + name ILIKE %s + OR COALESCE(sku_internal, '') ILIKE %s + OR COALESCE(supplier_sku, '') ILIKE %s + OR COALESCE(manufacturer, '') ILIKE %s + ) + AND deleted_at IS NULL + ORDER BY name ASC + LIMIT 20 + """ + term = f"%{q}%" + results = execute_query(sql, (term, term, term, term)) + return results diff --git a/app/services/customer_consistency.py b/app/services/customer_consistency.py index b3640f0..16fe712 100644 --- a/app/services/customer_consistency.py +++ b/app/services/customer_consistency.py @@ -5,7 +5,7 @@ Compares customer data across BMC Hub, vTiger Cloud, and e-conomic import logging import asyncio from typing import Dict, List, Optional, Tuple, Any -from app.core.database import execute_query_single, execute_update +from app.core.database import execute_query, execute_query_single, execute_update, execute_insert from app.services.vtiger_service import VTigerService from app.services.economic_service import EconomicService from app.core.config import settings @@ -34,6 +34,111 @@ class CustomerConsistencyService: def __init__(self): self.vtiger = VTigerService() self.economic = EconomicService() + + @staticmethod + def _clean_contact_value(value: Any) -> Optional[str]: + if value is None: + return None + text = str(value).strip() + return text or None + + @classmethod + def _normalize_contact(cls, source: str, row: Dict[str, Any]) -> Dict[str, Any]: + first_name = cls._clean_contact_value( + row.get('first_name') if source == 'hub' else row.get('firstname') + ) + last_name = cls._clean_contact_value( + row.get('last_name') if source == 'hub' else row.get('lastname') + ) + email = cls._clean_contact_value(row.get('email')) + phone = cls._clean_contact_value(row.get('phone')) + mobile = cls._clean_contact_value(row.get('mobile')) + title = cls._clean_contact_value(row.get('title')) + department = cls._clean_contact_value(row.get('department')) + vtiger_id = cls._clean_contact_value( + row.get('vtiger_id') if source == 'hub' else row.get('id') + ) + display_name = " ".join(part for part in [first_name, last_name] if part).strip() + if not display_name: + display_name = email or phone or mobile or vtiger_id or 'Ukendt kontakt' + + return { + 'id': row.get('id'), + 'vtiger_id': vtiger_id, + 'first_name': first_name, + 'last_name': last_name, + 'email': email, + 'phone': phone, + 'mobile': mobile, + 'title': title, + 'department': department, + 'display_name': display_name, + 'is_primary': bool(row.get('is_primary')) if source == 'hub' else False, + 'role': cls._clean_contact_value(row.get('role')) if source == 'hub' else None, + 'customer_count': row.get('customer_count', 0) or 0, + } + + @staticmethod + def _contact_match_key(contact: Dict[str, Any]) -> str: + vtiger_id = str(contact.get('vtiger_id') or '').strip().lower() + if vtiger_id: + return f"vtiger:{vtiger_id}" + + email = str(contact.get('email') or '').strip().lower() + if email: + return f"email:{email}" + + first_name = str(contact.get('first_name') or '').strip().lower() + last_name = str(contact.get('last_name') or '').strip().lower() + phone = str(contact.get('phone') or '').strip().lower() + mobile = str(contact.get('mobile') or '').strip().lower() + return f"name:{first_name}|{last_name}|{phone}|{mobile}" + + async def fetch_hub_contacts(self, customer_id: int) -> List[Dict[str, Any]]: + query = """ + SELECT + c.*, + cc.is_primary, + cc.role, + ( + SELECT COUNT(*) + FROM contact_companies cc2 + WHERE cc2.contact_id = c.id + ) AS customer_count + FROM contacts c + JOIN contact_companies cc ON cc.contact_id = c.id + WHERE cc.customer_id = %s + AND c.is_active = TRUE + ORDER BY cc.is_primary DESC, c.first_name, c.last_name, c.id + """ + rows = await asyncio.to_thread(execute_query, query, (customer_id,)) + return [self._normalize_contact('hub', row) for row in (rows or [])] + + async def fetch_vtiger_contacts(self, vtiger_customer_id: Optional[str]) -> List[Dict[str, Any]]: + if not vtiger_customer_id or not settings.VTIGER_URL: + return [] + + safe_customer_id = self.vtiger._sanitize_vtiger_id(vtiger_customer_id) + if not safe_customer_id: + return [] + + rows: List[Dict[str, Any]] = [] + seen_ids = set() + + # vTiger installations are inconsistent about which relation field is exposed on Contacts. + relation_fields = ("account_id", "accountid", "parent_id", "account") + for field_name in relation_fields: + query = f"SELECT * FROM Contacts WHERE {field_name}='{safe_customer_id}';" + result = await self.vtiger.query(query) + for row in (result or []): + row_id = str(row.get("id") or "").strip() + dedupe_key = row_id or f"{row.get('email')}|{row.get('firstname')}|{row.get('lastname')}" + if dedupe_key in seen_ids: + continue + seen_ids.add(dedupe_key) + rows.append(row) + + return [self._normalize_contact('vtiger', row) for row in rows] @staticmethod def normalize_value(value: Any) -> Optional[str]: @@ -57,7 +162,7 @@ class CustomerConsistencyService: # Lowercase for case-insensitive comparison return str_value.lower() - async def fetch_all_data(self, customer_id: int) -> Dict[str, Optional[Dict[str, Any]]]: + async def fetch_all_data(self, customer_id: int) -> Dict[str, Any]: """ Fetch customer data from all three systems in parallel @@ -81,10 +186,13 @@ class CustomerConsistencyService: # Prepare async tasks for vTiger and e-conomic vtiger_task = None economic_task = None + hub_contacts_task = self.fetch_hub_contacts(customer_id) + vtiger_contacts_task = None # Fetch vTiger data if we have an ID and vTiger is configured if hub_data.get('vtiger_id') and settings.VTIGER_URL: vtiger_task = self.vtiger.get_account_by_id(hub_data['vtiger_id']) + vtiger_contacts_task = self.fetch_vtiger_contacts(hub_data['vtiger_id']) # Fetch e-conomic data if we have a customer number and e-conomic is configured if hub_data.get('economic_customer_number') and settings.ECONOMIC_APP_SECRET_TOKEN: @@ -96,6 +204,9 @@ class CustomerConsistencyService: tasks['vtiger'] = vtiger_task if economic_task: tasks['economic'] = economic_task + tasks['hub_contacts'] = hub_contacts_task + if vtiger_contacts_task: + tasks['vtiger_contacts'] = vtiger_contacts_task results = {} if tasks: @@ -115,7 +226,9 @@ class CustomerConsistencyService: return { 'hub': hub_data, 'vtiger': results.get('vtiger'), - 'economic': results.get('economic') + 'economic': results.get('economic'), + 'hub_contacts': results.get('hub_contacts') or [], + 'vtiger_contacts': results.get('vtiger_contacts') or [], } @classmethod @@ -173,6 +286,56 @@ class CustomerConsistencyService: } return discrepancies + + @classmethod + def compare_contacts(cls, all_data: Dict[str, Any]) -> List[Dict[str, Any]]: + hub_contacts = all_data.get('hub_contacts') or [] + vtiger_contacts = all_data.get('vtiger_contacts') or [] + + hub_by_key = {cls._contact_match_key(contact): contact for contact in hub_contacts} + discrepancies: List[Dict[str, Any]] = [] + + for vtiger_contact in vtiger_contacts: + match_key = cls._contact_match_key(vtiger_contact) + hub_contact = hub_by_key.get(match_key) + if not hub_contact: + discrepancies.append({ + 'match_key': match_key, + 'action': 'create_or_link', + 'reason': 'Kontakten findes i vTiger men ikke på denne kunde i Hub', + 'hub': None, + 'vtiger': vtiger_contact, + 'selectable': True, + }) + continue + + changed_fields = [] + for field in ('first_name', 'last_name', 'email', 'phone', 'mobile', 'title', 'department'): + if cls.normalize_value(hub_contact.get(field)) != cls.normalize_value(vtiger_contact.get(field)): + changed_fields.append(field) + + if changed_fields: + discrepancies.append({ + 'match_key': match_key, + 'action': 'update_hub', + 'reason': 'Kontakt findes begge steder men har feltforskelle', + 'hub': hub_contact, + 'vtiger': vtiger_contact, + 'changed_fields': changed_fields, + 'selectable': True, + }) + else: + discrepancies.append({ + 'match_key': match_key, + 'action': 'matched', + 'reason': 'Kontakt findes allerede i Hub og matcher vTiger', + 'hub': hub_contact, + 'vtiger': vtiger_contact, + 'changed_fields': [], + 'selectable': False, + }) + + return discrepancies async def sync_field( self, @@ -198,7 +361,7 @@ class CustomerConsistencyService: if field_name not in self.FIELD_MAP: raise ValueError(f"Unknown field: {field_name}") - vtiger_field, economic_field = self.FIELD_MAP[field_name] + _, economic_field = self.FIELD_MAP[field_name] # Fetch Hub data to get mapping IDs hub_query = "SELECT * FROM customers WHERE id = %s" @@ -222,22 +385,8 @@ class CustomerConsistencyService: else: results['hub'] = True # Already correct - # Update vTiger if enabled and not the source - if settings.VTIGER_SYNC_ENABLED and source_system != 'vtiger' and hub_data.get('vtiger_id'): - try: - update_data = {vtiger_field: source_value} - success = await self.vtiger.update_account(hub_data['vtiger_id'], update_data) - if success: - results['vtiger'] = True - logger.info(f"✅ vTiger {vtiger_field} updated") - else: - results['vtiger'] = False - logger.error(f"❌ vTiger update failed - API returned False") - except Exception as e: - logger.error(f"❌ Failed to update vTiger: {e}") - results['vtiger'] = False - else: - results['vtiger'] = True # Not applicable or already correct + # vTiger is read-only for this workflow. + results['vtiger'] = True # Update e-conomic if enabled and not the source if settings.ECONOMIC_SYNC_ENABLED and source_system != 'economic' and hub_data.get('economic_customer_number'): @@ -258,5 +407,130 @@ class CustomerConsistencyService: results['economic'] = False else: results['economic'] = True # Not applicable or already correct - + return results + + async def sync_vtiger_contacts_to_hub( + self, + customer_id: int, + selected_match_keys: List[str], + ) -> Dict[str, Any]: + if not selected_match_keys: + return {"selected": 0, "created": 0, "updated": 0, "linked": 0, "skipped": 0} + + all_data = await self.fetch_all_data(customer_id) + discrepancies = self.compare_contacts(all_data) + selected = {str(item or '').strip() for item in selected_match_keys if str(item or '').strip()} + selected_rows = [row for row in discrepancies if row.get('match_key') in selected] + + stats = {"selected": len(selected_rows), "created": 0, "updated": 0, "linked": 0, "skipped": 0} + + for row in selected_rows: + vtiger_contact = row.get('vtiger') or {} + hub_contact = row.get('hub') + if not vtiger_contact: + stats["skipped"] += 1 + continue + + contact_id = hub_contact.get('id') if hub_contact else None + if not contact_id: + vtiger_id = vtiger_contact.get("vtiger_id") + email = vtiger_contact.get("email") + if vtiger_id: + existing_global = await asyncio.to_thread( + execute_query_single, + "SELECT id FROM contacts WHERE vtiger_id = %s LIMIT 1", + (vtiger_id,), + ) + contact_id = existing_global.get("id") if existing_global else None + if not contact_id and email: + existing_global = await asyncio.to_thread( + execute_query_single, + "SELECT id FROM contacts WHERE LOWER(COALESCE(email, '')) = %s LIMIT 1", + (str(email).strip().lower(),), + ) + contact_id = existing_global.get("id") if existing_global else None + + if contact_id: + update_fields = { + "first_name": vtiger_contact.get("first_name"), + "last_name": vtiger_contact.get("last_name"), + "email": vtiger_contact.get("email"), + "phone": vtiger_contact.get("phone"), + "mobile": vtiger_contact.get("mobile"), + "title": vtiger_contact.get("title"), + "department": vtiger_contact.get("department"), + "vtiger_id": vtiger_contact.get("vtiger_id"), + } + await asyncio.to_thread( + execute_update, + """ + UPDATE contacts + SET first_name = %s, + last_name = %s, + email = %s, + phone = %s, + mobile = %s, + title = %s, + department = %s, + vtiger_id = COALESCE(%s, vtiger_id) + WHERE id = %s + """, + ( + update_fields["first_name"], + update_fields["last_name"], + update_fields["email"], + update_fields["phone"], + update_fields["mobile"], + update_fields["title"], + update_fields["department"], + update_fields["vtiger_id"], + contact_id, + ), + ) + stats["updated"] += 1 + else: + contact_id = await asyncio.to_thread( + execute_insert, + """ + INSERT INTO contacts ( + first_name, last_name, email, phone, mobile, title, department, vtiger_id + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + RETURNING id + """, + ( + vtiger_contact.get("first_name"), + vtiger_contact.get("last_name"), + vtiger_contact.get("email"), + vtiger_contact.get("phone"), + vtiger_contact.get("mobile"), + vtiger_contact.get("title"), + vtiger_contact.get("department"), + vtiger_contact.get("vtiger_id"), + ), + ) + stats["created"] += 1 + + link_exists = await asyncio.to_thread( + execute_query_single, + """ + SELECT id + FROM contact_companies + WHERE contact_id = %s AND customer_id = %s + """, + (contact_id, customer_id), + ) + if not link_exists: + await asyncio.to_thread( + execute_update, + """ + INSERT INTO contact_companies (contact_id, customer_id, is_primary, role) + VALUES (%s, %s, %s, %s) + ON CONFLICT (contact_id, customer_id) DO NOTHING + """, + (contact_id, customer_id, False, None), + ) + stats["linked"] += 1 + + return stats diff --git a/app/services/email_workflow_service.py b/app/services/email_workflow_service.py index 1a0fcfb..b115afb 100644 --- a/app/services/email_workflow_service.py +++ b/app/services/email_workflow_service.py @@ -19,6 +19,7 @@ from uuid import uuid4 from app.core.database import execute_query, execute_insert, execute_update, table_has_column from app.core.config import settings +from app.modules.also.backend.service import also_service from app.services.email_activity_logger import email_activity_logger logger = logging.getLogger(__name__) @@ -1236,7 +1237,7 @@ class EmailWorkflowService: confidence_threshold, workflow_steps, priority, stop_on_match FROM email_workflows WHERE enabled = true - AND classification_trigger = %s + AND LOWER(classification_trigger) IN (%s, 'any') AND confidence_threshold <= %s ORDER BY priority ASC """ @@ -1412,6 +1413,7 @@ class EmailWorkflowService: 'extract_invoice_data': self._action_extract_invoice_data, 'extract_tracking_number': self._action_extract_tracking_number, 'regex_extract_and_link': self._action_regex_extract_and_link, + 'process_also_cloud_billing': self._action_process_also_cloud_billing, 'send_slack_notification': self._action_send_slack_notification, 'send_email_notification': self._action_send_email_notification, 'mark_as_processed': self._action_mark_as_processed, @@ -1439,6 +1441,52 @@ class EmailWorkflowService: 'status': 'failed', 'error': str(e) } + + def _extract_also_billing_download_link(self, email_data: Dict) -> Optional[str]: + candidates = [ + email_data.get('body_html') or '', + email_data.get('body_text') or '', + email_data.get('subject') or '', + ] + pattern = re.compile(r'https://marketplace\.also\.[^"\s<]+?/download/[^"\s<]+', re.IGNORECASE) + for value in candidates: + match = pattern.search(str(value)) + if match: + return html.unescape(match.group(0)) + return None + + async def _action_process_also_cloud_billing(self, params: Dict, email_data: Dict) -> Dict: + """Download ALSO Cloud Marketplace billing ZIP, import lines, and create ordre drafts.""" + download_url = params.get('download_url') or self._extract_also_billing_download_link(email_data) + if not download_url: + return { + 'action': 'process_also_cloud_billing', + 'success': False, + 'reason': 'download_link_not_found', + } + + sender_email = str(email_data.get('sender_email') or '') + if params.get('require_sender_match', True): + sender_pattern = params.get('sender_pattern') or r'also\.[a-z]{2,}$' + if sender_email and not re.search(sender_pattern, sender_email, re.IGNORECASE): + return { + 'action': 'process_also_cloud_billing', + 'success': False, + 'reason': 'sender_pattern_no_match', + 'sender_email': sender_email, + } + + result = await also_service.import_billing_zip_from_url( + download_url=download_url, + imported_by_user_id=params.get('imported_by_user_id'), + email_id=email_data.get('id'), + source_label=params.get('source_label') or 'ALSO Cloud Marketplace email', + ) + return { + 'action': 'process_also_cloud_billing', + 'success': True, + **result, + } # Action Handlers diff --git a/app/services/simplycrm_service.py b/app/services/simplycrm_service.py index 6733790..5afd8fb 100644 --- a/app/services/simplycrm_service.py +++ b/app/services/simplycrm_service.py @@ -202,7 +202,7 @@ class SimplyCRMService: Returns: List of ticket records """ - module_name = getattr(settings, "SIMPLYCRM_TICKET_MODULE", "Tickets") + module_name = getattr(settings, "SIMPLYCRM_TICKET_MODULE", "HelpDesk") all_records: List[Dict] = [] offset = 0 batch_size = 200 diff --git a/app/settings/backend/router.py b/app/settings/backend/router.py index 1d3d36a..9b348f5 100644 --- a/app/settings/backend/router.py +++ b/app/settings/backend/router.py @@ -5,6 +5,7 @@ Settings and User Management API Router from fastapi import APIRouter, HTTPException, Request from typing import List, Optional, Dict from pydantic import BaseModel +from datetime import datetime from app.core.database import execute_query from app.core.config import settings import httpx @@ -57,8 +58,8 @@ class User(BaseModel): email: Optional[str] full_name: Optional[str] is_active: bool - last_login: Optional[str] - created_at: str + last_login: Optional[datetime] + created_at: datetime class UserCreate(BaseModel): @@ -992,4 +993,3 @@ async def test_ai_prompt(key: str, payload: PromptTestRequest, http_request: Req err = str(e) or e.__class__.__name__ raise HTTPException(status_code=500, detail=f"Kunne ikke teste AI prompt: {err}") - diff --git a/app/settings/frontend/settings.html b/app/settings/frontend/settings.html index 6bf9281..c9d4fa3 100644 --- a/app/settings/frontend/settings.html +++ b/app/settings/frontend/settings.html @@ -1047,7 +1047,7 @@
Beskeder lokalt: -
@@ -5145,9 +5145,9 @@ async function syncArchivedVtiger() { btn.innerHTML = 'Synkroniserer...'; try { - addSyncLogEntry('vTiger Archived Sync Startet', 'Importerer archived tickets fra vTiger Cases...', 'info'); + addSyncLogEntry('vTiger Archived Sync Startet', 'Importerer nye eller ændrede archived tickets fra vTiger Cases...', 'info'); - const response = await fetch('/api/v1/ticket/archived/vtiger/import?limit=5000&include_messages=true&force=false', { + const response = await fetch('/api/v1/ticket/archived/vtiger/import?limit=5000&include_messages=true&force=false&incremental=true', { method: 'POST' }); @@ -5158,6 +5158,8 @@ async function syncArchivedVtiger() { const result = await response.json(); const details = [ + `Mode: ${result.mode || 'ukendt'}`, + `Hentet remote: ${result.fetched_remote || 0}`, `Importeret: ${result.imported || 0}`, `Opdateret: ${result.updated || 0}`, `Sprunget over: ${result.skipped || 0}`, @@ -5173,7 +5175,7 @@ async function syncArchivedVtiger() { showNotification('Fejl: ' + error.message, 'error'); } finally { btn.disabled = false; - btn.innerHTML = 'Sync vTiger Archived'; + btn.innerHTML = 'Sync resten fra vTiger'; } } @@ -5891,6 +5893,7 @@ const MENU_VISIBILITY_GROUPS = [ { key: 'menu-salg-products', label: 'Produkter' }, { key: 'menu-salg-webshop', label: 'Webshop Administration' }, { key: 'menu-okonomi-time-queue', label: 'Time Queue' }, + { key: 'menu-okonomi-also-cloud', label: 'ALSO Cloud Marketplace' }, { key: 'menu-okonomi-supplier-invoices', label: 'Leverandør fakturaer' }, { key: 'menu-okonomi-prepaid', label: 'Prepaid Cards' }, { key: 'menu-okonomi-fixed-price', label: 'Fastpris Aftaler' }, diff --git a/app/shared/frontend/base.html b/app/shared/frontend/base.html index 6eac3c6..898f339 100644 --- a/app/shared/frontend/base.html +++ b/app/shared/frontend/base.html @@ -391,10 +391,14 @@ grid-template-columns: 160px minmax(0, 1fr); min-height: 240px; max-height: min(52vh, 420px); + height: min(52vh, 420px); overflow: hidden; box-shadow: inset 0 2px 10px rgba(0,0,0,0.02); margin-top: 0.5rem; } + .global-bottom-bar .bb-sheet-inner > * { + min-height: 0; + } .global-bottom-bar .bb-side-tabs { border-right: 1px solid rgba(var(--text-primary-rgb), 0.08); @@ -403,6 +407,8 @@ display: grid; gap: 0.4rem; align-content: start; + min-height: 0; + overflow-y: auto; } .global-bottom-bar .bb-tab-btn { @@ -424,6 +430,23 @@ font-size: 1rem; opacity: 0.7; } + .global-bottom-bar .bb-tab-btn .bb-tab-badge { + display: none; + align-items: center; + justify-content: center; + min-width: 1.2rem; + height: 1.2rem; + border-radius: 999px; + padding: 0 0.35rem; + margin-left: auto; + font-size: 0.7rem; + font-weight: 700; + background: rgba(220, 53, 69, 0.14); + color: #b02a37; + } + .global-bottom-bar .bb-tab-btn.has-unread .bb-tab-badge { + display: inline-flex; + } .global-bottom-bar .bb-tab-btn:hover { background: rgba(var(--text-primary-rgb), 0.05); color: var(--text-primary); @@ -439,20 +462,29 @@ opacity: 1; color: var(--accent); } + [data-bs-theme="dark"] .global-bottom-bar .bb-tab-btn .bb-tab-badge { + background: rgba(255, 138, 148, 0.18); + color: #ffb3ba; + } .global-bottom-bar .bb-tab-content { - padding: 1.2rem; - overflow: auto; + padding: 0.75rem 0.9rem 0.9rem; + overflow: hidden; + display: flex; + flex-direction: column; + height: 100%; + min-height: 0; } .global-bottom-bar .bb-tab-title { - font-size: 1.1rem; + font-size: 1rem; font-weight: 700; color: var(--text-primary); - margin-bottom: 1rem; + margin-bottom: 0.5rem; display: flex; align-items: center; gap: 0.5rem; + line-height: 1.15; } .global-bottom-bar .bb-tab-list { @@ -467,7 +499,7 @@ border-left: 4px solid var(--accent); background: var(--accent-light); border-radius: 6px 8px 8px 6px; - padding: 0.75rem 1rem; + padding: 0.65rem 0.85rem; font-size: 0.88rem; line-height: 1.4; color: var(--text-primary); @@ -478,6 +510,105 @@ transform: translateX(2px); box-shadow: 0 4px 14px rgba(0,0,0,0.08); } + .global-bottom-bar .bb-messages-layout { + display: flex; + flex-direction: column; + gap: 0.75rem; + min-height: 0; + } + .global-bottom-bar .bb-message-threads { + display: flex; + gap: 0.5rem; + overflow-x: auto; + padding-bottom: 0.1rem; + scrollbar-width: none; + } + .global-bottom-bar .bb-message-threads::-webkit-scrollbar { + display: none; + } + .global-bottom-bar .bb-message-thread { + border: 1px solid rgba(var(--text-primary-rgb), 0.1); + background: var(--bg-card); + color: var(--text-primary); + border-radius: 999px; + padding: 0.42rem 0.75rem; + display: inline-flex; + align-items: center; + gap: 0.45rem; + font-size: 0.78rem; + font-weight: 600; + white-space: nowrap; + } + .global-bottom-bar .bb-message-thread.is-active { + background: var(--accent); + color: #fff; + border-color: transparent; + box-shadow: 0 8px 20px rgba(15, 76, 117, 0.18); + } + .global-bottom-bar .bb-message-thread-count { + min-width: 1.2rem; + height: 1.2rem; + border-radius: 999px; + background: rgba(220, 53, 69, 0.14); + color: #b42318; + display: inline-flex; + align-items: center; + justify-content: center; + font-size: 0.7rem; + padding: 0 0.35rem; + } + .global-bottom-bar .bb-message-thread.is-active .bb-message-thread-count { + background: rgba(255, 255, 255, 0.18); + color: #fff; + } + .global-bottom-bar .bb-messages-list { + max-height: min(26vh, 240px); + overflow-y: auto; + padding-right: 0.2rem; + } + .global-bottom-bar #bbTabInnerContent { + flex: 1 1 auto; + display: flex; + flex-direction: column; + height: 100%; + min-height: 0; + overflow: hidden; + } + .global-bottom-bar .bb-notes-layout { + display: flex; + flex-direction: column; + gap: 0.75rem; + height: 100%; + min-height: 0; + overflow: hidden; + } + .global-bottom-bar .bb-notes-editor { + flex: 0 0 auto; + } + .global-bottom-bar .bb-notes-list { + flex: 1 1 auto; + min-height: 0; + overflow-y: auto; + padding-right: 0.2rem; + } + .global-bottom-bar .bb-messages-composer { + border-top: 1px solid rgba(var(--text-primary-rgb), 0.08); + padding-top: 0.55rem; + margin-top: 0.15rem; + } + .global-bottom-bar .bb-messages-layout { + height: 100%; + overflow: hidden; + } + .global-bottom-bar .bb-detail-line { + min-height: 28px; + padding: 0.2rem 0.2rem 0; + font-size: 0.8rem; + line-height: 1.2; + } + .global-bottom-bar.is-expanded .bb-detail-line { + padding-bottom: 0.1rem; + } #bbSwitchCaseModal .modal-content { border: 1px solid rgba(var(--text-primary-rgb), 0.12); @@ -777,7 +908,7 @@ {% set _xff = request.headers.get('x-forwarded-for') if request and request.headers else '' %} {% set _xff_first = _xff.split(',')[0].strip() if _xff else '' %} {% set _client_ip = (request.headers.get('cf-connecting-ip') if request and request.headers else '') or (request.headers.get('true-client-ip') if request and request.headers else '') or _xff_first or (request.headers.get('x-real-ip') if request and request.headers else '') or (request.client.host if request and request.client else '') %} -{% set _can_click_to_call = _client_ip.startswith('172.16.31.') %} +{% set _can_click_to_call = true %}