feat: Implement Email Workflow System with comprehensive documentation and migration scripts
- Added Email Workflow System with automated actions based on email classification. - Created database schema with tables for workflows, executions, and actions. - Developed API endpoints for CRUD operations on workflows and execution history. - Included pre-configured workflows for invoice processing, time confirmation, and bankruptcy alerts. - Introduced user guide and workflow system improvements for better usability. - Implemented backup system for automated backup jobs and notifications. - Established email activity log to track all actions and events related to emails.
This commit is contained in:
parent
38fa3b6c0a
commit
3fb43783a6
@ -74,6 +74,9 @@ TIMETRACKING_ROUND_INCREMENT=0.5 # Afrundingsinterval (0.25, 0.5, 1.0)
|
|||||||
TIMETRACKING_ROUND_METHOD=up # up (op til), nearest (nærmeste), down (ned til)
|
TIMETRACKING_ROUND_METHOD=up # up (op til), nearest (nærmeste), down (ned til)
|
||||||
TIMETRACKING_REQUIRE_APPROVAL=true # Kræv manuel godkendelse
|
TIMETRACKING_REQUIRE_APPROVAL=true # Kræv manuel godkendelse
|
||||||
|
|
||||||
|
# Order Management Security
|
||||||
|
TIMETRACKING_ADMIN_UNLOCK_CODE= # 🔐 Admin kode til at låse eksporterede ordrer op (sæt en stærk kode!)
|
||||||
|
|
||||||
# =====================================================
|
# =====================================================
|
||||||
# OLLAMA AI Integration (Optional - for document extraction)
|
# OLLAMA AI Integration (Optional - for document extraction)
|
||||||
# =====================================================
|
# =====================================================
|
||||||
|
|||||||
@ -8,6 +8,7 @@ RUN apt-get update && apt-get install -y \
|
|||||||
git \
|
git \
|
||||||
libpq-dev \
|
libpq-dev \
|
||||||
gcc \
|
gcc \
|
||||||
|
postgresql-client \
|
||||||
tesseract-ocr \
|
tesseract-ocr \
|
||||||
tesseract-ocr-dan \
|
tesseract-ocr-dan \
|
||||||
tesseract-ocr-eng \
|
tesseract-ocr-eng \
|
||||||
|
|||||||
10
app/backups/__init__.py
Normal file
10
app/backups/__init__.py
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
"""
|
||||||
|
Backup System Module for BMC Hub
|
||||||
|
|
||||||
|
Provides automated backup and restore functionality for:
|
||||||
|
- PostgreSQL database (compressed .dump + plain .sql)
|
||||||
|
- File storage (uploads/, data/, logs/)
|
||||||
|
- Automated rotation (30 days + monthly for 12 months)
|
||||||
|
- Offsite upload via SFTP/SSH
|
||||||
|
- Mattermost notifications
|
||||||
|
"""
|
||||||
1
app/backups/backend/__init__.py
Normal file
1
app/backups/backend/__init__.py
Normal file
@ -0,0 +1 @@
|
|||||||
|
"""Backup backend services, API routes, and scheduler."""
|
||||||
394
app/backups/backend/notifications.py
Normal file
394
app/backups/backend/notifications.py
Normal file
@ -0,0 +1,394 @@
|
|||||||
|
"""
|
||||||
|
Notification Service for Backup System
|
||||||
|
Sends rich formatted notifications to Mattermost webhook
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import aiohttp
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Dict, Optional, List
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.core.database import execute_insert
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class MattermostNotification:
|
||||||
|
"""Service for sending Mattermost webhook notifications"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.webhook_url = settings.MATTERMOST_WEBHOOK_URL
|
||||||
|
self.enabled = settings.MATTERMOST_ENABLED
|
||||||
|
self.channel = settings.MATTERMOST_CHANNEL
|
||||||
|
|
||||||
|
async def send_backup_success(self, job_id: int, job_type: str, file_size_bytes: int,
|
||||||
|
duration_seconds: float, checksum: str, is_monthly: bool = False):
|
||||||
|
"""
|
||||||
|
Send success notification for completed backup
|
||||||
|
|
||||||
|
Args:
|
||||||
|
job_id: Backup job ID
|
||||||
|
job_type: database, files, or full
|
||||||
|
file_size_bytes: Size of backup file
|
||||||
|
duration_seconds: Time taken to complete backup
|
||||||
|
checksum: SHA256 checksum of backup
|
||||||
|
is_monthly: Whether this is a monthly backup
|
||||||
|
"""
|
||||||
|
if not self._should_send_notification('backup_success'):
|
||||||
|
return
|
||||||
|
|
||||||
|
file_size_mb = file_size_bytes / 1024 / 1024
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"channel": self.channel,
|
||||||
|
"username": "BMC Hub Backup",
|
||||||
|
"icon_emoji": ":white_check_mark:",
|
||||||
|
"text": f"✅ **Backup Complete** - {job_type.capitalize()}",
|
||||||
|
"attachments": [{
|
||||||
|
"color": "#36a64f", # Green
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Job ID",
|
||||||
|
"value": f"#{job_id}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Type",
|
||||||
|
"value": f"{job_type.capitalize()} {'(Monthly)' if is_monthly else '(Daily)'}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Size",
|
||||||
|
"value": f"{file_size_mb:.2f} MB"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Duration",
|
||||||
|
"value": f"{duration_seconds:.1f}s"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": False,
|
||||||
|
"title": "Checksum (SHA256)",
|
||||||
|
"value": f"`{checksum[:16]}...{checksum[-16:]}`"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"footer": "BMC Hub Backup System",
|
||||||
|
"footer_icon": "https://platform.slack-edge.com/img/default_application_icon.png",
|
||||||
|
"ts": int(datetime.now().timestamp())
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
|
||||||
|
await self._send_webhook(payload, 'backup_success', job_id)
|
||||||
|
|
||||||
|
async def send_backup_failed(self, job_id: int, job_type: str, error_message: str):
|
||||||
|
"""
|
||||||
|
Send failure notification for failed backup
|
||||||
|
|
||||||
|
Args:
|
||||||
|
job_id: Backup job ID
|
||||||
|
job_type: database, files, or full
|
||||||
|
error_message: Error message from failed backup
|
||||||
|
"""
|
||||||
|
if not self._should_send_notification('backup_failed'):
|
||||||
|
return
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"channel": self.channel,
|
||||||
|
"username": "BMC Hub Backup",
|
||||||
|
"icon_emoji": ":x:",
|
||||||
|
"text": f"❌ **Backup Failed** - {job_type.capitalize()}",
|
||||||
|
"attachments": [{
|
||||||
|
"color": "#ff0000", # Red
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Job ID",
|
||||||
|
"value": f"#{job_id}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Type",
|
||||||
|
"value": job_type.capitalize()
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": False,
|
||||||
|
"title": "Error",
|
||||||
|
"value": f"```{error_message[:500]}```"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"actions": [
|
||||||
|
{
|
||||||
|
"name": "view_dashboard",
|
||||||
|
"type": "button",
|
||||||
|
"text": "View Dashboard",
|
||||||
|
"url": f"{self._get_hub_url()}/backups"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"footer": "BMC Hub Backup System",
|
||||||
|
"ts": int(datetime.now().timestamp())
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
|
||||||
|
await self._send_webhook(payload, 'backup_failed', job_id)
|
||||||
|
|
||||||
|
async def send_offsite_success(self, job_id: int, backup_name: str, file_size_bytes: int):
|
||||||
|
"""
|
||||||
|
Send notification for successful offsite upload
|
||||||
|
|
||||||
|
Args:
|
||||||
|
job_id: Backup job ID
|
||||||
|
backup_name: Name of uploaded backup file
|
||||||
|
file_size_bytes: Size of uploaded file
|
||||||
|
"""
|
||||||
|
if not settings.NOTIFY_ON_SUCCESS_OFFSITE:
|
||||||
|
return
|
||||||
|
|
||||||
|
file_size_mb = file_size_bytes / 1024 / 1024
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"channel": self.channel,
|
||||||
|
"username": "BMC Hub Backup",
|
||||||
|
"icon_emoji": ":cloud:",
|
||||||
|
"text": f"☁️ **Offsite Upload Complete**",
|
||||||
|
"attachments": [{
|
||||||
|
"color": "#0066cc", # Blue
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Job ID",
|
||||||
|
"value": f"#{job_id}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Destination",
|
||||||
|
"value": f"{settings.SFTP_HOST}:{settings.SFTP_REMOTE_PATH}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Filename",
|
||||||
|
"value": backup_name
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Size",
|
||||||
|
"value": f"{file_size_mb:.2f} MB"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"footer": "BMC Hub Backup System",
|
||||||
|
"ts": int(datetime.now().timestamp())
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
|
||||||
|
await self._send_webhook(payload, 'backup_success', job_id)
|
||||||
|
|
||||||
|
async def send_offsite_failed(self, job_id: int, backup_name: str, error_message: str,
|
||||||
|
retry_count: int):
|
||||||
|
"""
|
||||||
|
Send notification for failed offsite upload
|
||||||
|
|
||||||
|
Args:
|
||||||
|
job_id: Backup job ID
|
||||||
|
backup_name: Name of backup file
|
||||||
|
error_message: Error message
|
||||||
|
retry_count: Current retry attempt number
|
||||||
|
"""
|
||||||
|
if not self._should_send_notification('offsite_failed'):
|
||||||
|
return
|
||||||
|
|
||||||
|
max_retries = settings.OFFSITE_RETRY_MAX_ATTEMPTS
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"channel": self.channel,
|
||||||
|
"username": "BMC Hub Backup",
|
||||||
|
"icon_emoji": ":warning:",
|
||||||
|
"text": f"⚠️ **Offsite Upload Failed** (Retry {retry_count}/{max_retries})",
|
||||||
|
"attachments": [{
|
||||||
|
"color": "#ff9900", # Orange
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Job ID",
|
||||||
|
"value": f"#{job_id}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Destination",
|
||||||
|
"value": f"{settings.SFTP_HOST}:{settings.SFTP_REMOTE_PATH}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Filename",
|
||||||
|
"value": backup_name
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Retry Status",
|
||||||
|
"value": f"{retry_count}/{max_retries}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": False,
|
||||||
|
"title": "Error",
|
||||||
|
"value": f"```{error_message[:300]}```"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"footer": "BMC Hub Backup System",
|
||||||
|
"ts": int(datetime.now().timestamp())
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
|
||||||
|
await self._send_webhook(payload, 'offsite_failed', job_id)
|
||||||
|
|
||||||
|
async def send_storage_warning(self, usage_pct: float, used_gb: float, max_gb: int):
|
||||||
|
"""
|
||||||
|
Send notification for high storage usage
|
||||||
|
|
||||||
|
Args:
|
||||||
|
usage_pct: Percentage of storage used
|
||||||
|
used_gb: Gigabytes used
|
||||||
|
max_gb: Maximum storage capacity
|
||||||
|
"""
|
||||||
|
if not self._should_send_notification('storage_low'):
|
||||||
|
return
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"channel": self.channel,
|
||||||
|
"username": "BMC Hub Backup",
|
||||||
|
"icon_emoji": ":warning:",
|
||||||
|
"text": f"⚠️ **Backup Storage Warning**",
|
||||||
|
"attachments": [{
|
||||||
|
"color": "#ff9900", # Orange
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Usage",
|
||||||
|
"value": f"{usage_pct:.1f}%"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Space Used",
|
||||||
|
"value": f"{used_gb:.2f} GB / {max_gb} GB"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": False,
|
||||||
|
"title": "Recommendation",
|
||||||
|
"value": "Consider running backup rotation or increasing storage capacity."
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"actions": [
|
||||||
|
{
|
||||||
|
"name": "view_dashboard",
|
||||||
|
"type": "button",
|
||||||
|
"text": "View Dashboard",
|
||||||
|
"url": f"{self._get_hub_url()}/backups"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"footer": "BMC Hub Backup System",
|
||||||
|
"ts": int(datetime.now().timestamp())
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
|
||||||
|
await self._send_webhook(payload, 'storage_low')
|
||||||
|
|
||||||
|
async def send_restore_started(self, job_id: int, backup_name: str, eta_minutes: int):
|
||||||
|
"""
|
||||||
|
Send notification when restore operation starts
|
||||||
|
|
||||||
|
Args:
|
||||||
|
job_id: Backup job ID being restored
|
||||||
|
backup_name: Name of backup file
|
||||||
|
eta_minutes: Estimated time to completion
|
||||||
|
"""
|
||||||
|
payload = {
|
||||||
|
"channel": self.channel,
|
||||||
|
"username": "BMC Hub Backup",
|
||||||
|
"icon_emoji": ":gear:",
|
||||||
|
"text": f"🔧 **System Maintenance: Restore in Progress**",
|
||||||
|
"attachments": [{
|
||||||
|
"color": "#ffcc00", # Yellow
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "Backup Job ID",
|
||||||
|
"value": f"#{job_id}"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": True,
|
||||||
|
"title": "ETA",
|
||||||
|
"value": f"{eta_minutes} minutes"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": False,
|
||||||
|
"title": "Backup File",
|
||||||
|
"value": backup_name
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"short": False,
|
||||||
|
"title": "Status",
|
||||||
|
"value": "System is in maintenance mode. All services temporarily unavailable."
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"footer": "BMC Hub Backup System",
|
||||||
|
"ts": int(datetime.now().timestamp())
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
|
||||||
|
await self._send_webhook(payload, 'restore_started', job_id)
|
||||||
|
|
||||||
|
async def _send_webhook(self, payload: Dict, event_type: str, job_id: Optional[int] = None):
|
||||||
|
"""
|
||||||
|
Send webhook to Mattermost and log notification
|
||||||
|
|
||||||
|
Args:
|
||||||
|
payload: Mattermost webhook payload
|
||||||
|
event_type: Type of notification event
|
||||||
|
job_id: Optional backup job ID
|
||||||
|
"""
|
||||||
|
if not self.enabled or not self.webhook_url:
|
||||||
|
logger.info("📢 Notification (disabled): %s - job_id=%s", event_type, job_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
async with session.post(self.webhook_url, json=payload, timeout=10) as response:
|
||||||
|
if response.status == 200:
|
||||||
|
logger.info("📢 Notification sent: %s - job_id=%s", event_type, job_id)
|
||||||
|
|
||||||
|
# Log to database
|
||||||
|
execute_insert(
|
||||||
|
"""INSERT INTO backup_notifications
|
||||||
|
(backup_job_id, event_type, message, mattermost_payload)
|
||||||
|
VALUES (%s, %s, %s, %s)""",
|
||||||
|
(job_id, event_type, payload.get('text', ''), str(payload))
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
error_text = await response.text()
|
||||||
|
logger.error("❌ Notification failed: HTTP %s - %s",
|
||||||
|
response.status, error_text)
|
||||||
|
|
||||||
|
except aiohttp.ClientError as e:
|
||||||
|
logger.error("❌ Notification connection error: %s", str(e))
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Notification error: %s", str(e))
|
||||||
|
|
||||||
|
def _should_send_notification(self, event_type: str) -> bool:
|
||||||
|
"""Check if notification should be sent based on settings"""
|
||||||
|
if not self.enabled or not self.webhook_url:
|
||||||
|
return False
|
||||||
|
|
||||||
|
if event_type in ['backup_failed', 'offsite_failed', 'storage_low']:
|
||||||
|
return settings.NOTIFY_ON_FAILURE
|
||||||
|
|
||||||
|
if event_type == 'backup_success' and 'offsite' in event_type:
|
||||||
|
return settings.NOTIFY_ON_SUCCESS_OFFSITE
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _get_hub_url(self) -> str:
|
||||||
|
"""Get BMC Hub base URL for action buttons"""
|
||||||
|
# TODO: Add HUB_BASE_URL to config
|
||||||
|
return "http://localhost:8000" # Fallback
|
||||||
|
|
||||||
|
|
||||||
|
# Singleton instance
|
||||||
|
notifications = MattermostNotification()
|
||||||
513
app/backups/backend/router.py
Normal file
513
app/backups/backend/router.py
Normal file
@ -0,0 +1,513 @@
|
|||||||
|
"""
|
||||||
|
Backup System API Router
|
||||||
|
REST endpoints for backup management
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import List, Optional
|
||||||
|
from datetime import datetime, date, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
from fastapi import APIRouter, HTTPException, Query, UploadFile, File
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
from app.core.database import execute_query, execute_update, execute_insert
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.backups.backend.service import backup_service
|
||||||
|
from app.backups.backend.notifications import notifications
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
# Pydantic Models
|
||||||
|
class BackupCreate(BaseModel):
|
||||||
|
"""Request model for creating a new backup"""
|
||||||
|
job_type: str = Field(..., description="Type of backup: database, files, or full")
|
||||||
|
is_monthly: bool = Field(False, description="Create monthly backup (uses SQL format)")
|
||||||
|
|
||||||
|
|
||||||
|
class BackupJob(BaseModel):
|
||||||
|
"""Response model for backup job"""
|
||||||
|
id: int
|
||||||
|
job_type: str
|
||||||
|
status: str
|
||||||
|
backup_format: str
|
||||||
|
file_path: Optional[str]
|
||||||
|
file_size_bytes: Optional[int]
|
||||||
|
checksum_sha256: Optional[str]
|
||||||
|
is_monthly: bool
|
||||||
|
includes_uploads: bool
|
||||||
|
includes_logs: bool
|
||||||
|
includes_data: bool
|
||||||
|
started_at: Optional[datetime]
|
||||||
|
completed_at: Optional[datetime]
|
||||||
|
error_message: Optional[str]
|
||||||
|
retention_until: Optional[date]
|
||||||
|
offsite_uploaded_at: Optional[datetime]
|
||||||
|
offsite_retry_count: int
|
||||||
|
notification_sent: bool
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class RestoreRequest(BaseModel):
|
||||||
|
"""Request model for restoring from backup"""
|
||||||
|
confirmation: bool = Field(..., description="Must be true to confirm restore operation")
|
||||||
|
message: Optional[str] = Field(None, description="Optional restore reason/notes")
|
||||||
|
|
||||||
|
|
||||||
|
class MaintenanceStatus(BaseModel):
|
||||||
|
"""Response model for maintenance mode status"""
|
||||||
|
maintenance_mode: bool
|
||||||
|
maintenance_message: str
|
||||||
|
maintenance_eta_minutes: Optional[int]
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class NotificationRecord(BaseModel):
|
||||||
|
"""Response model for notification record"""
|
||||||
|
id: int
|
||||||
|
backup_job_id: Optional[int]
|
||||||
|
event_type: str
|
||||||
|
message: str
|
||||||
|
sent_at: datetime
|
||||||
|
acknowledged: bool
|
||||||
|
acknowledged_at: Optional[datetime]
|
||||||
|
|
||||||
|
|
||||||
|
class StorageStats(BaseModel):
|
||||||
|
"""Response model for storage statistics"""
|
||||||
|
total_size_bytes: int
|
||||||
|
total_size_gb: float
|
||||||
|
max_size_gb: int
|
||||||
|
usage_pct: float
|
||||||
|
file_count: int
|
||||||
|
warning: bool
|
||||||
|
|
||||||
|
|
||||||
|
# API Endpoints
|
||||||
|
|
||||||
|
@router.post("/backups", response_model=dict, tags=["Backups"])
|
||||||
|
async def create_backup(backup: BackupCreate):
|
||||||
|
"""
|
||||||
|
Create a new backup manually
|
||||||
|
|
||||||
|
- **job_type**: database, files, or full
|
||||||
|
- **is_monthly**: Use plain SQL format for database (monthly backups)
|
||||||
|
"""
|
||||||
|
if not settings.BACKUP_ENABLED:
|
||||||
|
raise HTTPException(status_code=503, detail="Backup system is disabled (BACKUP_ENABLED=false)")
|
||||||
|
|
||||||
|
logger.info("📦 Manual backup requested: type=%s, monthly=%s", backup.job_type, backup.is_monthly)
|
||||||
|
|
||||||
|
try:
|
||||||
|
if backup.job_type == 'database':
|
||||||
|
job_id = await backup_service.create_database_backup(is_monthly=backup.is_monthly)
|
||||||
|
if job_id:
|
||||||
|
return {"success": True, "job_id": job_id, "message": "Database backup created successfully"}
|
||||||
|
else:
|
||||||
|
raise HTTPException(status_code=500, detail="Database backup failed - check logs")
|
||||||
|
|
||||||
|
elif backup.job_type == 'files':
|
||||||
|
job_id = await backup_service.create_files_backup()
|
||||||
|
if job_id:
|
||||||
|
return {"success": True, "job_id": job_id, "message": "Files backup created successfully"}
|
||||||
|
else:
|
||||||
|
raise HTTPException(status_code=500, detail="Files backup failed - check logs")
|
||||||
|
|
||||||
|
elif backup.job_type == 'full':
|
||||||
|
db_job_id, files_job_id = await backup_service.create_full_backup(is_monthly=backup.is_monthly)
|
||||||
|
if db_job_id and files_job_id:
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"db_job_id": db_job_id,
|
||||||
|
"files_job_id": files_job_id,
|
||||||
|
"message": "Full backup created successfully"
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
raise HTTPException(status_code=500, detail=f"Full backup partially failed: db={db_job_id}, files={files_job_id}")
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise HTTPException(status_code=400, detail="Invalid job_type. Must be: database, files, or full")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Manual backup error: %s", str(e), exc_info=True)
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/backups/jobs", response_model=List[BackupJob], tags=["Backups"])
|
||||||
|
async def list_backups(
|
||||||
|
status: Optional[str] = Query(None, description="Filter by status: pending, running, completed, failed"),
|
||||||
|
job_type: Optional[str] = Query(None, description="Filter by type: database, files, full"),
|
||||||
|
limit: int = Query(50, ge=1, le=500),
|
||||||
|
offset: int = Query(0, ge=0)
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
List backup jobs with optional filtering and pagination
|
||||||
|
"""
|
||||||
|
# Build query
|
||||||
|
query = "SELECT * FROM backup_jobs WHERE 1=1"
|
||||||
|
params = []
|
||||||
|
|
||||||
|
if status:
|
||||||
|
query += " AND status = %s"
|
||||||
|
params.append(status)
|
||||||
|
|
||||||
|
if job_type:
|
||||||
|
query += " AND job_type = %s"
|
||||||
|
params.append(job_type)
|
||||||
|
|
||||||
|
query += " ORDER BY created_at DESC LIMIT %s OFFSET %s"
|
||||||
|
params.extend([limit, offset])
|
||||||
|
|
||||||
|
backups = execute_query(query, tuple(params))
|
||||||
|
|
||||||
|
return backups if backups else []
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/backups/jobs/{job_id}", response_model=BackupJob, tags=["Backups"])
|
||||||
|
async def get_backup(job_id: int):
|
||||||
|
"""Get details of a specific backup job"""
|
||||||
|
backup = execute_query(
|
||||||
|
"SELECT * FROM backup_jobs WHERE id = %s",
|
||||||
|
(job_id,),
|
||||||
|
fetchone=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if not backup:
|
||||||
|
raise HTTPException(status_code=404, detail=f"Backup job {job_id} not found")
|
||||||
|
|
||||||
|
return backup
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/backups/upload", response_model=dict, tags=["Backups"])
|
||||||
|
async def upload_backup(
|
||||||
|
file: UploadFile = File(...),
|
||||||
|
backup_type: str = Query(..., description="Type: database or files"),
|
||||||
|
is_monthly: bool = Query(False, description="Mark as monthly backup")
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Upload a previously downloaded backup file
|
||||||
|
|
||||||
|
Validates file format and creates backup job record
|
||||||
|
"""
|
||||||
|
if settings.BACKUP_READ_ONLY:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=403,
|
||||||
|
detail="Upload blocked: BACKUP_READ_ONLY=true"
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("📤 Backup upload: filename=%s, type=%s, size=%d bytes",
|
||||||
|
file.filename, backup_type, file.size if hasattr(file, 'size') else 0)
|
||||||
|
|
||||||
|
# Validate file type
|
||||||
|
allowed_extensions = {
|
||||||
|
'database': ['.dump', '.sql', '.sql.gz'],
|
||||||
|
'files': ['.tar.gz', '.tgz']
|
||||||
|
}
|
||||||
|
|
||||||
|
if backup_type not in allowed_extensions:
|
||||||
|
raise HTTPException(status_code=400, detail="Invalid backup_type. Must be: database or files")
|
||||||
|
|
||||||
|
file_ext = ''.join(Path(file.filename).suffixes)
|
||||||
|
if file_ext not in allowed_extensions[backup_type]:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"Invalid file extension '{file_ext}' for type '{backup_type}'. Allowed: {allowed_extensions[backup_type]}"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Determine storage path and format
|
||||||
|
backup_dir = Path(settings.BACKUP_STORAGE_PATH)
|
||||||
|
|
||||||
|
if backup_type == 'database':
|
||||||
|
target_dir = backup_dir / "database"
|
||||||
|
backup_format = 'dump' if file_ext == '.dump' else 'sql'
|
||||||
|
else:
|
||||||
|
target_dir = backup_dir / "files"
|
||||||
|
backup_format = 'tar.gz'
|
||||||
|
|
||||||
|
target_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Generate unique filename with timestamp
|
||||||
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||||
|
original_name = Path(file.filename).stem
|
||||||
|
new_filename = f"{original_name}_uploaded_{timestamp}{file_ext}"
|
||||||
|
target_path = target_dir / new_filename
|
||||||
|
|
||||||
|
# Save uploaded file
|
||||||
|
logger.info("💾 Saving upload to: %s", target_path)
|
||||||
|
|
||||||
|
content = await file.read()
|
||||||
|
with open(target_path, 'wb') as f:
|
||||||
|
f.write(content)
|
||||||
|
|
||||||
|
file_size = target_path.stat().st_size
|
||||||
|
|
||||||
|
# Calculate checksum
|
||||||
|
import hashlib
|
||||||
|
checksum = hashlib.sha256(content).hexdigest()
|
||||||
|
|
||||||
|
logger.info("✅ File saved: %d bytes, checksum=%s", file_size, checksum[:16])
|
||||||
|
|
||||||
|
# Calculate retention date
|
||||||
|
if is_monthly:
|
||||||
|
retention_until = datetime.now() + timedelta(days=settings.MONTHLY_KEEP_MONTHS * 30)
|
||||||
|
else:
|
||||||
|
retention_until = datetime.now() + timedelta(days=settings.RETENTION_DAYS)
|
||||||
|
|
||||||
|
# Create backup job record
|
||||||
|
job_id = execute_insert(
|
||||||
|
"""INSERT INTO backup_jobs
|
||||||
|
(job_type, status, backup_format, file_path, file_size_bytes,
|
||||||
|
checksum_sha256, is_monthly, started_at, completed_at, retention_until)
|
||||||
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
|
||||||
|
(backup_type, 'completed', backup_format, str(target_path), file_size,
|
||||||
|
checksum, is_monthly, datetime.now(), datetime.now(), retention_until.date())
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("✅ Backup upload registered: job_id=%s", job_id)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"job_id": job_id,
|
||||||
|
"message": f"Backup uploaded successfully: {new_filename}",
|
||||||
|
"file_size_mb": round(file_size / 1024 / 1024, 2),
|
||||||
|
"checksum": checksum
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Upload failed: %s", str(e), exc_info=True)
|
||||||
|
# Clean up partial file
|
||||||
|
if target_path.exists():
|
||||||
|
target_path.unlink()
|
||||||
|
raise HTTPException(status_code=500, detail=f"Upload failed: {str(e)}")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/backups/restore/{job_id}", response_model=dict, tags=["Backups"])
|
||||||
|
async def restore_backup(job_id: int, request: RestoreRequest):
|
||||||
|
"""
|
||||||
|
Restore from a backup (database or files)
|
||||||
|
|
||||||
|
**WARNING**: This will enable maintenance mode and temporarily shut down the system
|
||||||
|
"""
|
||||||
|
if not request.confirmation:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail="Restore operation requires confirmation=true"
|
||||||
|
)
|
||||||
|
|
||||||
|
if settings.BACKUP_READ_ONLY:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=403,
|
||||||
|
detail="Restore blocked: BACKUP_READ_ONLY=true. Update configuration to enable restores."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Get backup job
|
||||||
|
backup = execute_query(
|
||||||
|
"SELECT * FROM backup_jobs WHERE id = %s",
|
||||||
|
(job_id,),
|
||||||
|
fetchone=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if not backup:
|
||||||
|
raise HTTPException(status_code=404, detail=f"Backup job {job_id} not found")
|
||||||
|
|
||||||
|
if backup['status'] != 'completed':
|
||||||
|
raise HTTPException(status_code=400, detail=f"Cannot restore from backup with status: {backup['status']}")
|
||||||
|
|
||||||
|
logger.warning("🔧 Restore initiated: job_id=%s, type=%s, user_message=%s",
|
||||||
|
job_id, backup['job_type'], request.message)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Send notification
|
||||||
|
await notifications.send_restore_started(
|
||||||
|
job_id=job_id,
|
||||||
|
backup_name=backup['file_path'].split('/')[-1],
|
||||||
|
eta_minutes=5
|
||||||
|
)
|
||||||
|
|
||||||
|
# Perform restore based on type
|
||||||
|
if backup['job_type'] == 'database':
|
||||||
|
success = await backup_service.restore_database(job_id)
|
||||||
|
elif backup['job_type'] == 'files':
|
||||||
|
success = await backup_service.restore_files(job_id)
|
||||||
|
elif backup['job_type'] == 'full':
|
||||||
|
# Restore both database and files
|
||||||
|
db_success = await backup_service.restore_database(job_id)
|
||||||
|
files_success = await backup_service.restore_files(job_id)
|
||||||
|
success = db_success and files_success
|
||||||
|
else:
|
||||||
|
raise HTTPException(status_code=400, detail=f"Unknown backup type: {backup['job_type']}")
|
||||||
|
|
||||||
|
if success:
|
||||||
|
logger.info("✅ Restore completed successfully: job_id=%s", job_id)
|
||||||
|
return {"success": True, "message": "Restore completed successfully"}
|
||||||
|
else:
|
||||||
|
logger.error("❌ Restore failed: job_id=%s", job_id)
|
||||||
|
raise HTTPException(status_code=500, detail="Restore operation failed - check logs")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Restore error: %s", str(e), exc_info=True)
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/backups/jobs/{job_id}", response_model=dict, tags=["Backups"])
|
||||||
|
async def delete_backup(job_id: int):
|
||||||
|
"""
|
||||||
|
Delete a backup job and its associated file
|
||||||
|
"""
|
||||||
|
# Get backup job
|
||||||
|
backup = execute_query(
|
||||||
|
"SELECT * FROM backup_jobs WHERE id = %s",
|
||||||
|
(job_id,),
|
||||||
|
fetchone=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if not backup:
|
||||||
|
raise HTTPException(status_code=404, detail=f"Backup job {job_id} not found")
|
||||||
|
|
||||||
|
logger.info("🗑️ Deleting backup: job_id=%s, file=%s", job_id, backup['file_path'])
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Delete file if exists
|
||||||
|
from pathlib import Path
|
||||||
|
if backup['file_path']:
|
||||||
|
file_path = Path(backup['file_path'])
|
||||||
|
if file_path.exists():
|
||||||
|
file_path.unlink()
|
||||||
|
logger.info("✅ Deleted backup file: %s", file_path.name)
|
||||||
|
|
||||||
|
# Delete database record
|
||||||
|
execute_update("DELETE FROM backup_jobs WHERE id = %s", (job_id,))
|
||||||
|
|
||||||
|
return {"success": True, "message": f"Backup {job_id} deleted successfully"}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Delete backup error: %s", str(e), exc_info=True)
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/backups/offsite/{job_id}", response_model=dict, tags=["Backups"])
|
||||||
|
async def upload_offsite(job_id: int):
|
||||||
|
"""
|
||||||
|
Manually trigger offsite upload for a specific backup
|
||||||
|
"""
|
||||||
|
if not settings.OFFSITE_ENABLED:
|
||||||
|
raise HTTPException(status_code=503, detail="Offsite uploads are disabled (OFFSITE_ENABLED=false)")
|
||||||
|
|
||||||
|
logger.info("☁️ Manual offsite upload requested: job_id=%s", job_id)
|
||||||
|
|
||||||
|
try:
|
||||||
|
success = await backup_service.upload_offsite(job_id)
|
||||||
|
|
||||||
|
if success:
|
||||||
|
return {"success": True, "message": f"Backup {job_id} uploaded to offsite successfully"}
|
||||||
|
else:
|
||||||
|
raise HTTPException(status_code=500, detail="Offsite upload failed - check logs")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Manual offsite upload error: %s", str(e), exc_info=True)
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/backups/maintenance", response_model=MaintenanceStatus, tags=["System"])
|
||||||
|
async def get_maintenance_status():
|
||||||
|
"""
|
||||||
|
Get current maintenance mode status
|
||||||
|
|
||||||
|
Used by frontend to display maintenance overlay
|
||||||
|
"""
|
||||||
|
status = execute_query(
|
||||||
|
"SELECT * FROM system_status WHERE id = 1",
|
||||||
|
fetchone=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if not status:
|
||||||
|
# Return default status if not found
|
||||||
|
return {
|
||||||
|
"maintenance_mode": False,
|
||||||
|
"maintenance_message": "",
|
||||||
|
"maintenance_eta_minutes": None,
|
||||||
|
"updated_at": datetime.now()
|
||||||
|
}
|
||||||
|
|
||||||
|
return status
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/backups/notifications", response_model=List[NotificationRecord], tags=["Backups"])
|
||||||
|
async def list_notifications(
|
||||||
|
acknowledged: Optional[bool] = Query(None, description="Filter by acknowledged status"),
|
||||||
|
limit: int = Query(50, ge=1, le=200),
|
||||||
|
offset: int = Query(0, ge=0)
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
List backup notifications (alerts, warnings, errors)
|
||||||
|
"""
|
||||||
|
query = "SELECT * FROM backup_notifications WHERE 1=1"
|
||||||
|
params = []
|
||||||
|
|
||||||
|
if acknowledged is not None:
|
||||||
|
query += " AND acknowledged = %s"
|
||||||
|
params.append(acknowledged)
|
||||||
|
|
||||||
|
query += " ORDER BY sent_at DESC LIMIT %s OFFSET %s"
|
||||||
|
params.extend([limit, offset])
|
||||||
|
|
||||||
|
notifications_list = execute_query(query, tuple(params))
|
||||||
|
|
||||||
|
return notifications_list if notifications_list else []
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/backups/notifications/{notification_id}/acknowledge", response_model=dict, tags=["Backups"])
|
||||||
|
async def acknowledge_notification(notification_id: int):
|
||||||
|
"""
|
||||||
|
Acknowledge a notification (mark as read)
|
||||||
|
"""
|
||||||
|
execute_update(
|
||||||
|
"""UPDATE backup_notifications
|
||||||
|
SET acknowledged = true, acknowledged_at = %s
|
||||||
|
WHERE id = %s""",
|
||||||
|
(datetime.now(), notification_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
return {"success": True, "message": f"Notification {notification_id} acknowledged"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/backups/storage", response_model=StorageStats, tags=["System"])
|
||||||
|
async def get_storage_stats():
|
||||||
|
"""
|
||||||
|
Get backup storage usage statistics
|
||||||
|
"""
|
||||||
|
stats = await backup_service.check_storage_usage()
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/backups/scheduler/status", response_model=dict, tags=["System"])
|
||||||
|
async def get_scheduler_status():
|
||||||
|
"""
|
||||||
|
Get backup scheduler status and job information
|
||||||
|
"""
|
||||||
|
from app.backups.backend.scheduler import backup_scheduler
|
||||||
|
|
||||||
|
if not backup_scheduler.running:
|
||||||
|
return {
|
||||||
|
"enabled": settings.BACKUP_ENABLED,
|
||||||
|
"running": False,
|
||||||
|
"message": "Backup scheduler is not running"
|
||||||
|
}
|
||||||
|
|
||||||
|
jobs = []
|
||||||
|
for job in backup_scheduler.scheduler.get_jobs():
|
||||||
|
jobs.append({
|
||||||
|
"id": job.id,
|
||||||
|
"name": job.name,
|
||||||
|
"next_run": job.next_run_time.isoformat() if job.next_run_time else None,
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
"enabled": settings.BACKUP_ENABLED,
|
||||||
|
"running": backup_scheduler.running,
|
||||||
|
"jobs": jobs
|
||||||
|
}
|
||||||
401
app/backups/backend/scheduler.py
Normal file
401
app/backups/backend/scheduler.py
Normal file
@ -0,0 +1,401 @@
|
|||||||
|
"""
|
||||||
|
Backup Scheduler
|
||||||
|
Manages scheduled backup jobs, rotation, offsite uploads, and retry logic
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from datetime import datetime, time
|
||||||
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||||
|
from apscheduler.triggers.cron import CronTrigger
|
||||||
|
from apscheduler.triggers.interval import IntervalTrigger
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.core.database import execute_query
|
||||||
|
from app.backups.backend.service import backup_service
|
||||||
|
from app.backups.backend.notifications import notifications
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class BackupScheduler:
|
||||||
|
"""Scheduler for automated backup operations"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.scheduler = AsyncIOScheduler()
|
||||||
|
self.enabled = settings.BACKUP_ENABLED
|
||||||
|
self.running = False
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
"""Start the backup scheduler with all jobs"""
|
||||||
|
if not self.enabled:
|
||||||
|
logger.info("⏭️ Backup scheduler disabled (BACKUP_ENABLED=false)")
|
||||||
|
return
|
||||||
|
|
||||||
|
if self.running:
|
||||||
|
logger.warning("⚠️ Backup scheduler already running")
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info("🚀 Starting backup scheduler...")
|
||||||
|
|
||||||
|
# Daily full backup at 02:00 CET
|
||||||
|
self.scheduler.add_job(
|
||||||
|
func=self._daily_backup_job,
|
||||||
|
trigger=CronTrigger(hour=2, minute=0),
|
||||||
|
id='daily_backup',
|
||||||
|
name='Daily Full Backup',
|
||||||
|
max_instances=1,
|
||||||
|
replace_existing=True
|
||||||
|
)
|
||||||
|
logger.info("✅ Scheduled: Daily backup at 02:00")
|
||||||
|
|
||||||
|
# Monthly backup on the 1st at 02:00 CET
|
||||||
|
self.scheduler.add_job(
|
||||||
|
func=self._monthly_backup_job,
|
||||||
|
trigger=CronTrigger(day=1, hour=2, minute=0),
|
||||||
|
id='monthly_backup',
|
||||||
|
name='Monthly Full Backup (SQL format)',
|
||||||
|
max_instances=1,
|
||||||
|
replace_existing=True
|
||||||
|
)
|
||||||
|
logger.info("✅ Scheduled: Monthly backup on 1st at 02:00")
|
||||||
|
|
||||||
|
# Weekly offsite upload (configurable day, default Sunday at 03:00)
|
||||||
|
offsite_day = self._get_weekday_number(settings.OFFSITE_WEEKLY_DAY)
|
||||||
|
self.scheduler.add_job(
|
||||||
|
func=self._offsite_upload_job,
|
||||||
|
trigger=CronTrigger(day_of_week=offsite_day, hour=3, minute=0),
|
||||||
|
id='offsite_upload',
|
||||||
|
name=f'Weekly Offsite Upload ({settings.OFFSITE_WEEKLY_DAY.capitalize()})',
|
||||||
|
max_instances=1,
|
||||||
|
replace_existing=True
|
||||||
|
)
|
||||||
|
logger.info("✅ Scheduled: Weekly offsite upload on %s at 03:00",
|
||||||
|
settings.OFFSITE_WEEKLY_DAY.capitalize())
|
||||||
|
|
||||||
|
# Offsite retry job (every hour)
|
||||||
|
self.scheduler.add_job(
|
||||||
|
func=self._offsite_retry_job,
|
||||||
|
trigger=IntervalTrigger(hours=settings.OFFSITE_RETRY_DELAY_HOURS),
|
||||||
|
id='offsite_retry',
|
||||||
|
name='Offsite Upload Retry',
|
||||||
|
max_instances=1,
|
||||||
|
replace_existing=True
|
||||||
|
)
|
||||||
|
logger.info("✅ Scheduled: Offsite retry every %d hour(s)",
|
||||||
|
settings.OFFSITE_RETRY_DELAY_HOURS)
|
||||||
|
|
||||||
|
# Backup rotation (daily at 01:00)
|
||||||
|
self.scheduler.add_job(
|
||||||
|
func=self._rotation_job,
|
||||||
|
trigger=CronTrigger(hour=1, minute=0),
|
||||||
|
id='backup_rotation',
|
||||||
|
name='Backup Rotation',
|
||||||
|
max_instances=1,
|
||||||
|
replace_existing=True
|
||||||
|
)
|
||||||
|
logger.info("✅ Scheduled: Backup rotation at 01:00")
|
||||||
|
|
||||||
|
# Storage check (daily at 01:30)
|
||||||
|
self.scheduler.add_job(
|
||||||
|
func=self._storage_check_job,
|
||||||
|
trigger=CronTrigger(hour=1, minute=30),
|
||||||
|
id='storage_check',
|
||||||
|
name='Storage Usage Check',
|
||||||
|
max_instances=1,
|
||||||
|
replace_existing=True
|
||||||
|
)
|
||||||
|
logger.info("✅ Scheduled: Storage check at 01:30")
|
||||||
|
|
||||||
|
# Start the scheduler
|
||||||
|
self.scheduler.start()
|
||||||
|
self.running = True
|
||||||
|
|
||||||
|
logger.info("✅ Backup scheduler started successfully")
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
"""Stop the backup scheduler"""
|
||||||
|
if not self.running:
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info("🛑 Stopping backup scheduler...")
|
||||||
|
self.scheduler.shutdown(wait=True)
|
||||||
|
self.running = False
|
||||||
|
logger.info("✅ Backup scheduler stopped")
|
||||||
|
|
||||||
|
def pause(self):
|
||||||
|
"""Pause all scheduled jobs (for maintenance)"""
|
||||||
|
if not self.running:
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info("⏸️ Pausing backup scheduler...")
|
||||||
|
self.scheduler.pause()
|
||||||
|
logger.info("✅ Backup scheduler paused")
|
||||||
|
|
||||||
|
def resume(self):
|
||||||
|
"""Resume all scheduled jobs"""
|
||||||
|
if not self.running:
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info("▶️ Resuming backup scheduler...")
|
||||||
|
self.scheduler.resume()
|
||||||
|
logger.info("✅ Backup scheduler resumed")
|
||||||
|
|
||||||
|
async def _daily_backup_job(self):
|
||||||
|
"""Daily full backup job (database + files)"""
|
||||||
|
logger.info("🔄 Starting daily backup job...")
|
||||||
|
|
||||||
|
try:
|
||||||
|
start_time = datetime.now()
|
||||||
|
|
||||||
|
# Create full backup (database uses compressed .dump format)
|
||||||
|
db_job_id, files_job_id = await backup_service.create_full_backup(is_monthly=False)
|
||||||
|
|
||||||
|
end_time = datetime.now()
|
||||||
|
duration = (end_time - start_time).total_seconds()
|
||||||
|
|
||||||
|
if db_job_id and files_job_id:
|
||||||
|
logger.info("✅ Daily backup completed: db=%s, files=%s (%.1fs)",
|
||||||
|
db_job_id, files_job_id, duration)
|
||||||
|
|
||||||
|
# Send success notification for database backup
|
||||||
|
db_backup = execute_query(
|
||||||
|
"SELECT * FROM backup_jobs WHERE id = %s",
|
||||||
|
(db_job_id,),
|
||||||
|
fetchone=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if db_backup:
|
||||||
|
await notifications.send_backup_success(
|
||||||
|
job_id=db_job_id,
|
||||||
|
job_type='database',
|
||||||
|
file_size_bytes=db_backup['file_size_bytes'],
|
||||||
|
duration_seconds=duration / 2, # Rough estimate
|
||||||
|
checksum=db_backup['checksum_sha256'],
|
||||||
|
is_monthly=False
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.error("❌ Daily backup failed: db=%s, files=%s", db_job_id, files_job_id)
|
||||||
|
|
||||||
|
# Send failure notification
|
||||||
|
if not db_job_id:
|
||||||
|
await notifications.send_backup_failed(
|
||||||
|
job_id=0,
|
||||||
|
job_type='database',
|
||||||
|
error_message='Database backup failed - see logs for details'
|
||||||
|
)
|
||||||
|
|
||||||
|
if not files_job_id:
|
||||||
|
await notifications.send_backup_failed(
|
||||||
|
job_id=0,
|
||||||
|
job_type='files',
|
||||||
|
error_message='Files backup failed - see logs for details'
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Daily backup job error: %s", str(e), exc_info=True)
|
||||||
|
await notifications.send_backup_failed(
|
||||||
|
job_id=0,
|
||||||
|
job_type='full',
|
||||||
|
error_message=str(e)
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _monthly_backup_job(self):
|
||||||
|
"""Monthly full backup job (database uses plain SQL format)"""
|
||||||
|
logger.info("🔄 Starting monthly backup job...")
|
||||||
|
|
||||||
|
try:
|
||||||
|
start_time = datetime.now()
|
||||||
|
|
||||||
|
# Create full backup with is_monthly=True (uses plain SQL format)
|
||||||
|
db_job_id, files_job_id = await backup_service.create_full_backup(is_monthly=True)
|
||||||
|
|
||||||
|
end_time = datetime.now()
|
||||||
|
duration = (end_time - start_time).total_seconds()
|
||||||
|
|
||||||
|
if db_job_id and files_job_id:
|
||||||
|
logger.info("✅ Monthly backup completed: db=%s, files=%s (%.1fs)",
|
||||||
|
db_job_id, files_job_id, duration)
|
||||||
|
|
||||||
|
# Send success notification for database backup
|
||||||
|
db_backup = execute_query(
|
||||||
|
"SELECT * FROM backup_jobs WHERE id = %s",
|
||||||
|
(db_job_id,),
|
||||||
|
fetchone=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if db_backup:
|
||||||
|
await notifications.send_backup_success(
|
||||||
|
job_id=db_job_id,
|
||||||
|
job_type='database',
|
||||||
|
file_size_bytes=db_backup['file_size_bytes'],
|
||||||
|
duration_seconds=duration / 2,
|
||||||
|
checksum=db_backup['checksum_sha256'],
|
||||||
|
is_monthly=True
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.error("❌ Monthly backup failed: db=%s, files=%s", db_job_id, files_job_id)
|
||||||
|
|
||||||
|
await notifications.send_backup_failed(
|
||||||
|
job_id=0,
|
||||||
|
job_type='monthly',
|
||||||
|
error_message='Monthly backup failed - see logs for details'
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Monthly backup job error: %s", str(e), exc_info=True)
|
||||||
|
await notifications.send_backup_failed(
|
||||||
|
job_id=0,
|
||||||
|
job_type='monthly',
|
||||||
|
error_message=str(e)
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _offsite_upload_job(self):
|
||||||
|
"""Weekly offsite upload job - uploads all backups not yet uploaded"""
|
||||||
|
if not settings.OFFSITE_ENABLED:
|
||||||
|
logger.info("⏭️ Offsite upload skipped (OFFSITE_ENABLED=false)")
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info("☁️ Starting weekly offsite upload job...")
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Find all completed backups not yet uploaded
|
||||||
|
pending_backups = execute_query(
|
||||||
|
"""SELECT * FROM backup_jobs
|
||||||
|
WHERE status = 'completed'
|
||||||
|
AND offsite_uploaded_at IS NULL
|
||||||
|
AND offsite_retry_count < %s
|
||||||
|
ORDER BY completed_at ASC""",
|
||||||
|
(settings.OFFSITE_RETRY_MAX_ATTEMPTS,)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not pending_backups:
|
||||||
|
logger.info("✅ No pending backups for offsite upload")
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info("📦 Found %d backups pending offsite upload", len(pending_backups))
|
||||||
|
|
||||||
|
success_count = 0
|
||||||
|
fail_count = 0
|
||||||
|
|
||||||
|
for backup in pending_backups:
|
||||||
|
success = await backup_service.upload_offsite(backup['id'])
|
||||||
|
|
||||||
|
if success:
|
||||||
|
success_count += 1
|
||||||
|
|
||||||
|
# Send success notification
|
||||||
|
await notifications.send_offsite_success(
|
||||||
|
job_id=backup['id'],
|
||||||
|
backup_name=backup['file_path'].split('/')[-1],
|
||||||
|
file_size_bytes=backup['file_size_bytes']
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
fail_count += 1
|
||||||
|
|
||||||
|
# Get updated retry count
|
||||||
|
updated_backup = execute_query(
|
||||||
|
"SELECT offsite_retry_count FROM backup_jobs WHERE id = %s",
|
||||||
|
(backup['id'],),
|
||||||
|
fetchone=True
|
||||||
|
)
|
||||||
|
|
||||||
|
# Send failure notification
|
||||||
|
await notifications.send_offsite_failed(
|
||||||
|
job_id=backup['id'],
|
||||||
|
backup_name=backup['file_path'].split('/')[-1],
|
||||||
|
error_message='Offsite upload failed - will retry',
|
||||||
|
retry_count=updated_backup['offsite_retry_count']
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("✅ Offsite upload job completed: %d success, %d failed",
|
||||||
|
success_count, fail_count)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Offsite upload job error: %s", str(e), exc_info=True)
|
||||||
|
|
||||||
|
async def _offsite_retry_job(self):
|
||||||
|
"""Retry failed offsite uploads"""
|
||||||
|
if not settings.OFFSITE_ENABLED:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Find backups that failed offsite upload and haven't exceeded retry limit
|
||||||
|
retry_backups = execute_query(
|
||||||
|
"""SELECT * FROM backup_jobs
|
||||||
|
WHERE status = 'completed'
|
||||||
|
AND offsite_uploaded_at IS NULL
|
||||||
|
AND offsite_retry_count > 0
|
||||||
|
AND offsite_retry_count < %s
|
||||||
|
ORDER BY offsite_retry_count ASC, completed_at ASC
|
||||||
|
LIMIT 5""", # Limit to 5 retries per run
|
||||||
|
(settings.OFFSITE_RETRY_MAX_ATTEMPTS,)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not retry_backups:
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info("🔁 Retrying %d failed offsite uploads...", len(retry_backups))
|
||||||
|
|
||||||
|
for backup in retry_backups:
|
||||||
|
logger.info("🔁 Retry attempt %d/%d for backup %s",
|
||||||
|
backup['offsite_retry_count'] + 1,
|
||||||
|
settings.OFFSITE_RETRY_MAX_ATTEMPTS,
|
||||||
|
backup['id'])
|
||||||
|
|
||||||
|
success = await backup_service.upload_offsite(backup['id'])
|
||||||
|
|
||||||
|
if success:
|
||||||
|
logger.info("✅ Offsite upload succeeded on retry: backup %s", backup['id'])
|
||||||
|
|
||||||
|
await notifications.send_offsite_success(
|
||||||
|
job_id=backup['id'],
|
||||||
|
backup_name=backup['file_path'].split('/')[-1],
|
||||||
|
file_size_bytes=backup['file_size_bytes']
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _rotation_job(self):
|
||||||
|
"""Backup rotation job - removes expired backups"""
|
||||||
|
logger.info("🔄 Starting backup rotation job...")
|
||||||
|
|
||||||
|
try:
|
||||||
|
await backup_service.rotate_backups()
|
||||||
|
logger.info("✅ Backup rotation completed")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Backup rotation error: %s", str(e), exc_info=True)
|
||||||
|
|
||||||
|
async def _storage_check_job(self):
|
||||||
|
"""Storage usage check job - warns if storage is running low"""
|
||||||
|
logger.info("🔄 Starting storage check job...")
|
||||||
|
|
||||||
|
try:
|
||||||
|
stats = await backup_service.check_storage_usage()
|
||||||
|
|
||||||
|
if stats['warning']:
|
||||||
|
await notifications.send_storage_warning(
|
||||||
|
usage_pct=stats['usage_pct'],
|
||||||
|
used_gb=stats['total_size_gb'],
|
||||||
|
max_gb=settings.BACKUP_MAX_SIZE_GB
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("✅ Storage check completed: %.1f%% used (%.2f GB / %d GB)",
|
||||||
|
stats['usage_pct'], stats['total_size_gb'], settings.BACKUP_MAX_SIZE_GB)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Storage check error: %s", str(e), exc_info=True)
|
||||||
|
|
||||||
|
def _get_weekday_number(self, day_name: str) -> int:
|
||||||
|
"""Convert day name to APScheduler weekday number (0=Monday, 6=Sunday)"""
|
||||||
|
days = {
|
||||||
|
'monday': 0,
|
||||||
|
'tuesday': 1,
|
||||||
|
'wednesday': 2,
|
||||||
|
'thursday': 3,
|
||||||
|
'friday': 4,
|
||||||
|
'saturday': 5,
|
||||||
|
'sunday': 6
|
||||||
|
}
|
||||||
|
return days.get(day_name.lower(), 6) # Default to Sunday
|
||||||
|
|
||||||
|
|
||||||
|
# Singleton instance
|
||||||
|
backup_scheduler = BackupScheduler()
|
||||||
696
app/backups/backend/service.py
Normal file
696
app/backups/backend/service.py
Normal file
@ -0,0 +1,696 @@
|
|||||||
|
"""
|
||||||
|
Backup Service
|
||||||
|
Handles database and file backup operations, rotation, restore, and offsite uploads.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import logging
|
||||||
|
import hashlib
|
||||||
|
import tarfile
|
||||||
|
import subprocess
|
||||||
|
import fcntl
|
||||||
|
from pathlib import Path
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from typing import Optional, Dict, List, Tuple
|
||||||
|
import paramiko
|
||||||
|
from stat import S_ISDIR
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.core.database import execute_query, execute_insert, execute_update
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class BackupService:
|
||||||
|
"""Service for managing backup operations"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.backup_dir = Path(settings.BACKUP_STORAGE_PATH)
|
||||||
|
self.backup_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Subdirectories for different backup types
|
||||||
|
self.db_dir = self.backup_dir / "database"
|
||||||
|
self.files_dir = self.backup_dir / "files"
|
||||||
|
self.db_dir.mkdir(exist_ok=True)
|
||||||
|
self.files_dir.mkdir(exist_ok=True)
|
||||||
|
|
||||||
|
async def create_database_backup(self, is_monthly: bool = False) -> Optional[int]:
|
||||||
|
"""
|
||||||
|
Create PostgreSQL database backup using pg_dump
|
||||||
|
|
||||||
|
Args:
|
||||||
|
is_monthly: If True, creates plain SQL backup for readability
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
backup_job_id or None if failed
|
||||||
|
"""
|
||||||
|
if settings.BACKUP_DRY_RUN:
|
||||||
|
logger.info("🔄 DRY RUN: Would create database backup (monthly=%s)", is_monthly)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Determine format based on monthly flag
|
||||||
|
backup_format = settings.DB_MONTHLY_FORMAT if is_monthly else settings.DB_DAILY_FORMAT
|
||||||
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||||
|
filename = f"db_{timestamp}_{'monthly' if is_monthly else 'daily'}.{backup_format}"
|
||||||
|
backup_path = self.db_dir / filename
|
||||||
|
|
||||||
|
# Create backup job record
|
||||||
|
job_id = execute_insert(
|
||||||
|
"""INSERT INTO backup_jobs (job_type, status, backup_format, is_monthly, started_at)
|
||||||
|
VALUES (%s, %s, %s, %s, %s)""",
|
||||||
|
('database', 'running', backup_format, is_monthly, datetime.now())
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("🔄 Starting database backup: job_id=%s, format=%s, monthly=%s",
|
||||||
|
job_id, backup_format, is_monthly)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Build pg_dump command - connect via network to postgres service
|
||||||
|
env = os.environ.copy()
|
||||||
|
env['PGPASSWORD'] = settings.DATABASE_URL.split(':')[2].split('@')[0] # Extract password
|
||||||
|
|
||||||
|
# Parse database connection info from DATABASE_URL
|
||||||
|
# Format: postgresql://user:pass@host:port/dbname
|
||||||
|
db_parts = settings.DATABASE_URL.replace('postgresql://', '').split('@')
|
||||||
|
user_pass = db_parts[0].split(':')
|
||||||
|
host_db = db_parts[1].split('/')
|
||||||
|
|
||||||
|
user = user_pass[0]
|
||||||
|
password = user_pass[1] if len(user_pass) > 1 else ''
|
||||||
|
host = host_db[0].split(':')[0] if ':' in host_db[0] else host_db[0]
|
||||||
|
dbname = host_db[1] if len(host_db) > 1 else 'bmc_hub'
|
||||||
|
|
||||||
|
env['PGPASSWORD'] = password
|
||||||
|
|
||||||
|
if backup_format == 'dump':
|
||||||
|
# Compressed custom format (-Fc)
|
||||||
|
cmd = ['pg_dump', '-h', host, '-U', user, '-Fc', dbname]
|
||||||
|
else:
|
||||||
|
# Plain SQL format
|
||||||
|
cmd = ['pg_dump', '-h', host, '-U', user, dbname]
|
||||||
|
|
||||||
|
# Execute pg_dump and write to file
|
||||||
|
logger.info("📦 Executing: %s > %s", ' '.join(cmd), backup_path)
|
||||||
|
|
||||||
|
with open(backup_path, 'wb') as f:
|
||||||
|
result = subprocess.run(cmd, stdout=f, stderr=subprocess.PIPE, check=True, env=env)
|
||||||
|
|
||||||
|
# Calculate file size and checksum
|
||||||
|
file_size = backup_path.stat().st_size
|
||||||
|
checksum = self._calculate_checksum(backup_path)
|
||||||
|
|
||||||
|
# Calculate retention date
|
||||||
|
if is_monthly:
|
||||||
|
retention_until = datetime.now() + timedelta(days=settings.MONTHLY_KEEP_MONTHS * 30)
|
||||||
|
else:
|
||||||
|
retention_until = datetime.now() + timedelta(days=settings.RETENTION_DAYS)
|
||||||
|
|
||||||
|
# Update job record
|
||||||
|
execute_update(
|
||||||
|
"""UPDATE backup_jobs
|
||||||
|
SET status = %s, completed_at = %s, file_path = %s,
|
||||||
|
file_size_bytes = %s, checksum_sha256 = %s, retention_until = %s
|
||||||
|
WHERE id = %s""",
|
||||||
|
('completed', datetime.now(), str(backup_path), file_size, checksum,
|
||||||
|
retention_until.date(), job_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("✅ Database backup completed: %s (%.2f MB)",
|
||||||
|
filename, file_size / 1024 / 1024)
|
||||||
|
|
||||||
|
return job_id
|
||||||
|
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
error_msg = e.stderr.decode() if e.stderr else str(e)
|
||||||
|
logger.error("❌ Database backup failed: %s", error_msg)
|
||||||
|
|
||||||
|
execute_update(
|
||||||
|
"""UPDATE backup_jobs
|
||||||
|
SET status = %s, completed_at = %s, error_message = %s
|
||||||
|
WHERE id = %s""",
|
||||||
|
('failed', datetime.now(), error_msg, job_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Clean up partial backup file
|
||||||
|
if backup_path.exists():
|
||||||
|
backup_path.unlink()
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def create_files_backup(self) -> Optional[int]:
|
||||||
|
"""
|
||||||
|
Create tar.gz backup of file directories (uploads/, data/, logs/)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
backup_job_id or None if failed
|
||||||
|
"""
|
||||||
|
if settings.BACKUP_DRY_RUN:
|
||||||
|
logger.info("🔄 DRY RUN: Would create files backup")
|
||||||
|
return None
|
||||||
|
|
||||||
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||||
|
filename = f"files_{timestamp}.tar.gz"
|
||||||
|
backup_path = self.files_dir / filename
|
||||||
|
|
||||||
|
# Paths to backup (relative to project root)
|
||||||
|
base_path = Path.cwd()
|
||||||
|
paths_to_backup = []
|
||||||
|
|
||||||
|
if settings.BACKUP_INCLUDE_UPLOADS:
|
||||||
|
uploads_path = base_path / settings.UPLOAD_DIR
|
||||||
|
if uploads_path.exists():
|
||||||
|
paths_to_backup.append((uploads_path, 'uploads'))
|
||||||
|
|
||||||
|
if settings.BACKUP_INCLUDE_DATA:
|
||||||
|
data_path = base_path / 'data'
|
||||||
|
if data_path.exists():
|
||||||
|
paths_to_backup.append((data_path, 'data'))
|
||||||
|
|
||||||
|
if settings.BACKUP_INCLUDE_LOGS:
|
||||||
|
logs_path = base_path / 'logs'
|
||||||
|
if logs_path.exists():
|
||||||
|
paths_to_backup.append((logs_path, 'logs'))
|
||||||
|
|
||||||
|
if not paths_to_backup:
|
||||||
|
logger.warning("⚠️ No file directories to backup")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Create backup job record
|
||||||
|
job_id = execute_insert(
|
||||||
|
"""INSERT INTO backup_jobs
|
||||||
|
(job_type, status, backup_format, includes_uploads, includes_logs, includes_data, started_at)
|
||||||
|
VALUES (%s, %s, %s, %s, %s, %s, %s)""",
|
||||||
|
('files', 'running', 'tar.gz',
|
||||||
|
settings.BACKUP_INCLUDE_UPLOADS,
|
||||||
|
settings.BACKUP_INCLUDE_LOGS,
|
||||||
|
settings.BACKUP_INCLUDE_DATA,
|
||||||
|
datetime.now())
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("🔄 Starting files backup: job_id=%s, paths=%s",
|
||||||
|
job_id, [name for _, name in paths_to_backup])
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Exclude patterns
|
||||||
|
exclude_patterns = [
|
||||||
|
'__pycache__',
|
||||||
|
'*.pyc',
|
||||||
|
'*.pyo',
|
||||||
|
'*.pyd',
|
||||||
|
'.DS_Store',
|
||||||
|
'.git',
|
||||||
|
'backup', # Don't backup the backup directory itself!
|
||||||
|
]
|
||||||
|
|
||||||
|
# Create tar.gz archive
|
||||||
|
with tarfile.open(backup_path, 'w:gz') as tar:
|
||||||
|
for path, arcname in paths_to_backup:
|
||||||
|
tar.add(
|
||||||
|
path,
|
||||||
|
arcname=arcname,
|
||||||
|
recursive=True,
|
||||||
|
filter=lambda ti: None if any(
|
||||||
|
pattern in ti.name for pattern in exclude_patterns
|
||||||
|
) else ti
|
||||||
|
)
|
||||||
|
|
||||||
|
# Calculate file size and checksum
|
||||||
|
file_size = backup_path.stat().st_size
|
||||||
|
checksum = self._calculate_checksum(backup_path)
|
||||||
|
|
||||||
|
# Calculate retention date (files use daily retention)
|
||||||
|
retention_until = datetime.now() + timedelta(days=settings.RETENTION_DAYS)
|
||||||
|
|
||||||
|
# Update job record
|
||||||
|
execute_update(
|
||||||
|
"""UPDATE backup_jobs
|
||||||
|
SET status = %s, completed_at = %s, file_path = %s,
|
||||||
|
file_size_bytes = %s, checksum_sha256 = %s, retention_until = %s
|
||||||
|
WHERE id = %s""",
|
||||||
|
('completed', datetime.now(), str(backup_path), file_size, checksum,
|
||||||
|
retention_until.date(), job_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("✅ Files backup completed: %s (%.2f MB)",
|
||||||
|
filename, file_size / 1024 / 1024)
|
||||||
|
|
||||||
|
return job_id
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Files backup failed: %s", str(e))
|
||||||
|
|
||||||
|
execute_update(
|
||||||
|
"""UPDATE backup_jobs
|
||||||
|
SET status = %s, completed_at = %s, error_message = %s
|
||||||
|
WHERE id = %s""",
|
||||||
|
('failed', datetime.now(), str(e), job_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Clean up partial backup file
|
||||||
|
if backup_path.exists():
|
||||||
|
backup_path.unlink()
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def create_full_backup(self, is_monthly: bool = False) -> Tuple[Optional[int], Optional[int]]:
|
||||||
|
"""
|
||||||
|
Create full backup (database + files)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(db_job_id, files_job_id) tuple
|
||||||
|
"""
|
||||||
|
logger.info("🔄 Starting full backup (database + files)")
|
||||||
|
|
||||||
|
db_job_id = await self.create_database_backup(is_monthly=is_monthly)
|
||||||
|
files_job_id = await self.create_files_backup()
|
||||||
|
|
||||||
|
if db_job_id and files_job_id:
|
||||||
|
logger.info("✅ Full backup completed: db=%s, files=%s", db_job_id, files_job_id)
|
||||||
|
else:
|
||||||
|
logger.warning("⚠️ Full backup partially failed: db=%s, files=%s",
|
||||||
|
db_job_id, files_job_id)
|
||||||
|
|
||||||
|
return (db_job_id, files_job_id)
|
||||||
|
|
||||||
|
async def rotate_backups(self):
|
||||||
|
"""
|
||||||
|
Remove old backups based on retention policy:
|
||||||
|
- Daily backups: Keep for RETENTION_DAYS (default 30 days)
|
||||||
|
- Monthly backups: Keep for MONTHLY_KEEP_MONTHS (default 12 months)
|
||||||
|
"""
|
||||||
|
if settings.BACKUP_DRY_RUN:
|
||||||
|
logger.info("🔄 DRY RUN: Would rotate backups")
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info("🔄 Starting backup rotation")
|
||||||
|
|
||||||
|
# Find expired backups
|
||||||
|
expired_backups = execute_query(
|
||||||
|
"""SELECT id, file_path, is_monthly, retention_until
|
||||||
|
FROM backup_jobs
|
||||||
|
WHERE status = 'completed'
|
||||||
|
AND retention_until < CURRENT_DATE
|
||||||
|
ORDER BY retention_until ASC"""
|
||||||
|
)
|
||||||
|
|
||||||
|
deleted_count = 0
|
||||||
|
freed_bytes = 0
|
||||||
|
|
||||||
|
for backup in expired_backups:
|
||||||
|
file_path = Path(backup['file_path'])
|
||||||
|
|
||||||
|
if file_path.exists():
|
||||||
|
file_size = file_path.stat().st_size
|
||||||
|
file_path.unlink()
|
||||||
|
freed_bytes += file_size
|
||||||
|
logger.info("🗑️ Deleted expired backup: %s (%.2f MB, retention_until=%s)",
|
||||||
|
file_path.name, file_size / 1024 / 1024, backup['retention_until'])
|
||||||
|
|
||||||
|
# Delete from database
|
||||||
|
execute_update("DELETE FROM backup_jobs WHERE id = %s", (backup['id'],))
|
||||||
|
deleted_count += 1
|
||||||
|
|
||||||
|
if deleted_count > 0:
|
||||||
|
logger.info("✅ Rotation complete: deleted %d backups, freed %.2f MB",
|
||||||
|
deleted_count, freed_bytes / 1024 / 1024)
|
||||||
|
else:
|
||||||
|
logger.info("✅ Rotation complete: no expired backups")
|
||||||
|
|
||||||
|
async def restore_database(self, job_id: int) -> bool:
|
||||||
|
"""
|
||||||
|
Restore database from backup with maintenance mode
|
||||||
|
|
||||||
|
Args:
|
||||||
|
job_id: Backup job ID to restore from
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if successful, False otherwise
|
||||||
|
"""
|
||||||
|
if settings.BACKUP_READ_ONLY:
|
||||||
|
logger.error("❌ Restore blocked: BACKUP_READ_ONLY=true")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Get backup job
|
||||||
|
backup = execute_query(
|
||||||
|
"SELECT * FROM backup_jobs WHERE id = %s AND job_type = 'database'",
|
||||||
|
(job_id,),
|
||||||
|
fetchone=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if not backup:
|
||||||
|
logger.error("❌ Backup job not found: %s", job_id)
|
||||||
|
return False
|
||||||
|
|
||||||
|
backup_path = Path(backup['file_path'])
|
||||||
|
|
||||||
|
if not backup_path.exists():
|
||||||
|
logger.error("❌ Backup file not found: %s", backup_path)
|
||||||
|
return False
|
||||||
|
|
||||||
|
logger.info("🔄 Starting database restore from backup: %s", backup_path.name)
|
||||||
|
|
||||||
|
# Enable maintenance mode
|
||||||
|
await self.set_maintenance_mode(True, "Database restore i gang", eta_minutes=5)
|
||||||
|
|
||||||
|
# TODO: Stop scheduler (will be implemented in scheduler.py)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Verify checksum
|
||||||
|
current_checksum = self._calculate_checksum(backup_path)
|
||||||
|
if current_checksum != backup['checksum_sha256']:
|
||||||
|
raise ValueError(f"Checksum mismatch! Expected {backup['checksum_sha256']}, got {current_checksum}")
|
||||||
|
|
||||||
|
logger.info("✅ Checksum verified")
|
||||||
|
|
||||||
|
# Acquire file lock to prevent concurrent operations
|
||||||
|
lock_file = self.backup_dir / ".restore.lock"
|
||||||
|
with open(lock_file, 'w') as f:
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_EX)
|
||||||
|
|
||||||
|
# Parse database connection info
|
||||||
|
env = os.environ.copy()
|
||||||
|
db_parts = settings.DATABASE_URL.replace('postgresql://', '').split('@')
|
||||||
|
user_pass = db_parts[0].split(':')
|
||||||
|
host_db = db_parts[1].split('/')
|
||||||
|
|
||||||
|
user = user_pass[0]
|
||||||
|
password = user_pass[1] if len(user_pass) > 1 else ''
|
||||||
|
host = host_db[0].split(':')[0] if ':' in host_db[0] else host_db[0]
|
||||||
|
dbname = host_db[1] if len(host_db) > 1 else 'bmc_hub'
|
||||||
|
|
||||||
|
env['PGPASSWORD'] = password
|
||||||
|
|
||||||
|
# Build restore command based on format
|
||||||
|
if backup['backup_format'] == 'dump':
|
||||||
|
# Restore from compressed custom format
|
||||||
|
cmd = ['pg_restore', '-h', host, '-U', user, '-d', dbname, '--clean', '--if-exists']
|
||||||
|
|
||||||
|
logger.info("📥 Executing: %s < %s", ' '.join(cmd), backup_path)
|
||||||
|
|
||||||
|
with open(backup_path, 'rb') as f:
|
||||||
|
result = subprocess.run(cmd, stdin=f, stderr=subprocess.PIPE, check=True, env=env)
|
||||||
|
|
||||||
|
else:
|
||||||
|
# Restore from plain SQL
|
||||||
|
cmd = ['psql', '-h', host, '-U', user, '-d', dbname]
|
||||||
|
|
||||||
|
logger.info("📥 Executing: %s < %s", ' '.join(cmd), backup_path)
|
||||||
|
|
||||||
|
with open(backup_path, 'rb') as f:
|
||||||
|
result = subprocess.run(cmd, stdin=f, stderr=subprocess.PIPE, check=True, env=env)
|
||||||
|
|
||||||
|
# Release file lock
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
|
||||||
|
|
||||||
|
logger.info("✅ Database restore completed successfully")
|
||||||
|
|
||||||
|
# Log notification
|
||||||
|
execute_insert(
|
||||||
|
"""INSERT INTO backup_notifications (backup_job_id, event_type, message)
|
||||||
|
VALUES (%s, %s, %s)""",
|
||||||
|
(job_id, 'restore_started', f'Database restored from backup: {backup_path.name}')
|
||||||
|
)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Database restore failed: %s", str(e))
|
||||||
|
return False
|
||||||
|
|
||||||
|
finally:
|
||||||
|
# Disable maintenance mode
|
||||||
|
await self.set_maintenance_mode(False)
|
||||||
|
|
||||||
|
# TODO: Restart scheduler (will be implemented in scheduler.py)
|
||||||
|
|
||||||
|
# Clean up lock file
|
||||||
|
if lock_file.exists():
|
||||||
|
lock_file.unlink()
|
||||||
|
|
||||||
|
async def restore_files(self, job_id: int) -> bool:
|
||||||
|
"""
|
||||||
|
Restore files from tar.gz backup
|
||||||
|
|
||||||
|
Args:
|
||||||
|
job_id: Backup job ID to restore from
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if successful, False otherwise
|
||||||
|
"""
|
||||||
|
if settings.BACKUP_READ_ONLY:
|
||||||
|
logger.error("❌ Restore blocked: BACKUP_READ_ONLY=true")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Get backup job
|
||||||
|
backup = execute_query(
|
||||||
|
"SELECT * FROM backup_jobs WHERE id = %s AND job_type = 'files'",
|
||||||
|
(job_id,),
|
||||||
|
fetchone=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if not backup:
|
||||||
|
logger.error("❌ Backup job not found: %s", job_id)
|
||||||
|
return False
|
||||||
|
|
||||||
|
backup_path = Path(backup['file_path'])
|
||||||
|
|
||||||
|
if not backup_path.exists():
|
||||||
|
logger.error("❌ Backup file not found: %s", backup_path)
|
||||||
|
return False
|
||||||
|
|
||||||
|
logger.info("🔄 Starting files restore from backup: %s", backup_path.name)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Verify checksum
|
||||||
|
current_checksum = self._calculate_checksum(backup_path)
|
||||||
|
if current_checksum != backup['checksum_sha256']:
|
||||||
|
raise ValueError(f"Checksum mismatch! Expected {backup['checksum_sha256']}, got {current_checksum}")
|
||||||
|
|
||||||
|
logger.info("✅ Checksum verified")
|
||||||
|
|
||||||
|
# Acquire file lock
|
||||||
|
lock_file = self.backup_dir / ".restore_files.lock"
|
||||||
|
with open(lock_file, 'w') as f:
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_EX)
|
||||||
|
|
||||||
|
# Extract tar.gz to project root
|
||||||
|
base_path = Path.cwd()
|
||||||
|
|
||||||
|
with tarfile.open(backup_path, 'r:gz') as tar:
|
||||||
|
# Extract all files, excluding backup directory
|
||||||
|
members = [m for m in tar.getmembers() if 'backup' not in m.name]
|
||||||
|
tar.extractall(path=base_path, members=members)
|
||||||
|
|
||||||
|
# Release file lock
|
||||||
|
fcntl.flock(f.fileno(), fcntl.LOCK_UN)
|
||||||
|
|
||||||
|
logger.info("✅ Files restore completed successfully")
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Files restore failed: %s", str(e))
|
||||||
|
return False
|
||||||
|
|
||||||
|
finally:
|
||||||
|
# Clean up lock file
|
||||||
|
if lock_file.exists():
|
||||||
|
lock_file.unlink()
|
||||||
|
|
||||||
|
async def upload_offsite(self, job_id: int) -> bool:
|
||||||
|
"""
|
||||||
|
Upload backup to offsite location via SFTP/SSH
|
||||||
|
|
||||||
|
Args:
|
||||||
|
job_id: Backup job ID to upload
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if successful, False otherwise
|
||||||
|
"""
|
||||||
|
if not settings.OFFSITE_ENABLED:
|
||||||
|
logger.info("⏭️ Offsite upload disabled")
|
||||||
|
return False
|
||||||
|
|
||||||
|
if settings.BACKUP_DRY_RUN:
|
||||||
|
logger.info("🔄 DRY RUN: Would upload backup to offsite")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Get backup job
|
||||||
|
backup = execute_query(
|
||||||
|
"SELECT * FROM backup_jobs WHERE id = %s",
|
||||||
|
(job_id,),
|
||||||
|
fetchone=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if not backup:
|
||||||
|
logger.error("❌ Backup job not found: %s", job_id)
|
||||||
|
return False
|
||||||
|
|
||||||
|
if backup['offsite_uploaded_at']:
|
||||||
|
logger.info("⏭️ Backup already uploaded to offsite: %s", job_id)
|
||||||
|
return True
|
||||||
|
|
||||||
|
backup_path = Path(backup['file_path'])
|
||||||
|
|
||||||
|
if not backup_path.exists():
|
||||||
|
logger.error("❌ Backup file not found: %s", backup_path)
|
||||||
|
return False
|
||||||
|
|
||||||
|
logger.info("☁️ Starting offsite upload: %s to %s:%s",
|
||||||
|
backup_path.name, settings.SFTP_HOST, settings.SFTP_REMOTE_PATH)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Connect via SFTP
|
||||||
|
transport = paramiko.Transport((settings.SFTP_HOST, settings.SFTP_PORT))
|
||||||
|
|
||||||
|
if settings.SSH_KEY_PATH:
|
||||||
|
# Use SSH key authentication
|
||||||
|
private_key = paramiko.RSAKey.from_private_key_file(settings.SSH_KEY_PATH)
|
||||||
|
transport.connect(username=settings.SFTP_USER, pkey=private_key)
|
||||||
|
else:
|
||||||
|
# Use password authentication
|
||||||
|
transport.connect(username=settings.SFTP_USER, password=settings.SFTP_PASSWORD)
|
||||||
|
|
||||||
|
sftp = paramiko.SFTPClient.from_transport(transport)
|
||||||
|
|
||||||
|
# Create remote directory if needed
|
||||||
|
remote_path = settings.SFTP_REMOTE_PATH
|
||||||
|
self._ensure_remote_directory(sftp, remote_path)
|
||||||
|
|
||||||
|
# Upload file
|
||||||
|
remote_file = f"{remote_path}/{backup_path.name}"
|
||||||
|
sftp.put(str(backup_path), remote_file)
|
||||||
|
|
||||||
|
# Verify upload
|
||||||
|
remote_stat = sftp.stat(remote_file)
|
||||||
|
local_size = backup_path.stat().st_size
|
||||||
|
|
||||||
|
if remote_stat.st_size != local_size:
|
||||||
|
raise ValueError(f"Upload verification failed: remote size {remote_stat.st_size} != local size {local_size}")
|
||||||
|
|
||||||
|
# Close connection
|
||||||
|
sftp.close()
|
||||||
|
transport.close()
|
||||||
|
|
||||||
|
# Update job record
|
||||||
|
execute_update(
|
||||||
|
"""UPDATE backup_jobs
|
||||||
|
SET offsite_uploaded_at = %s, offsite_retry_count = 0
|
||||||
|
WHERE id = %s""",
|
||||||
|
(datetime.now(), job_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("✅ Offsite upload completed: %s", backup_path.name)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("❌ Offsite upload failed: %s", str(e))
|
||||||
|
|
||||||
|
# Increment retry count
|
||||||
|
execute_update(
|
||||||
|
"""UPDATE backup_jobs
|
||||||
|
SET offsite_retry_count = offsite_retry_count + 1
|
||||||
|
WHERE id = %s""",
|
||||||
|
(job_id,)
|
||||||
|
)
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def check_storage_usage(self) -> Dict[str, any]:
|
||||||
|
"""
|
||||||
|
Check backup storage usage and warn if exceeding threshold
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with storage statistics
|
||||||
|
"""
|
||||||
|
total_size = 0
|
||||||
|
file_count = 0
|
||||||
|
|
||||||
|
for backup_file in self.backup_dir.rglob('*'):
|
||||||
|
if backup_file.is_file() and not backup_file.name.startswith('.'):
|
||||||
|
total_size += backup_file.stat().st_size
|
||||||
|
file_count += 1
|
||||||
|
|
||||||
|
max_size_bytes = settings.BACKUP_MAX_SIZE_GB * 1024 * 1024 * 1024
|
||||||
|
usage_pct = (total_size / max_size_bytes) * 100 if max_size_bytes > 0 else 0
|
||||||
|
|
||||||
|
stats = {
|
||||||
|
'total_size_bytes': total_size,
|
||||||
|
'total_size_gb': total_size / 1024 / 1024 / 1024,
|
||||||
|
'max_size_gb': settings.BACKUP_MAX_SIZE_GB,
|
||||||
|
'usage_pct': usage_pct,
|
||||||
|
'file_count': file_count,
|
||||||
|
'warning': usage_pct >= settings.STORAGE_WARNING_THRESHOLD_PCT
|
||||||
|
}
|
||||||
|
|
||||||
|
if stats['warning']:
|
||||||
|
logger.warning("⚠️ Backup storage usage high: %.1f%% (%.2f GB / %d GB)",
|
||||||
|
usage_pct, stats['total_size_gb'], settings.BACKUP_MAX_SIZE_GB)
|
||||||
|
|
||||||
|
# Log notification
|
||||||
|
execute_insert(
|
||||||
|
"""INSERT INTO backup_notifications (event_type, message)
|
||||||
|
VALUES (%s, %s)""",
|
||||||
|
('storage_low',
|
||||||
|
f"Backup storage usage at {usage_pct:.1f}% ({stats['total_size_gb']:.2f} GB / {settings.BACKUP_MAX_SIZE_GB} GB)")
|
||||||
|
)
|
||||||
|
|
||||||
|
return stats
|
||||||
|
|
||||||
|
async def set_maintenance_mode(self, enabled: bool, message: str = None, eta_minutes: int = None):
|
||||||
|
"""
|
||||||
|
Enable or disable system maintenance mode
|
||||||
|
|
||||||
|
Args:
|
||||||
|
enabled: True to enable maintenance mode, False to disable
|
||||||
|
message: Custom maintenance message
|
||||||
|
eta_minutes: Estimated time to completion in minutes
|
||||||
|
"""
|
||||||
|
if message is None:
|
||||||
|
message = "System under vedligeholdelse" if enabled else ""
|
||||||
|
|
||||||
|
execute_update(
|
||||||
|
"""UPDATE system_status
|
||||||
|
SET maintenance_mode = %s, maintenance_message = %s,
|
||||||
|
maintenance_eta_minutes = %s, updated_at = %s
|
||||||
|
WHERE id = 1""",
|
||||||
|
(enabled, message, eta_minutes, datetime.now())
|
||||||
|
)
|
||||||
|
|
||||||
|
if enabled:
|
||||||
|
logger.warning("🔧 Maintenance mode ENABLED: %s (ETA: %s min)", message, eta_minutes)
|
||||||
|
else:
|
||||||
|
logger.info("✅ Maintenance mode DISABLED")
|
||||||
|
|
||||||
|
def _calculate_checksum(self, file_path: Path) -> str:
|
||||||
|
"""Calculate SHA256 checksum of file"""
|
||||||
|
sha256_hash = hashlib.sha256()
|
||||||
|
|
||||||
|
with open(file_path, "rb") as f:
|
||||||
|
for byte_block in iter(lambda: f.read(4096), b""):
|
||||||
|
sha256_hash.update(byte_block)
|
||||||
|
|
||||||
|
return sha256_hash.hexdigest()
|
||||||
|
|
||||||
|
def _ensure_remote_directory(self, sftp: paramiko.SFTPClient, path: str):
|
||||||
|
"""Create remote directory if it doesn't exist (recursive)"""
|
||||||
|
dirs = []
|
||||||
|
current = path
|
||||||
|
|
||||||
|
while current != '/':
|
||||||
|
dirs.append(current)
|
||||||
|
current = os.path.dirname(current)
|
||||||
|
|
||||||
|
dirs.reverse()
|
||||||
|
|
||||||
|
for dir_path in dirs:
|
||||||
|
try:
|
||||||
|
sftp.stat(dir_path)
|
||||||
|
except FileNotFoundError:
|
||||||
|
sftp.mkdir(dir_path)
|
||||||
|
logger.info("📁 Created remote directory: %s", dir_path)
|
||||||
|
|
||||||
|
|
||||||
|
# Singleton instance
|
||||||
|
backup_service = BackupService()
|
||||||
1
app/backups/frontend/__init__.py
Normal file
1
app/backups/frontend/__init__.py
Normal file
@ -0,0 +1 @@
|
|||||||
|
"""Backup frontend views and templates."""
|
||||||
20
app/backups/frontend/views.py
Normal file
20
app/backups/frontend/views.py
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
"""
|
||||||
|
Backup Frontend Views
|
||||||
|
Serves HTML pages for backup system dashboard
|
||||||
|
"""
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Request
|
||||||
|
from fastapi.templating import Jinja2Templates
|
||||||
|
from fastapi.responses import HTMLResponse
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
templates = Jinja2Templates(directory="app")
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/backups", response_class=HTMLResponse)
|
||||||
|
async def backups_dashboard(request: Request):
|
||||||
|
"""Backup system dashboard page"""
|
||||||
|
return templates.TemplateResponse("backups/templates/index.html", {
|
||||||
|
"request": request,
|
||||||
|
"title": "Backup System"
|
||||||
|
})
|
||||||
778
app/backups/templates/index.html
Normal file
778
app/backups/templates/index.html
Normal file
@ -0,0 +1,778 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="da" data-bs-theme="light">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>Backup System - BMC Hub</title>
|
||||||
|
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
|
||||||
|
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.0/font/bootstrap-icons.css">
|
||||||
|
<style>
|
||||||
|
:root {
|
||||||
|
--primary-color: #0f4c75;
|
||||||
|
--secondary-color: #3282b8;
|
||||||
|
--accent-color: #bbe1fa;
|
||||||
|
--success-color: #28a745;
|
||||||
|
--warning-color: #ffc107;
|
||||||
|
--danger-color: #dc3545;
|
||||||
|
--light-bg: #f8f9fa;
|
||||||
|
--dark-bg: #1b2838;
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-bs-theme="dark"] {
|
||||||
|
--light-bg: #1b2838;
|
||||||
|
--primary-color: #3282b8;
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
background-color: var(--light-bg);
|
||||||
|
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||||||
|
}
|
||||||
|
|
||||||
|
.navbar {
|
||||||
|
background-color: var(--primary-color) !important;
|
||||||
|
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
||||||
|
}
|
||||||
|
|
||||||
|
.card {
|
||||||
|
border: none;
|
||||||
|
border-radius: 10px;
|
||||||
|
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||||||
|
margin-bottom: 1.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.card-header {
|
||||||
|
background-color: var(--primary-color);
|
||||||
|
color: white;
|
||||||
|
border-radius: 10px 10px 0 0 !important;
|
||||||
|
padding: 1rem 1.25rem;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
.stat-card {
|
||||||
|
text-align: center;
|
||||||
|
padding: 1.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.stat-card .stat-value {
|
||||||
|
font-size: 2.5rem;
|
||||||
|
font-weight: bold;
|
||||||
|
color: var(--primary-color);
|
||||||
|
}
|
||||||
|
|
||||||
|
.stat-card .stat-label {
|
||||||
|
font-size: 0.9rem;
|
||||||
|
color: #6c757d;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
.badge-type {
|
||||||
|
font-size: 0.85rem;
|
||||||
|
padding: 0.4rem 0.8rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.progress {
|
||||||
|
height: 25px;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.btn-action {
|
||||||
|
margin: 0.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.offsite-badge {
|
||||||
|
font-size: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.notification-item {
|
||||||
|
border-left: 4px solid;
|
||||||
|
padding-left: 1rem;
|
||||||
|
margin-bottom: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.notification-item.backup_failed {
|
||||||
|
border-color: var(--danger-color);
|
||||||
|
}
|
||||||
|
|
||||||
|
.notification-item.storage_low {
|
||||||
|
border-color: var(--warning-color);
|
||||||
|
}
|
||||||
|
|
||||||
|
.notification-item.backup_success {
|
||||||
|
border-color: var(--success-color);
|
||||||
|
}
|
||||||
|
|
||||||
|
.theme-toggle {
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 1.25rem;
|
||||||
|
color: white;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<!-- Navbar -->
|
||||||
|
<nav class="navbar navbar-expand-lg navbar-dark">
|
||||||
|
<div class="container-fluid">
|
||||||
|
<a class="navbar-brand" href="/">
|
||||||
|
<i class="bi bi-hdd-network"></i> BMC Hub - Backup System
|
||||||
|
</a>
|
||||||
|
<div class="d-flex align-items-center">
|
||||||
|
<span class="theme-toggle me-3" onclick="toggleTheme()">
|
||||||
|
<i class="bi bi-moon-stars" id="theme-icon"></i>
|
||||||
|
</span>
|
||||||
|
<a href="/api/docs" class="btn btn-outline-light btn-sm">
|
||||||
|
<i class="bi bi-code-square"></i> API Docs
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
<div class="container-fluid py-4">
|
||||||
|
<!-- Stats Row -->
|
||||||
|
<div class="row mb-4">
|
||||||
|
<div class="col-md-3">
|
||||||
|
<div class="card stat-card">
|
||||||
|
<div class="stat-value" id="total-backups">-</div>
|
||||||
|
<div class="stat-label">Total Backups</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-3">
|
||||||
|
<div class="card stat-card">
|
||||||
|
<div class="stat-value text-success" id="completed-backups">-</div>
|
||||||
|
<div class="stat-label">Completed</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-3">
|
||||||
|
<div class="card stat-card">
|
||||||
|
<div class="stat-value text-warning" id="pending-offsite">-</div>
|
||||||
|
<div class="stat-label">Pending Offsite</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-3">
|
||||||
|
<div class="card stat-card">
|
||||||
|
<div class="stat-value" id="storage-usage">-</div>
|
||||||
|
<div class="stat-label">Storage Used</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Storage Usage -->
|
||||||
|
<div class="row mb-4">
|
||||||
|
<div class="col-12">
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-header">
|
||||||
|
<i class="bi bi-hdd"></i> Storage Usage
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<div class="progress" id="storage-progress">
|
||||||
|
<div class="progress-bar" role="progressbar" style="width: 0%" id="storage-bar">
|
||||||
|
0%
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<p class="text-muted mt-2 mb-0" id="storage-details">Loading...</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Actions and Scheduler Status -->
|
||||||
|
<div class="row mb-4">
|
||||||
|
<div class="col-md-8">
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-header">
|
||||||
|
<i class="bi bi-play-circle"></i> Manual Backup
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<form id="backup-form">
|
||||||
|
<div class="row g-3 align-items-end">
|
||||||
|
<div class="col-md-4">
|
||||||
|
<label class="form-label">Backup Type</label>
|
||||||
|
<select class="form-select" id="backup-type">
|
||||||
|
<option value="full">Full (Database + Files)</option>
|
||||||
|
<option value="database">Database Only</option>
|
||||||
|
<option value="files">Files Only</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-4">
|
||||||
|
<div class="form-check mt-4">
|
||||||
|
<input class="form-check-input" type="checkbox" id="is-monthly">
|
||||||
|
<label class="form-check-label" for="is-monthly">
|
||||||
|
Monthly Backup (SQL Format)
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-4">
|
||||||
|
<button type="submit" class="btn btn-primary w-100">
|
||||||
|
<i class="bi bi-download"></i> Create Backup
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
<div id="backup-result" class="mt-3"></div>
|
||||||
|
|
||||||
|
<hr class="my-4">
|
||||||
|
|
||||||
|
<!-- Upload Backup Form -->
|
||||||
|
<h6 class="mb-3"><i class="bi bi-cloud-upload"></i> Upload Backup</h6>
|
||||||
|
<form id="upload-form" onsubmit="uploadBackup(event)">
|
||||||
|
<div class="row g-2">
|
||||||
|
<div class="col-md-6">
|
||||||
|
<label for="backup-file" class="form-label">Backup File</label>
|
||||||
|
<input type="file" class="form-control" id="backup-file" required
|
||||||
|
accept=".dump,.sql,.sql.gz,.tar.gz,.tgz">
|
||||||
|
</div>
|
||||||
|
<div class="col-md-6">
|
||||||
|
<label for="upload-type" class="form-label">Backup Type</label>
|
||||||
|
<select class="form-select" id="upload-type" required>
|
||||||
|
<option value="database">Database</option>
|
||||||
|
<option value="files">Files</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-6">
|
||||||
|
<div class="form-check mt-4">
|
||||||
|
<input class="form-check-input" type="checkbox" id="upload-monthly">
|
||||||
|
<label class="form-check-label" for="upload-monthly">
|
||||||
|
Monthly Backup
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-6">
|
||||||
|
<button type="submit" class="btn btn-success w-100 mt-4">
|
||||||
|
<i class="bi bi-upload"></i> Upload Backup
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
<div id="upload-result" class="mt-3"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-md-4">
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-header">
|
||||||
|
<i class="bi bi-clock-history"></i> Scheduler Status
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<div id="scheduler-status">
|
||||||
|
<div class="spinner-border spinner-border-sm" role="status"></div>
|
||||||
|
<span class="ms-2">Loading...</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Backup History -->
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-md-8">
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-header d-flex justify-content-between align-items-center">
|
||||||
|
<span><i class="bi bi-clock-history"></i> Backup History</span>
|
||||||
|
<button class="btn btn-light btn-sm" onclick="refreshBackups()">
|
||||||
|
<i class="bi bi-arrow-clockwise"></i> Refresh
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<div class="table-responsive">
|
||||||
|
<table class="table table-hover">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>ID</th>
|
||||||
|
<th>Type</th>
|
||||||
|
<th>Format</th>
|
||||||
|
<th>Size</th>
|
||||||
|
<th>Status</th>
|
||||||
|
<th>Offsite</th>
|
||||||
|
<th>Created</th>
|
||||||
|
<th>Actions</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody id="backups-table">
|
||||||
|
<tr>
|
||||||
|
<td colspan="8" class="text-center">
|
||||||
|
<div class="spinner-border" role="status"></div>
|
||||||
|
<p class="mt-2">Loading backups...</p>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Notifications -->
|
||||||
|
<div class="col-md-4">
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-header">
|
||||||
|
<i class="bi bi-bell"></i> Recent Notifications
|
||||||
|
</div>
|
||||||
|
<div class="card-body" style="max-height: 500px; overflow-y: auto;">
|
||||||
|
<div id="notifications-list">
|
||||||
|
<div class="text-center">
|
||||||
|
<div class="spinner-border spinner-border-sm" role="status"></div>
|
||||||
|
<p class="mt-2 text-muted">Loading...</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Restore Confirmation Modal -->
|
||||||
|
<div class="modal fade" id="restoreModal" tabindex="-1">
|
||||||
|
<div class="modal-dialog">
|
||||||
|
<div class="modal-content">
|
||||||
|
<div class="modal-header bg-warning">
|
||||||
|
<h5 class="modal-title">
|
||||||
|
<i class="bi bi-exclamation-triangle"></i> Bekræft Restore
|
||||||
|
</h5>
|
||||||
|
<button type="button" class="btn-close" data-bs-dismiss="modal"></button>
|
||||||
|
</div>
|
||||||
|
<div class="modal-body">
|
||||||
|
<div class="alert alert-danger">
|
||||||
|
<strong>ADVARSEL:</strong> Systemet vil blive lukket ned under restore-processen.
|
||||||
|
Alle aktive brugere vil miste forbindelsen.
|
||||||
|
</div>
|
||||||
|
<p>Er du sikker på, at du vil gendanne fra denne backup?</p>
|
||||||
|
<p class="text-muted mb-0">
|
||||||
|
<strong>Backup ID:</strong> <span id="restore-job-id"></span><br>
|
||||||
|
<strong>Type:</strong> <span id="restore-job-type"></span><br>
|
||||||
|
<strong>Estimeret tid:</strong> 5-10 minutter
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div class="modal-footer">
|
||||||
|
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Annuller</button>
|
||||||
|
<button type="button" class="btn btn-danger" onclick="confirmRestore()">
|
||||||
|
<i class="bi bi-arrow-counterclockwise"></i> Gendan Nu
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
|
||||||
|
<script>
|
||||||
|
let selectedJobId = null;
|
||||||
|
let restoreModal = null;
|
||||||
|
|
||||||
|
// Initialize
|
||||||
|
document.addEventListener('DOMContentLoaded', function() {
|
||||||
|
restoreModal = new bootstrap.Modal(document.getElementById('restoreModal'));
|
||||||
|
loadDashboard();
|
||||||
|
|
||||||
|
// Refresh every 30 seconds
|
||||||
|
setInterval(loadDashboard, 30000);
|
||||||
|
|
||||||
|
// Setup backup form
|
||||||
|
document.getElementById('backup-form').addEventListener('submit', createBackup);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Load all dashboard data
|
||||||
|
async function loadDashboard() {
|
||||||
|
await Promise.all([
|
||||||
|
loadBackups(),
|
||||||
|
loadStorageStats(),
|
||||||
|
loadNotifications(),
|
||||||
|
loadSchedulerStatus()
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load backups list
|
||||||
|
async function loadBackups() {
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/v1/backups/jobs?limit=50');
|
||||||
|
const backups = await response.json();
|
||||||
|
|
||||||
|
const tbody = document.getElementById('backups-table');
|
||||||
|
|
||||||
|
if (backups.length === 0) {
|
||||||
|
tbody.innerHTML = '<tr><td colspan="8" class="text-center text-muted">No backups found</td></tr>';
|
||||||
|
updateStats(backups);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
tbody.innerHTML = backups.map(b => `
|
||||||
|
<tr>
|
||||||
|
<td><strong>#${b.id}</strong></td>
|
||||||
|
<td>
|
||||||
|
<span class="badge badge-type ${getTypeBadgeClass(b.job_type)}">
|
||||||
|
${b.job_type} ${b.is_monthly ? '(Monthly)' : ''}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td><code>${b.backup_format}</code></td>
|
||||||
|
<td>${formatBytes(b.file_size_bytes)}</td>
|
||||||
|
<td>${getStatusBadge(b.status)}</td>
|
||||||
|
<td>${getOffsiteBadge(b)}</td>
|
||||||
|
<td>${formatDate(b.created_at)}</td>
|
||||||
|
<td>
|
||||||
|
${b.status === 'completed' ? `
|
||||||
|
<button class="btn btn-sm btn-warning btn-action" onclick="showRestore(${b.id}, '${b.job_type}')" title="Restore">
|
||||||
|
<i class="bi bi-arrow-counterclockwise"></i>
|
||||||
|
</button>
|
||||||
|
${b.offsite_uploaded_at === null ? `
|
||||||
|
<button class="btn btn-sm btn-info btn-action" onclick="uploadOffsite(${b.id})" title="Upload Offsite">
|
||||||
|
<i class="bi bi-cloud-upload"></i>
|
||||||
|
</button>
|
||||||
|
` : ''}
|
||||||
|
` : ''}
|
||||||
|
<button class="btn btn-sm btn-danger btn-action" onclick="deleteBackup(${b.id})" title="Delete">
|
||||||
|
<i class="bi bi-trash"></i>
|
||||||
|
</button>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
`).join('');
|
||||||
|
|
||||||
|
updateStats(backups);
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Load backups error:', error);
|
||||||
|
document.getElementById('backups-table').innerHTML =
|
||||||
|
'<tr><td colspan="8" class="text-center text-danger">Failed to load backups</td></tr>';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load storage stats
|
||||||
|
async function loadStorageStats() {
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/v1/backups/storage');
|
||||||
|
const stats = await response.json();
|
||||||
|
|
||||||
|
const bar = document.getElementById('storage-bar');
|
||||||
|
const pct = Math.min(stats.usage_pct, 100);
|
||||||
|
bar.style.width = pct + '%';
|
||||||
|
bar.textContent = pct.toFixed(1) + '%';
|
||||||
|
|
||||||
|
if (stats.warning) {
|
||||||
|
bar.classList.remove('bg-success', 'bg-info');
|
||||||
|
bar.classList.add('bg-danger');
|
||||||
|
} else if (pct > 60) {
|
||||||
|
bar.classList.remove('bg-success', 'bg-danger');
|
||||||
|
bar.classList.add('bg-warning');
|
||||||
|
} else {
|
||||||
|
bar.classList.remove('bg-warning', 'bg-danger');
|
||||||
|
bar.classList.add('bg-success');
|
||||||
|
}
|
||||||
|
|
||||||
|
document.getElementById('storage-details').textContent =
|
||||||
|
`${stats.total_size_gb.toFixed(2)} GB used of ${stats.max_size_gb} GB (${stats.file_count} files)`;
|
||||||
|
|
||||||
|
document.getElementById('storage-usage').textContent = stats.total_size_gb.toFixed(1) + ' GB';
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Load storage stats error:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load notifications
|
||||||
|
async function loadNotifications() {
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/v1/backups/notifications?limit=10');
|
||||||
|
const notifications = await response.json();
|
||||||
|
|
||||||
|
const container = document.getElementById('notifications-list');
|
||||||
|
|
||||||
|
if (notifications.length === 0) {
|
||||||
|
container.innerHTML = '<p class="text-muted text-center">No notifications</p>';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
container.innerHTML = notifications.map(n => `
|
||||||
|
<div class="notification-item ${n.event_type} ${n.acknowledged ? 'opacity-50' : ''}">
|
||||||
|
<small class="text-muted">${formatDate(n.sent_at)}</small>
|
||||||
|
<p class="mb-1">${n.message}</p>
|
||||||
|
${!n.acknowledged ? `
|
||||||
|
<button class="btn btn-sm btn-outline-secondary" onclick="acknowledgeNotification(${n.id})">
|
||||||
|
<i class="bi bi-check"></i> Acknowledge
|
||||||
|
</button>
|
||||||
|
` : '<small class="text-success"><i class="bi bi-check-circle"></i> Acknowledged</small>'}
|
||||||
|
</div>
|
||||||
|
`).join('');
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Load notifications error:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load scheduler status
|
||||||
|
async function loadSchedulerStatus() {
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/v1/backups/scheduler/status');
|
||||||
|
const status = await response.json();
|
||||||
|
|
||||||
|
const container = document.getElementById('scheduler-status');
|
||||||
|
|
||||||
|
if (!status.running) {
|
||||||
|
container.innerHTML = `
|
||||||
|
<div class="alert alert-warning mb-0">
|
||||||
|
<i class="bi bi-exclamation-triangle"></i> Scheduler not running
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
container.innerHTML = `
|
||||||
|
<div class="alert alert-success mb-0">
|
||||||
|
<i class="bi bi-check-circle"></i> Active
|
||||||
|
</div>
|
||||||
|
<small class="text-muted">Next jobs:</small>
|
||||||
|
<ul class="list-unstyled mb-0 mt-1">
|
||||||
|
${status.jobs.slice(0, 3).map(j => `
|
||||||
|
<li><small>${j.name}: ${j.next_run ? formatDate(j.next_run) : 'N/A'}</small></li>
|
||||||
|
`).join('')}
|
||||||
|
</ul>
|
||||||
|
`;
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Load scheduler status error:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create manual backup
|
||||||
|
async function createBackup(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
|
||||||
|
const type = document.getElementById('backup-type').value;
|
||||||
|
const isMonthly = document.getElementById('is-monthly').checked;
|
||||||
|
const resultDiv = document.getElementById('backup-result');
|
||||||
|
|
||||||
|
resultDiv.innerHTML = '<div class="alert alert-info"><i class="bi bi-hourglass-split"></i> Creating backup...</div>';
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/v1/backups', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {'Content-Type': 'application/json'},
|
||||||
|
body: JSON.stringify({job_type: type, is_monthly: isMonthly})
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await response.json();
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
resultDiv.innerHTML = `<div class="alert alert-success">${result.message}</div>`;
|
||||||
|
setTimeout(() => loadBackups(), 2000);
|
||||||
|
} else {
|
||||||
|
resultDiv.innerHTML = `<div class="alert alert-danger">Error: ${result.detail}</div>`;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
resultDiv.innerHTML = `<div class="alert alert-danger">Error: ${error.message}</div>`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload backup
|
||||||
|
async function uploadBackup(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
|
||||||
|
const fileInput = document.getElementById('backup-file');
|
||||||
|
const type = document.getElementById('upload-type').value;
|
||||||
|
const isMonthly = document.getElementById('upload-monthly').checked;
|
||||||
|
const resultDiv = document.getElementById('upload-result');
|
||||||
|
|
||||||
|
if (!fileInput.files || fileInput.files.length === 0) {
|
||||||
|
resultDiv.innerHTML = '<div class="alert alert-danger">Please select a file</div>';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const file = fileInput.files[0];
|
||||||
|
const formData = new FormData();
|
||||||
|
formData.append('file', file);
|
||||||
|
|
||||||
|
resultDiv.innerHTML = `<div class="alert alert-info">
|
||||||
|
<i class="bi bi-hourglass-split"></i> Uploading ${file.name} (${(file.size / 1024 / 1024).toFixed(2)} MB)...
|
||||||
|
</div>`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/v1/backups/upload?backup_type=${type}&is_monthly=${isMonthly}`, {
|
||||||
|
method: 'POST',
|
||||||
|
body: formData
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await response.json();
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
resultDiv.innerHTML = `
|
||||||
|
<div class="alert alert-success">
|
||||||
|
<strong>✅ Upload successful!</strong><br>
|
||||||
|
Job ID: ${result.job_id}<br>
|
||||||
|
Size: ${result.file_size_mb} MB<br>
|
||||||
|
Checksum: ${result.checksum.substring(0, 16)}...
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
fileInput.value = ''; // Clear file input
|
||||||
|
setTimeout(() => loadBackups(), 2000);
|
||||||
|
} else {
|
||||||
|
resultDiv.innerHTML = `<div class="alert alert-danger">Error: ${result.detail}</div>`;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
resultDiv.innerHTML = `<div class="alert alert-danger">Upload error: ${error.message}</div>`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show restore modal
|
||||||
|
function showRestore(jobId, jobType) {
|
||||||
|
selectedJobId = jobId;
|
||||||
|
document.getElementById('restore-job-id').textContent = jobId;
|
||||||
|
document.getElementById('restore-job-type').textContent = jobType;
|
||||||
|
restoreModal.show();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Confirm restore
|
||||||
|
async function confirmRestore() {
|
||||||
|
if (!selectedJobId) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/v1/backups/restore/${selectedJobId}`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {'Content-Type': 'application/json'},
|
||||||
|
body: JSON.stringify({confirmation: true})
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await response.json();
|
||||||
|
|
||||||
|
restoreModal.hide();
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
alert('Restore started! System entering maintenance mode.');
|
||||||
|
window.location.reload();
|
||||||
|
} else {
|
||||||
|
alert('Restore failed: ' + result.detail);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
alert('Restore error: ' + error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload to offsite
|
||||||
|
async function uploadOffsite(jobId) {
|
||||||
|
if (!confirm('Upload this backup to offsite storage?')) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/v1/backups/offsite/${jobId}`, {method: 'POST'});
|
||||||
|
const result = await response.json();
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
alert(result.message);
|
||||||
|
loadBackups();
|
||||||
|
} else {
|
||||||
|
alert('Upload failed: ' + result.detail);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
alert('Upload error: ' + error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete backup
|
||||||
|
async function deleteBackup(jobId) {
|
||||||
|
if (!confirm('Delete this backup? This cannot be undone.')) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/v1/backups/jobs/${jobId}`, {method: 'DELETE'});
|
||||||
|
const result = await response.json();
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
loadBackups();
|
||||||
|
} else {
|
||||||
|
alert('Delete failed: ' + result.detail);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
alert('Delete error: ' + error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Acknowledge notification
|
||||||
|
async function acknowledgeNotification(notificationId) {
|
||||||
|
try {
|
||||||
|
await fetch(`/api/v1/backups/notifications/${notificationId}/acknowledge`, {method: 'POST'});
|
||||||
|
loadNotifications();
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Acknowledge error:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Refresh backups
|
||||||
|
function refreshBackups() {
|
||||||
|
loadBackups();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update stats
|
||||||
|
function updateStats(backups) {
|
||||||
|
document.getElementById('total-backups').textContent = backups.length;
|
||||||
|
document.getElementById('completed-backups').textContent =
|
||||||
|
backups.filter(b => b.status === 'completed').length;
|
||||||
|
document.getElementById('pending-offsite').textContent =
|
||||||
|
backups.filter(b => b.status === 'completed' && !b.offsite_uploaded_at).length;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Utility functions
|
||||||
|
function formatBytes(bytes) {
|
||||||
|
if (!bytes) return '-';
|
||||||
|
const mb = bytes / 1024 / 1024;
|
||||||
|
return mb > 1024 ? `${(mb / 1024).toFixed(2)} GB` : `${mb.toFixed(2)} MB`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDate(dateStr) {
|
||||||
|
if (!dateStr) return '-';
|
||||||
|
const date = new Date(dateStr);
|
||||||
|
return date.toLocaleString('da-DK', {
|
||||||
|
year: 'numeric',
|
||||||
|
month: '2-digit',
|
||||||
|
day: '2-digit',
|
||||||
|
hour: '2-digit',
|
||||||
|
minute: '2-digit'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function getTypeBadgeClass(type) {
|
||||||
|
const classes = {
|
||||||
|
'database': 'bg-primary',
|
||||||
|
'files': 'bg-info',
|
||||||
|
'full': 'bg-success'
|
||||||
|
};
|
||||||
|
return classes[type] || 'bg-secondary';
|
||||||
|
}
|
||||||
|
|
||||||
|
function getStatusBadge(status) {
|
||||||
|
const badges = {
|
||||||
|
'pending': '<span class="badge bg-secondary">Pending</span>',
|
||||||
|
'running': '<span class="badge bg-warning">Running</span>',
|
||||||
|
'completed': '<span class="badge bg-success">Completed</span>',
|
||||||
|
'failed': '<span class="badge bg-danger">Failed</span>'
|
||||||
|
};
|
||||||
|
return badges[status] || status;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getOffsiteBadge(backup) {
|
||||||
|
if (backup.offsite_uploaded_at) {
|
||||||
|
return '<span class="badge bg-success offsite-badge"><i class="bi bi-cloud-check"></i> Uploaded</span>';
|
||||||
|
} else if (backup.offsite_retry_count > 0) {
|
||||||
|
return `<span class="badge bg-warning offsite-badge"><i class="bi bi-exclamation-triangle"></i> Retry ${backup.offsite_retry_count}</span>`;
|
||||||
|
} else {
|
||||||
|
return '<span class="badge bg-secondary offsite-badge">Pending</span>';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Theme toggle
|
||||||
|
function toggleTheme() {
|
||||||
|
const html = document.documentElement;
|
||||||
|
const icon = document.getElementById('theme-icon');
|
||||||
|
const currentTheme = html.getAttribute('data-bs-theme');
|
||||||
|
|
||||||
|
if (currentTheme === 'light') {
|
||||||
|
html.setAttribute('data-bs-theme', 'dark');
|
||||||
|
icon.classList.remove('bi-moon-stars');
|
||||||
|
icon.classList.add('bi-sun');
|
||||||
|
localStorage.setItem('theme', 'dark');
|
||||||
|
} else {
|
||||||
|
html.setAttribute('data-bs-theme', 'light');
|
||||||
|
icon.classList.remove('bi-sun');
|
||||||
|
icon.classList.add('bi-moon-stars');
|
||||||
|
localStorage.setItem('theme', 'light');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load saved theme
|
||||||
|
(function() {
|
||||||
|
const savedTheme = localStorage.getItem('theme') || 'light';
|
||||||
|
document.documentElement.setAttribute('data-bs-theme', savedTheme);
|
||||||
|
if (savedTheme === 'dark') {
|
||||||
|
document.getElementById('theme-icon').classList.replace('bi-moon-stars', 'bi-sun');
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@ -4,6 +4,7 @@ Backend API for managing supplier invoices that integrate with e-conomic
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
from fastapi import APIRouter, HTTPException, UploadFile, File
|
from fastapi import APIRouter, HTTPException, UploadFile, File
|
||||||
|
from pydantic import BaseModel
|
||||||
from typing import List, Dict, Optional
|
from typing import List, Dict, Optional
|
||||||
from datetime import datetime, date, timedelta
|
from datetime import datetime, date, timedelta
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
@ -276,21 +277,35 @@ async def get_pending_files():
|
|||||||
logger.info(f"📋 Checking invoice2data templates: {len(invoice2data.templates)} loaded")
|
logger.info(f"📋 Checking invoice2data templates: {len(invoice2data.templates)} loaded")
|
||||||
|
|
||||||
for file in files:
|
for file in files:
|
||||||
# Check if there's an invoice2data template for this vendor's CVR
|
# Check if there's an invoice2data template for this vendor's CVR or name
|
||||||
vendor_cvr = file.get('matched_vendor_cvr_number') or file.get('detected_vendor_cvr') or file.get('vendor_cvr')
|
vendor_cvr = file.get('matched_vendor_cvr_number') or file.get('detected_vendor_cvr') or file.get('vendor_cvr')
|
||||||
|
vendor_name = file.get('vendor_name') or file.get('detected_vendor_name') or file.get('matched_vendor_name')
|
||||||
file['has_invoice2data_template'] = False
|
file['has_invoice2data_template'] = False
|
||||||
|
|
||||||
logger.debug(f" File {file['file_id']}: CVR={vendor_cvr}")
|
logger.debug(f" File {file['file_id']}: CVR={vendor_cvr}, name={vendor_name}")
|
||||||
|
|
||||||
if vendor_cvr:
|
# Check all templates
|
||||||
# Check all templates for this CVR in keywords
|
|
||||||
for template_name, template_data in invoice2data.templates.items():
|
for template_name, template_data in invoice2data.templates.items():
|
||||||
keywords = template_data.get('keywords', [])
|
keywords = template_data.get('keywords', [])
|
||||||
logger.debug(f" Template {template_name}: keywords={keywords}")
|
logger.debug(f" Template {template_name}: keywords={keywords}")
|
||||||
if str(vendor_cvr) in [str(k) for k in keywords]:
|
|
||||||
|
# Match by CVR
|
||||||
|
if vendor_cvr and str(vendor_cvr) in [str(k) for k in keywords]:
|
||||||
file['has_invoice2data_template'] = True
|
file['has_invoice2data_template'] = True
|
||||||
file['invoice2data_template_name'] = template_name
|
file['invoice2data_template_name'] = template_name
|
||||||
logger.info(f" ✅ File {file['file_id']} matched template: {template_name}")
|
logger.info(f" ✅ File {file['file_id']} matched template {template_name} by CVR")
|
||||||
|
break
|
||||||
|
|
||||||
|
# Match by vendor name
|
||||||
|
if vendor_name:
|
||||||
|
for keyword in keywords:
|
||||||
|
if str(keyword).upper() in str(vendor_name).upper():
|
||||||
|
file['has_invoice2data_template'] = True
|
||||||
|
file['invoice2data_template_name'] = template_name
|
||||||
|
logger.info(f" ✅ File {file['file_id']} matched template {template_name} by name")
|
||||||
|
break
|
||||||
|
|
||||||
|
if file['has_invoice2data_template']:
|
||||||
break
|
break
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"❌ Failed to check invoice2data templates: {e}", exc_info=True)
|
logger.error(f"❌ Failed to check invoice2data templates: {e}", exc_info=True)
|
||||||
@ -413,12 +428,23 @@ async def get_file_extracted_data(file_id: int):
|
|||||||
# Build llm_data response
|
# Build llm_data response
|
||||||
llm_data = None
|
llm_data = None
|
||||||
if llm_json_data:
|
if llm_json_data:
|
||||||
|
# Normalize common invoice2data field names to our API schema
|
||||||
|
total_amount_value = llm_json_data.get('total_amount')
|
||||||
|
if total_amount_value is None:
|
||||||
|
total_amount_value = llm_json_data.get('amount_total')
|
||||||
|
|
||||||
|
invoice_date_value = llm_json_data.get('invoice_date')
|
||||||
|
if invoice_date_value is None:
|
||||||
|
invoice_date_value = llm_json_data.get('document_date')
|
||||||
|
|
||||||
|
due_date_value = llm_json_data.get('due_date')
|
||||||
|
|
||||||
# Use invoice_number from LLM JSON (works for both AI and template extraction)
|
# Use invoice_number from LLM JSON (works for both AI and template extraction)
|
||||||
llm_data = {
|
llm_data = {
|
||||||
"invoice_number": llm_json_data.get('invoice_number'),
|
"invoice_number": llm_json_data.get('invoice_number'),
|
||||||
"invoice_date": llm_json_data.get('invoice_date'),
|
"invoice_date": invoice_date_value,
|
||||||
"due_date": llm_json_data.get('due_date'),
|
"due_date": due_date_value,
|
||||||
"total_amount": float(llm_json_data.get('total_amount')) if llm_json_data.get('total_amount') else None,
|
"total_amount": float(total_amount_value) if total_amount_value else None,
|
||||||
"currency": llm_json_data.get('currency') or 'DKK',
|
"currency": llm_json_data.get('currency') or 'DKK',
|
||||||
"document_type": llm_json_data.get('document_type'),
|
"document_type": llm_json_data.get('document_type'),
|
||||||
"lines": formatted_lines
|
"lines": formatted_lines
|
||||||
@ -1377,8 +1403,11 @@ async def delete_supplier_invoice(invoice_id: int):
|
|||||||
|
|
||||||
# ========== E-CONOMIC INTEGRATION ==========
|
# ========== E-CONOMIC INTEGRATION ==========
|
||||||
|
|
||||||
|
class ApproveRequest(BaseModel):
|
||||||
|
approved_by: str
|
||||||
|
|
||||||
@router.post("/supplier-invoices/{invoice_id}/approve")
|
@router.post("/supplier-invoices/{invoice_id}/approve")
|
||||||
async def approve_supplier_invoice(invoice_id: int, approved_by: str):
|
async def approve_supplier_invoice(invoice_id: int, request: ApproveRequest):
|
||||||
"""Approve supplier invoice for payment"""
|
"""Approve supplier invoice for payment"""
|
||||||
try:
|
try:
|
||||||
invoice = execute_query(
|
invoice = execute_query(
|
||||||
@ -1388,21 +1417,21 @@ async def approve_supplier_invoice(invoice_id: int, approved_by: str):
|
|||||||
)
|
)
|
||||||
|
|
||||||
if not invoice:
|
if not invoice:
|
||||||
raise HTTPException(status_code=404, detail=f"Invoice {invoice_id} not found")
|
raise HTTPException(status_code=404, detail=f"Faktura {invoice_id} ikke fundet")
|
||||||
|
|
||||||
if invoice['status'] != 'pending':
|
if invoice['status'] != 'pending':
|
||||||
raise HTTPException(status_code=400, detail=f"Invoice is already {invoice['status']}")
|
raise HTTPException(status_code=400, detail=f"Faktura har allerede status '{invoice['status']}' - kan kun godkende fakturaer med status 'pending'")
|
||||||
|
|
||||||
execute_update(
|
execute_update(
|
||||||
"""UPDATE supplier_invoices
|
"""UPDATE supplier_invoices
|
||||||
SET status = 'approved', approved_by = %s, approved_at = CURRENT_TIMESTAMP
|
SET status = 'approved', approved_by = %s, approved_at = CURRENT_TIMESTAMP
|
||||||
WHERE id = %s""",
|
WHERE id = %s""",
|
||||||
(approved_by, invoice_id)
|
(request.approved_by, invoice_id)
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info(f"✅ Approved supplier invoice {invoice['invoice_number']} by {approved_by}")
|
logger.info(f"✅ Approved supplier invoice {invoice['invoice_number']} by {request.approved_by}")
|
||||||
|
|
||||||
return {"success": True, "invoice_id": invoice_id, "approved_by": approved_by}
|
return {"success": True, "invoice_id": invoice_id, "approved_by": request.approved_by}
|
||||||
|
|
||||||
except HTTPException:
|
except HTTPException:
|
||||||
raise
|
raise
|
||||||
@ -2058,6 +2087,35 @@ async def reprocess_uploaded_file(file_id: int):
|
|||||||
is_invoice2data = (template_id == -1)
|
is_invoice2data = (template_id == -1)
|
||||||
|
|
||||||
if is_invoice2data:
|
if is_invoice2data:
|
||||||
|
def _to_numeric(value):
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if isinstance(value, (int, float, Decimal)):
|
||||||
|
return float(value)
|
||||||
|
if not isinstance(value, str):
|
||||||
|
return None
|
||||||
|
|
||||||
|
cleaned = value.strip().replace(' ', '')
|
||||||
|
if not cleaned:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Common Danish formatting: 25.000,00 or 1.530,00
|
||||||
|
if ',' in cleaned:
|
||||||
|
cleaned = cleaned.replace('.', '').replace(',', '.')
|
||||||
|
|
||||||
|
try:
|
||||||
|
return float(cleaned)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _clean_document_id(value):
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if isinstance(value, str):
|
||||||
|
cleaned = value.strip()
|
||||||
|
return cleaned if cleaned and cleaned.lower() != 'none' else None
|
||||||
|
return str(value)
|
||||||
|
|
||||||
# Invoice2data doesn't have vendor in cache
|
# Invoice2data doesn't have vendor in cache
|
||||||
logger.info(f"📋 Using invoice2data template")
|
logger.info(f"📋 Using invoice2data template")
|
||||||
# Try to find vendor from extracted CVR
|
# Try to find vendor from extracted CVR
|
||||||
@ -2070,6 +2128,20 @@ async def reprocess_uploaded_file(file_id: int):
|
|||||||
if vendor:
|
if vendor:
|
||||||
vendor_id = vendor['id']
|
vendor_id = vendor['id']
|
||||||
|
|
||||||
|
# Fallback: use vendor detected during quick analysis (incoming_files.detected_vendor_id)
|
||||||
|
if vendor_id is None:
|
||||||
|
vendor_id = file_record.get('detected_vendor_id')
|
||||||
|
|
||||||
|
# Fallback: match by issuer name
|
||||||
|
if vendor_id is None and extracted_fields.get('issuer'):
|
||||||
|
vendor = execute_query(
|
||||||
|
"SELECT id FROM vendors WHERE name ILIKE %s ORDER BY id LIMIT 1",
|
||||||
|
(extracted_fields['issuer'],),
|
||||||
|
fetchone=True
|
||||||
|
)
|
||||||
|
if vendor:
|
||||||
|
vendor_id = vendor['id']
|
||||||
|
|
||||||
# Store invoice2data extraction in database
|
# Store invoice2data extraction in database
|
||||||
extraction_id = execute_insert(
|
extraction_id = execute_insert(
|
||||||
"""INSERT INTO extractions
|
"""INSERT INTO extractions
|
||||||
@ -2081,12 +2153,12 @@ async def reprocess_uploaded_file(file_id: int):
|
|||||||
(file_id, vendor_id,
|
(file_id, vendor_id,
|
||||||
extracted_fields.get('issuer'), # vendor_name
|
extracted_fields.get('issuer'), # vendor_name
|
||||||
extracted_fields.get('vendor_vat'), # vendor_cvr
|
extracted_fields.get('vendor_vat'), # vendor_cvr
|
||||||
str(extracted_fields.get('invoice_number')), # document_id
|
_clean_document_id(extracted_fields.get('invoice_number')), # document_id
|
||||||
extracted_fields.get('invoice_date'), # document_date
|
extracted_fields.get('invoice_date'), # document_date
|
||||||
extracted_fields.get('due_date'),
|
extracted_fields.get('due_date'),
|
||||||
'invoice', # document_type
|
'invoice', # document_type
|
||||||
'invoice', # document_type_detected
|
'invoice', # document_type_detected
|
||||||
extracted_fields.get('amount_total'),
|
_to_numeric(extracted_fields.get('amount_total') if extracted_fields.get('amount_total') is not None else extracted_fields.get('total_amount')),
|
||||||
extracted_fields.get('currency', 'DKK'),
|
extracted_fields.get('currency', 'DKK'),
|
||||||
1.0, # invoice2data always 100% confidence
|
1.0, # invoice2data always 100% confidence
|
||||||
json.dumps(extracted_fields), # llm_response_json
|
json.dumps(extracted_fields), # llm_response_json
|
||||||
@ -2096,6 +2168,9 @@ async def reprocess_uploaded_file(file_id: int):
|
|||||||
# Insert line items if extracted
|
# Insert line items if extracted
|
||||||
if extracted_fields.get('lines'):
|
if extracted_fields.get('lines'):
|
||||||
for idx, line in enumerate(extracted_fields['lines'], start=1):
|
for idx, line in enumerate(extracted_fields['lines'], start=1):
|
||||||
|
line_total = _to_numeric(line.get('line_total'))
|
||||||
|
unit_price = _to_numeric(line.get('unit_price'))
|
||||||
|
quantity = _to_numeric(line.get('quantity'))
|
||||||
execute_insert(
|
execute_insert(
|
||||||
"""INSERT INTO extraction_lines
|
"""INSERT INTO extraction_lines
|
||||||
(extraction_id, line_number, description, quantity, unit_price,
|
(extraction_id, line_number, description, quantity, unit_price,
|
||||||
@ -2104,8 +2179,8 @@ async def reprocess_uploaded_file(file_id: int):
|
|||||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||||
RETURNING line_id""",
|
RETURNING line_id""",
|
||||||
(extraction_id, idx, line.get('description'),
|
(extraction_id, idx, line.get('description'),
|
||||||
line.get('quantity'), line.get('unit_price'),
|
quantity, unit_price,
|
||||||
line.get('line_total'), None, None, 1.0,
|
line_total, None, None, 1.0,
|
||||||
line.get('ip_address'), line.get('contract_number'),
|
line.get('ip_address'), line.get('contract_number'),
|
||||||
line.get('location_street'), line.get('location_zip'), line.get('location_city'))
|
line.get('location_street'), line.get('location_zip'), line.get('location_city'))
|
||||||
)
|
)
|
||||||
@ -2137,8 +2212,14 @@ async def reprocess_uploaded_file(file_id: int):
|
|||||||
logger.info(f"🤖 Calling Ollama for AI extraction...")
|
logger.info(f"🤖 Calling Ollama for AI extraction...")
|
||||||
llm_result = await ollama_service.extract_from_text(text)
|
llm_result = await ollama_service.extract_from_text(text)
|
||||||
|
|
||||||
if not llm_result or 'error' in llm_result:
|
# Handle both dict and string error responses
|
||||||
error_msg = llm_result.get('error') if llm_result else 'AI extraction fejlede'
|
if not llm_result or isinstance(llm_result, str) or (isinstance(llm_result, dict) and 'error' in llm_result):
|
||||||
|
if isinstance(llm_result, dict):
|
||||||
|
error_msg = llm_result.get('error', 'AI extraction fejlede')
|
||||||
|
elif isinstance(llm_result, str):
|
||||||
|
error_msg = llm_result # Error message returned as string
|
||||||
|
else:
|
||||||
|
error_msg = 'AI extraction fejlede'
|
||||||
logger.error(f"❌ AI extraction failed: {error_msg}")
|
logger.error(f"❌ AI extraction failed: {error_msg}")
|
||||||
|
|
||||||
execute_update(
|
execute_update(
|
||||||
@ -2198,6 +2279,14 @@ async def reprocess_uploaded_file(file_id: int):
|
|||||||
logger.info(f"✅ AI extraction completed for file {file_id}")
|
logger.info(f"✅ AI extraction completed for file {file_id}")
|
||||||
|
|
||||||
# Return success with template data or AI extraction result
|
# Return success with template data or AI extraction result
|
||||||
|
# Determine confidence value safely
|
||||||
|
if template_id:
|
||||||
|
final_confidence = confidence
|
||||||
|
elif 'llm_result' in locals() and isinstance(llm_result, dict):
|
||||||
|
final_confidence = llm_result.get('confidence', 0.75)
|
||||||
|
else:
|
||||||
|
final_confidence = 0.0
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
"status": "success",
|
"status": "success",
|
||||||
"file_id": file_id,
|
"file_id": file_id,
|
||||||
@ -2205,7 +2294,7 @@ async def reprocess_uploaded_file(file_id: int):
|
|||||||
"template_matched": template_id is not None,
|
"template_matched": template_id is not None,
|
||||||
"template_id": template_id,
|
"template_id": template_id,
|
||||||
"vendor_id": vendor_id,
|
"vendor_id": vendor_id,
|
||||||
"confidence": confidence if template_id else llm_result.get('confidence', 0.75),
|
"confidence": final_confidence,
|
||||||
"extracted_fields": extracted_fields,
|
"extracted_fields": extracted_fields,
|
||||||
"pdf_text": text[:1000] if not template_id else text
|
"pdf_text": text[:1000] if not template_id else text
|
||||||
}
|
}
|
||||||
|
|||||||
@ -515,7 +515,7 @@
|
|||||||
|
|
||||||
<!-- Manual Entry Modal -->
|
<!-- Manual Entry Modal -->
|
||||||
<div class="modal fade" id="manualEntryModal" tabindex="-1">
|
<div class="modal fade" id="manualEntryModal" tabindex="-1">
|
||||||
<div class="modal-dialog modal-xl">
|
<div class="modal-dialog" style="max-width: 95%; width: 1800px;">
|
||||||
<div class="modal-content">
|
<div class="modal-content">
|
||||||
<div class="modal-header">
|
<div class="modal-header">
|
||||||
<h5 class="modal-title"><i class="bi bi-pencil-square me-2"></i>Manuel Indtastning af Faktura</h5>
|
<h5 class="modal-title"><i class="bi bi-pencil-square me-2"></i>Manuel Indtastning af Faktura</h5>
|
||||||
@ -525,16 +525,28 @@
|
|||||||
<input type="hidden" id="manualEntryFileId">
|
<input type="hidden" id="manualEntryFileId">
|
||||||
|
|
||||||
<div class="row">
|
<div class="row">
|
||||||
<!-- Left: PDF Viewer -->
|
<!-- Left: PDF Text -->
|
||||||
<div class="col-md-6">
|
<div class="col-md-4">
|
||||||
<h6 class="mb-3">PDF Dokument</h6>
|
<h6 class="mb-3">
|
||||||
<div style="border: 1px solid #ddd; border-radius: 4px; height: 600px; overflow: hidden;">
|
PDF Dokument
|
||||||
<iframe id="manualEntryPdfViewer" type="application/pdf" width="100%" height="100%" style="border: none;"></iframe>
|
<button type="button" class="btn btn-sm btn-outline-secondary float-end" onclick="togglePdfView()">
|
||||||
|
<i class="bi bi-file-earmark-text" id="pdfViewIcon"></i> Vis Original
|
||||||
|
</button>
|
||||||
|
</h6>
|
||||||
|
<!-- PDF Text View (default, selectable) -->
|
||||||
|
<div id="pdfTextView" style="border: 1px solid #ddd; border-radius: 4px; height: 700px; overflow: auto; background: white; padding: 15px; font-family: monospace; font-size: 0.85rem; line-height: 1.4; user-select: text;">
|
||||||
|
<div class="text-muted text-center py-5">Indlæser PDF tekst...</div>
|
||||||
|
</div>
|
||||||
|
<!-- PDF Original View (hidden by default) -->
|
||||||
|
<iframe id="manualEntryPdfViewer" type="application/pdf" width="100%" height="700px" style="border: 1px solid #ddd; border-radius: 4px; display: none;"></iframe>
|
||||||
|
|
||||||
|
<div class="alert alert-info mt-2 py-2 px-3" style="font-size: 0.85rem;">
|
||||||
|
💡 <strong>Tip:</strong> Markér tekst og klik på et felt for at indsætte
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Right: Form -->
|
<!-- Right: Form -->
|
||||||
<div class="col-md-6">
|
<div class="col-md-8">
|
||||||
<h6 class="mb-3">Faktura Detaljer</h6>
|
<h6 class="mb-3">Faktura Detaljer</h6>
|
||||||
<form id="manualEntryForm">
|
<form id="manualEntryForm">
|
||||||
<!-- Vendor Selection -->
|
<!-- Vendor Selection -->
|
||||||
@ -550,13 +562,13 @@
|
|||||||
|
|
||||||
<!-- Invoice Details -->
|
<!-- Invoice Details -->
|
||||||
<div class="row mb-3">
|
<div class="row mb-3">
|
||||||
<div class="col-md-6">
|
<div class="col-md-8">
|
||||||
<label class="form-label">Fakturanummer *</label>
|
<label class="form-label fw-bold">Fakturanummer *</label>
|
||||||
<input type="text" class="form-control" id="manualInvoiceNumber" required>
|
<input type="text" class="form-control form-control-lg manual-entry-field" id="manualInvoiceNumber" required>
|
||||||
</div>
|
</div>
|
||||||
<div class="col-md-6">
|
<div class="col-md-4">
|
||||||
<label class="form-label">Type</label>
|
<label class="form-label fw-bold">Type</label>
|
||||||
<select class="form-select" id="manualInvoiceType">
|
<select class="form-select form-select-lg" id="manualInvoiceType">
|
||||||
<option value="invoice">Faktura</option>
|
<option value="invoice">Faktura</option>
|
||||||
<option value="credit_note">Kreditnota</option>
|
<option value="credit_note">Kreditnota</option>
|
||||||
</select>
|
</select>
|
||||||
@ -565,23 +577,23 @@
|
|||||||
|
|
||||||
<div class="row mb-3">
|
<div class="row mb-3">
|
||||||
<div class="col-md-6">
|
<div class="col-md-6">
|
||||||
<label class="form-label">Fakturadato *</label>
|
<label class="form-label fw-bold">Fakturadato *</label>
|
||||||
<input type="date" class="form-control" id="manualInvoiceDate" required>
|
<input type="date" class="form-control form-control-lg manual-entry-field" id="manualInvoiceDate" required>
|
||||||
</div>
|
</div>
|
||||||
<div class="col-md-6">
|
<div class="col-md-6">
|
||||||
<label class="form-label">Forfaldsdato</label>
|
<label class="form-label fw-bold">Forfaldsdato</label>
|
||||||
<input type="date" class="form-control" id="manualDueDate">
|
<input type="date" class="form-control form-control-lg manual-entry-field" id="manualDueDate">
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="row mb-3">
|
<div class="row mb-3">
|
||||||
<div class="col-md-6">
|
<div class="col-md-8">
|
||||||
<label class="form-label">Total Beløb *</label>
|
<label class="form-label fw-bold">Total Beløb *</label>
|
||||||
<input type="number" step="0.01" class="form-control" id="manualTotalAmount" required>
|
<input type="number" step="0.01" class="form-control form-control-lg manual-entry-field" id="manualTotalAmount" required>
|
||||||
</div>
|
</div>
|
||||||
<div class="col-md-6">
|
<div class="col-md-4">
|
||||||
<label class="form-label">Valuta</label>
|
<label class="form-label fw-bold">Valuta</label>
|
||||||
<select class="form-select" id="manualCurrency">
|
<select class="form-select form-select-lg" id="manualCurrency">
|
||||||
<option value="DKK">DKK</option>
|
<option value="DKK">DKK</option>
|
||||||
<option value="EUR">EUR</option>
|
<option value="EUR">EUR</option>
|
||||||
<option value="USD">USD</option>
|
<option value="USD">USD</option>
|
||||||
@ -712,41 +724,102 @@
|
|||||||
let currentInvoiceId = null;
|
let currentInvoiceId = null;
|
||||||
let currentFilter = 'all';
|
let currentFilter = 'all';
|
||||||
let allInvoices = [];
|
let allInvoices = [];
|
||||||
|
let lastSelectedText = '';
|
||||||
|
let lastFocusedField = null;
|
||||||
|
|
||||||
// Load data on page load
|
// Load data on page load
|
||||||
document.addEventListener('DOMContentLoaded', () => {
|
document.addEventListener('DOMContentLoaded', () => {
|
||||||
loadStats();
|
loadStats();
|
||||||
loadInvoices();
|
loadInvoices();
|
||||||
loadVendors();
|
loadVendors();
|
||||||
|
setupManualEntryTextSelection();
|
||||||
setDefaultDates();
|
setDefaultDates();
|
||||||
loadPendingFilesCount(); // Load count for badge
|
loadPendingFilesCount(); // Load count for badge
|
||||||
checkEmailContext(); // Check if coming from email
|
checkEmailContext(); // Check if coming from email
|
||||||
});
|
});
|
||||||
|
|
||||||
// Check if coming from email context
|
// Check if coming from email context
|
||||||
function checkEmailContext() {
|
async function checkEmailContext() {
|
||||||
const emailContext = sessionStorage.getItem('supplierInvoiceContext');
|
const emailContext = sessionStorage.getItem('supplierInvoiceContext');
|
||||||
if (emailContext) {
|
if (emailContext) {
|
||||||
try {
|
try {
|
||||||
const context = JSON.parse(emailContext);
|
const context = JSON.parse(emailContext);
|
||||||
|
console.log('📧 Processing email context:', context);
|
||||||
|
|
||||||
// Show notification
|
// Show notification
|
||||||
showSuccess(`Opret faktura fra email: ${context.subject}`);
|
console.log('📧 Behandler faktura fra email:', context.subject);
|
||||||
|
|
||||||
// Pre-fill description field with email subject
|
// Process attachments if any
|
||||||
const descriptionField = document.getElementById('description');
|
if (context.attachments && context.attachments.length > 0) {
|
||||||
if (descriptionField) {
|
console.log(`📎 Found ${context.attachments.length} attachments`);
|
||||||
descriptionField.value = `Fra email: ${context.subject}\nAfsender: ${context.sender}`;
|
|
||||||
|
// Find PDF attachments
|
||||||
|
const pdfAttachments = context.attachments.filter(att =>
|
||||||
|
att.filename && att.filename.toLowerCase().endsWith('.pdf')
|
||||||
|
);
|
||||||
|
|
||||||
|
if (pdfAttachments.length > 0) {
|
||||||
|
console.log(`📄 Processing ${pdfAttachments.length} PDF attachments`);
|
||||||
|
|
||||||
|
for (const attachment of pdfAttachments) {
|
||||||
|
try {
|
||||||
|
// Download attachment and upload to supplier invoices
|
||||||
|
console.log(`⬇️ Downloading attachment: ${attachment.filename}`);
|
||||||
|
const attachmentResponse = await fetch(`/api/v1/emails/${context.emailId}/attachments/${attachment.id}`);
|
||||||
|
|
||||||
|
if (!attachmentResponse.ok) {
|
||||||
|
console.error(`Failed to download attachment ${attachment.id}`);
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Open create modal if exists
|
const blob = await attachmentResponse.blob();
|
||||||
const createModal = new bootstrap.Modal(document.getElementById('invoiceModal'));
|
const file = new File([blob], attachment.filename, { type: 'application/pdf' });
|
||||||
createModal.show();
|
|
||||||
|
// Upload to supplier invoices
|
||||||
|
console.log(`⬆️ Uploading to supplier invoices: ${attachment.filename}`);
|
||||||
|
const formData = new FormData();
|
||||||
|
formData.append('file', file);
|
||||||
|
|
||||||
|
const uploadResponse = await fetch('/api/v1/supplier-invoices/upload', {
|
||||||
|
method: 'POST',
|
||||||
|
body: formData
|
||||||
|
});
|
||||||
|
|
||||||
|
if (uploadResponse.ok) {
|
||||||
|
const result = await uploadResponse.json();
|
||||||
|
console.log('✅ Upload successful:', result);
|
||||||
|
alert(`✅ Faktura ${attachment.filename} uploadet og behandlet`);
|
||||||
|
} else {
|
||||||
|
const errorData = await uploadResponse.json();
|
||||||
|
console.error('Upload failed:', errorData);
|
||||||
|
alert(`❌ Kunne ikke uploade ${attachment.filename}: ${errorData.detail || 'Ukendt fejl'}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Error processing attachment ${attachment.filename}:`, error);
|
||||||
|
alert(`❌ Fejl ved behandling af ${attachment.filename}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reload pending files list after uploads
|
||||||
|
setTimeout(() => {
|
||||||
|
loadPendingFiles();
|
||||||
|
}, 1000);
|
||||||
|
|
||||||
|
} else {
|
||||||
|
alert('⚠️ Ingen PDF vedhæftninger fundet i emailen');
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
alert('⚠️ Emailen har ingen vedhæftninger');
|
||||||
|
}
|
||||||
|
|
||||||
// Clear context after use
|
// Clear context after use
|
||||||
sessionStorage.removeItem('supplierInvoiceContext');
|
sessionStorage.removeItem('supplierInvoiceContext');
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to parse email context:', error);
|
console.error('Failed to process email context:', error);
|
||||||
|
alert('❌ Kunne ikke behandle email vedhæftninger');
|
||||||
|
sessionStorage.removeItem('supplierInvoiceContext');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1167,6 +1240,10 @@ async function reviewExtractedData(fileId) {
|
|||||||
.replace(/^dk/i, '')
|
.replace(/^dk/i, '')
|
||||||
.trim();
|
.trim();
|
||||||
|
|
||||||
|
// Check for validation warnings
|
||||||
|
const hasValidationIssues = aiData?._validation_warning || aiData?._vat_warning;
|
||||||
|
const allValidationsPassed = !hasValidationIssues && aiData && aiData.lines && aiData.lines.length > 0;
|
||||||
|
|
||||||
// Build modal content
|
// Build modal content
|
||||||
let modalContent = `
|
let modalContent = `
|
||||||
<div class="alert alert-info">
|
<div class="alert alert-info">
|
||||||
@ -1179,6 +1256,19 @@ async function reviewExtractedData(fileId) {
|
|||||||
}
|
}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
${hasValidationIssues ? `
|
||||||
|
<div class="alert alert-warning">
|
||||||
|
<h6 class="alert-heading"><i class="bi bi-exclamation-triangle me-2"></i>Beløbs-validering</h6>
|
||||||
|
${aiData._validation_warning ? `<p class="mb-1">⚠️ ${aiData._validation_warning}</p>` : ''}
|
||||||
|
${aiData._vat_warning ? `<p class="mb-0">⚠️ ${aiData._vat_warning}</p>` : ''}
|
||||||
|
</div>
|
||||||
|
` : allValidationsPassed ? `
|
||||||
|
<div class="alert alert-success">
|
||||||
|
<h6 class="alert-heading"><i class="bi bi-check-circle me-2"></i>Beløbs-validering</h6>
|
||||||
|
<p class="mb-0">✅ Varelinjer summer korrekt til subtotal<br>✅ Moms beregning er korrekt (25%)</p>
|
||||||
|
</div>
|
||||||
|
` : ''}
|
||||||
|
|
||||||
<h5>Udtrukne Data:</h5>
|
<h5>Udtrukne Data:</h5>
|
||||||
<table class="table table-sm">
|
<table class="table table-sm">
|
||||||
<tr><th>Felt</th><th>Værdi</th></tr>
|
<tr><th>Felt</th><th>Værdi</th></tr>
|
||||||
@ -1213,7 +1303,7 @@ async function reviewExtractedData(fileId) {
|
|||||||
${data.pdf_text_preview ? `
|
${data.pdf_text_preview ? `
|
||||||
<h6 class="mt-3">PDF Tekst Preview:</h6>
|
<h6 class="mt-3">PDF Tekst Preview:</h6>
|
||||||
<div class="border rounded p-3 bg-light" style="max-height: 500px; overflow-y: auto;">
|
<div class="border rounded p-3 bg-light" style="max-height: 500px; overflow-y: auto;">
|
||||||
<pre class="mb-0" style="font-size: 0.85rem; white-space: pre-wrap; word-wrap: break-word;">${data.pdf_text_preview}</pre>
|
<pre class="mb-0" style="font-size: 0.85rem; white-space: pre-wrap; word-wrap: break-word; font-family: monospace; line-height: 1.3;">${escapeHtml(data.pdf_text_preview)}</pre>
|
||||||
</div>
|
</div>
|
||||||
` : ''}
|
` : ''}
|
||||||
`;
|
`;
|
||||||
@ -1624,8 +1714,22 @@ async function openManualEntryMode() {
|
|||||||
// Wait a bit for modal to render
|
// Wait a bit for modal to render
|
||||||
await new Promise(resolve => setTimeout(resolve, 300));
|
await new Promise(resolve => setTimeout(resolve, 300));
|
||||||
|
|
||||||
// Load PDF after modal is open
|
// Load PDF text after modal is open
|
||||||
console.log('Loading PDF...');
|
console.log('Loading PDF text...');
|
||||||
|
try {
|
||||||
|
const pdfResponse = await fetch(`/api/v1/supplier-invoices/files/${fileId}/pdf-text`);
|
||||||
|
if (pdfResponse.ok) {
|
||||||
|
const pdfData = await pdfResponse.json();
|
||||||
|
document.getElementById('pdfTextView').innerHTML = `<pre style="margin: 0; white-space: pre-wrap; word-wrap: break-word;">${escapeHtml(pdfData.pdf_text)}</pre>`;
|
||||||
|
} else {
|
||||||
|
document.getElementById('pdfTextView').innerHTML = '<div class="text-danger">Kunne ikke indlæse PDF tekst</div>';
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Error loading PDF text:', e);
|
||||||
|
document.getElementById('pdfTextView').innerHTML = '<div class="text-danger">Fejl ved indlæsning af PDF</div>';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also set iframe src for original view toggle
|
||||||
document.getElementById('manualEntryPdfViewer').src = `/api/v1/supplier-invoices/files/${fileId}/pdf`;
|
document.getElementById('manualEntryPdfViewer').src = `/api/v1/supplier-invoices/files/${fileId}/pdf`;
|
||||||
|
|
||||||
// Load vendors into dropdown
|
// Load vendors into dropdown
|
||||||
@ -1759,6 +1863,93 @@ async function openManualEntryMode() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Setup text selection feature for manual entry
|
||||||
|
function setupManualEntryTextSelection() {
|
||||||
|
// Track selection anywhere in document
|
||||||
|
document.addEventListener('mouseup', (e) => {
|
||||||
|
const selection = window.getSelection();
|
||||||
|
const selectedText = selection.toString().trim();
|
||||||
|
|
||||||
|
if (selectedText && selectedText.length > 0) {
|
||||||
|
// Check if selection is from PDF text view
|
||||||
|
const pdfTextView = document.getElementById('pdfTextView');
|
||||||
|
if (pdfTextView && pdfTextView.contains(selection.anchorNode)) {
|
||||||
|
lastSelectedText = selectedText;
|
||||||
|
console.log('✅ Selected from PDF:', selectedText);
|
||||||
|
|
||||||
|
// Visual feedback
|
||||||
|
const tip = document.querySelector('.alert-info');
|
||||||
|
if (tip && tip.closest('#manualEntryModal')) {
|
||||||
|
tip.classList.remove('alert-info');
|
||||||
|
tip.classList.add('alert-success');
|
||||||
|
tip.innerHTML = `✅ <strong>"${selectedText.substring(0, 50)}${selectedText.length > 50 ? '...' : ''}"</strong> markeret - klik på et felt for at indsætte`;
|
||||||
|
|
||||||
|
setTimeout(() => {
|
||||||
|
tip.classList.remove('alert-success');
|
||||||
|
tip.classList.add('alert-info');
|
||||||
|
tip.innerHTML = '💡 <strong>Tip:</strong> Markér tekst og klik på et felt for at indsætte';
|
||||||
|
}, 3000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Insert selected text on field click/focus
|
||||||
|
document.addEventListener('click', (e) => {
|
||||||
|
// Check if clicked element is an input/textarea in the modal
|
||||||
|
if (lastSelectedText && e.target.closest('#manualEntryModal')) {
|
||||||
|
if ((e.target.tagName === 'INPUT' &&
|
||||||
|
e.target.type !== 'button' &&
|
||||||
|
e.target.type !== 'submit' &&
|
||||||
|
e.target.type !== 'checkbox' &&
|
||||||
|
e.target.type !== 'radio') ||
|
||||||
|
e.target.tagName === 'TEXTAREA') {
|
||||||
|
|
||||||
|
console.log('🎯 Clicked field:', e.target.id || e.target.name, 'Current value:', e.target.value);
|
||||||
|
console.log('📝 Will insert:', lastSelectedText);
|
||||||
|
|
||||||
|
// Only insert if field is empty or user confirms
|
||||||
|
if (!e.target.value || e.target.value.trim() === '') {
|
||||||
|
e.target.value = lastSelectedText;
|
||||||
|
console.log('✅ Auto-inserted into', e.target.id || e.target.name);
|
||||||
|
e.target.focus();
|
||||||
|
// Clear selection
|
||||||
|
lastSelectedText = '';
|
||||||
|
window.getSelection().removeAllRanges();
|
||||||
|
} else {
|
||||||
|
// Ask to replace
|
||||||
|
if (confirm(`Erstat "${e.target.value}" med "${lastSelectedText}"?`)) {
|
||||||
|
e.target.value = lastSelectedText;
|
||||||
|
console.log('✅ Replaced content in', e.target.id || e.target.name);
|
||||||
|
// Clear selection
|
||||||
|
lastSelectedText = '';
|
||||||
|
window.getSelection().removeAllRanges();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, true); // Use capture phase
|
||||||
|
}
|
||||||
|
|
||||||
|
// Toggle between PDF text view and original PDF
|
||||||
|
function togglePdfView() {
|
||||||
|
const textView = document.getElementById('pdfTextView');
|
||||||
|
const pdfView = document.getElementById('manualEntryPdfViewer');
|
||||||
|
const icon = document.getElementById('pdfViewIcon');
|
||||||
|
|
||||||
|
if (textView.style.display === 'none') {
|
||||||
|
// Show text view
|
||||||
|
textView.style.display = 'block';
|
||||||
|
pdfView.style.display = 'none';
|
||||||
|
icon.className = 'bi bi-file-earmark-text';
|
||||||
|
} else {
|
||||||
|
// Show PDF view
|
||||||
|
textView.style.display = 'none';
|
||||||
|
pdfView.style.display = 'block';
|
||||||
|
icon.className = 'bi bi-file-earmark-pdf';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function loadVendorsForManual() {
|
async function loadVendorsForManual() {
|
||||||
try {
|
try {
|
||||||
console.log('Fetching vendors from API...');
|
console.log('Fetching vendors from API...');
|
||||||
|
|||||||
@ -340,10 +340,13 @@ async function runTest() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Load PDF text
|
// Load PDF text from dedicated endpoint (not reprocess)
|
||||||
const fileResponse = await fetch(`/api/v1/supplier-invoices/reprocess/${fileId}`, {
|
const fileResponse = await fetch(`/api/v1/supplier-invoices/files/${fileId}/pdf-text`);
|
||||||
method: 'POST'
|
|
||||||
});
|
if (!fileResponse.ok) {
|
||||||
|
throw new Error(`Kunne ikke læse PDF: ${fileResponse.status} ${fileResponse.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
const fileData = await fileResponse.json();
|
const fileData = await fileResponse.json();
|
||||||
const pdfText = fileData.pdf_text;
|
const pdfText = fileData.pdf_text;
|
||||||
|
|
||||||
@ -396,6 +399,8 @@ async function runTest() {
|
|||||||
let linesHtml = '';
|
let linesHtml = '';
|
||||||
const lineItems = result.line_items || [];
|
const lineItems = result.line_items || [];
|
||||||
if (lineItems.length > 0) {
|
if (lineItems.length > 0) {
|
||||||
|
const hasLineTotal = lineItems.some(l => (l.line_total !== undefined && l.line_total !== null && l.line_total !== '') || (l.lineTotal !== undefined && l.lineTotal !== null && l.lineTotal !== ''));
|
||||||
|
|
||||||
linesHtml = `
|
linesHtml = `
|
||||||
<h6 class="mt-3">Varelinjer (${lineItems.length} stk):</h6>
|
<h6 class="mt-3">Varelinjer (${lineItems.length} stk):</h6>
|
||||||
<div class="table-responsive">
|
<div class="table-responsive">
|
||||||
@ -406,6 +411,7 @@ async function runTest() {
|
|||||||
${lineItems[0].description ? '<th>Beskrivelse</th>' : ''}
|
${lineItems[0].description ? '<th>Beskrivelse</th>' : ''}
|
||||||
${lineItems[0].quantity ? '<th>Antal</th>' : ''}
|
${lineItems[0].quantity ? '<th>Antal</th>' : ''}
|
||||||
${lineItems[0].unit_price ? '<th>Pris</th>' : ''}
|
${lineItems[0].unit_price ? '<th>Pris</th>' : ''}
|
||||||
|
${hasLineTotal ? '<th>Beløb</th>' : ''}
|
||||||
${lineItems.some(l => l.circuit_id || l.ip_address) ? '<th>Kredsløb/IP</th>' : ''}
|
${lineItems.some(l => l.circuit_id || l.ip_address) ? '<th>Kredsløb/IP</th>' : ''}
|
||||||
${lineItems.some(l => l.location_street) ? '<th>Adresse</th>' : ''}
|
${lineItems.some(l => l.location_street) ? '<th>Adresse</th>' : ''}
|
||||||
</tr>
|
</tr>
|
||||||
@ -415,12 +421,14 @@ async function runTest() {
|
|||||||
lineItems.forEach((line, idx) => {
|
lineItems.forEach((line, idx) => {
|
||||||
const locationText = [line.location_street, line.location_zip, line.location_city].filter(x => x).join(' ');
|
const locationText = [line.location_street, line.location_zip, line.location_city].filter(x => x).join(' ');
|
||||||
const circuitText = line.circuit_id || line.ip_address || '';
|
const circuitText = line.circuit_id || line.ip_address || '';
|
||||||
|
const lineTotal = (line.line_total !== undefined && line.line_total !== null) ? line.line_total : line.lineTotal;
|
||||||
|
|
||||||
linesHtml += `<tr>
|
linesHtml += `<tr>
|
||||||
<td>${idx + 1}</td>
|
<td>${idx + 1}</td>
|
||||||
${line.description ? `<td>${line.description}</td>` : ''}
|
${line.description ? `<td>${line.description}</td>` : ''}
|
||||||
${line.quantity ? `<td>${line.quantity}</td>` : ''}
|
${line.quantity ? `<td>${line.quantity}</td>` : ''}
|
||||||
${line.unit_price ? `<td>${line.unit_price}</td>` : ''}
|
${line.unit_price ? `<td>${line.unit_price}</td>` : ''}
|
||||||
|
${hasLineTotal ? `<td>${(lineTotal !== undefined && lineTotal !== null) ? lineTotal : ''}</td>` : ''}
|
||||||
${lineItems.some(l => l.circuit_id || l.ip_address) ? `<td><small>${circuitText}</small></td>` : ''}
|
${lineItems.some(l => l.circuit_id || l.ip_address) ? `<td><small>${circuitText}</small></td>` : ''}
|
||||||
${lineItems.some(l => l.location_street) ? `<td><small>${locationText}</small></td>` : ''}
|
${lineItems.some(l => l.location_street) ? `<td><small>${locationText}</small></td>` : ''}
|
||||||
</tr>`;
|
</tr>`;
|
||||||
|
|||||||
@ -42,12 +42,15 @@ class Settings(BaseSettings):
|
|||||||
# Simply-CRM Integration (Legacy System med CVR data)
|
# Simply-CRM Integration (Legacy System med CVR data)
|
||||||
OLD_VTIGER_URL: str = "https://bmcnetworks.simply-crm.dk"
|
OLD_VTIGER_URL: str = "https://bmcnetworks.simply-crm.dk"
|
||||||
OLD_VTIGER_USERNAME: str = "ct"
|
OLD_VTIGER_USERNAME: str = "ct"
|
||||||
OLD_VTIGER_ACCESS_KEY: str = ""
|
OLD_VTIGER_ACCESS_KEY: str = "b00ff2b7c08d591"
|
||||||
|
|
||||||
# Time Tracking Module - vTiger Integration (Isoleret)
|
# Time Tracking Module - vTiger Integration (Isoleret)
|
||||||
TIMETRACKING_VTIGER_READ_ONLY: bool = True # 🚨 SAFETY: Bloker ALLE skrivninger til vTiger
|
TIMETRACKING_VTIGER_READ_ONLY: bool = True # 🚨 SAFETY: Bloker ALLE skrivninger til vTiger
|
||||||
TIMETRACKING_VTIGER_DRY_RUN: bool = True # 🚨 SAFETY: Log uden at synkronisere
|
TIMETRACKING_VTIGER_DRY_RUN: bool = True # 🚨 SAFETY: Log uden at synkronisere
|
||||||
|
|
||||||
|
# Time Tracking Module - Order Management
|
||||||
|
TIMETRACKING_ADMIN_UNLOCK_CODE: str = "" # Kode for at låse eksporterede ordrer op
|
||||||
|
|
||||||
# Time Tracking Module - e-conomic Integration (Isoleret)
|
# Time Tracking Module - e-conomic Integration (Isoleret)
|
||||||
TIMETRACKING_ECONOMIC_READ_ONLY: bool = True # 🚨 SAFETY: Bloker ALLE skrivninger til e-conomic
|
TIMETRACKING_ECONOMIC_READ_ONLY: bool = True # 🚨 SAFETY: Bloker ALLE skrivninger til e-conomic
|
||||||
TIMETRACKING_ECONOMIC_DRY_RUN: bool = True # 🚨 SAFETY: Log uden at eksportere
|
TIMETRACKING_ECONOMIC_DRY_RUN: bool = True # 🚨 SAFETY: Log uden at eksportere
|
||||||
@ -93,10 +96,13 @@ class Settings(BaseSettings):
|
|||||||
EMAIL_AI_CONFIDENCE_THRESHOLD: float = 0.7 # Minimum confidence for auto-processing
|
EMAIL_AI_CONFIDENCE_THRESHOLD: float = 0.7 # Minimum confidence for auto-processing
|
||||||
EMAIL_AUTO_CLASSIFY: bool = True # Run AI classification on new emails
|
EMAIL_AUTO_CLASSIFY: bool = True # Run AI classification on new emails
|
||||||
|
|
||||||
# Email Rules Engine
|
# Email Rules Engine (DEPRECATED - Use workflows instead)
|
||||||
EMAIL_RULES_ENABLED: bool = True
|
EMAIL_RULES_ENABLED: bool = False # 🚨 LEGACY: Disabled by default, use EMAIL_WORKFLOWS_ENABLED instead
|
||||||
EMAIL_RULES_AUTO_PROCESS: bool = False # 🚨 SAFETY: Require manual approval initially
|
EMAIL_RULES_AUTO_PROCESS: bool = False # 🚨 SAFETY: Require manual approval initially
|
||||||
|
|
||||||
|
# Email Workflows (RECOMMENDED)
|
||||||
|
EMAIL_WORKFLOWS_ENABLED: bool = True # Enable automated workflows based on classification (replaces rules)
|
||||||
|
|
||||||
# Company Info
|
# Company Info
|
||||||
OWN_CVR: str = "29522790" # BMC Denmark ApS - ignore when detecting vendors
|
OWN_CVR: str = "29522790" # BMC Denmark ApS - ignore when detecting vendors
|
||||||
|
|
||||||
@ -110,6 +116,49 @@ class Settings(BaseSettings):
|
|||||||
MODULES_DIR: str = "app/modules" # Directory for dynamic modules
|
MODULES_DIR: str = "app/modules" # Directory for dynamic modules
|
||||||
MODULES_AUTO_RELOAD: bool = True # Hot-reload modules on changes (dev only)
|
MODULES_AUTO_RELOAD: bool = True # Hot-reload modules on changes (dev only)
|
||||||
|
|
||||||
|
# Backup System Configuration
|
||||||
|
# Safety switches (default to safe mode)
|
||||||
|
BACKUP_ENABLED: bool = False # 🚨 SAFETY: Disable backups until explicitly enabled
|
||||||
|
BACKUP_DRY_RUN: bool = True # 🚨 SAFETY: Log operations without executing
|
||||||
|
BACKUP_READ_ONLY: bool = True # 🚨 SAFETY: Allow reads but block destructive operations
|
||||||
|
|
||||||
|
# Backup formats
|
||||||
|
DB_DAILY_FORMAT: str = "dump" # dump (compressed) or sql (plain text)
|
||||||
|
DB_MONTHLY_FORMAT: str = "sql" # Monthly backups use plain SQL for readability
|
||||||
|
|
||||||
|
# Backup scope
|
||||||
|
BACKUP_INCLUDE_UPLOADS: bool = True # Include uploads/ directory
|
||||||
|
BACKUP_INCLUDE_LOGS: bool = True # Include logs/ directory
|
||||||
|
BACKUP_INCLUDE_DATA: bool = True # Include data/ directory (templates, configs)
|
||||||
|
|
||||||
|
# Storage configuration
|
||||||
|
BACKUP_STORAGE_PATH: str = "/opt/backups" # Production: /opt/backups, Dev: ./backups
|
||||||
|
BACKUP_MAX_SIZE_GB: int = 50 # Maximum total backup storage size
|
||||||
|
STORAGE_WARNING_THRESHOLD_PCT: int = 80 # Warn when storage exceeds this percentage
|
||||||
|
|
||||||
|
# Rotation policy
|
||||||
|
RETENTION_DAYS: int = 30 # Keep daily backups for 30 days
|
||||||
|
MONTHLY_KEEP_MONTHS: int = 12 # Keep monthly backups for 12 months
|
||||||
|
|
||||||
|
# Offsite configuration (SFTP/SSH)
|
||||||
|
OFFSITE_ENABLED: bool = False # 🚨 SAFETY: Disable offsite uploads until configured
|
||||||
|
OFFSITE_WEEKLY_DAY: str = "sunday" # Day for weekly offsite upload (monday-sunday)
|
||||||
|
OFFSITE_RETRY_MAX_ATTEMPTS: int = 3 # Maximum retry attempts for failed uploads
|
||||||
|
OFFSITE_RETRY_DELAY_HOURS: int = 1 # Hours between retry attempts
|
||||||
|
SFTP_HOST: str = "" # SFTP server hostname or IP
|
||||||
|
SFTP_PORT: int = 22 # SFTP server port
|
||||||
|
SFTP_USER: str = "" # SFTP username
|
||||||
|
SFTP_PASSWORD: str = "" # SFTP password (if not using SSH key)
|
||||||
|
SSH_KEY_PATH: str = "" # Path to SSH private key (preferred over password)
|
||||||
|
SFTP_REMOTE_PATH: str = "/backups/bmc_hub" # Remote directory for backups
|
||||||
|
|
||||||
|
# Notification configuration (Mattermost)
|
||||||
|
MATTERMOST_ENABLED: bool = False # 🚨 SAFETY: Disable until webhook configured
|
||||||
|
MATTERMOST_WEBHOOK_URL: str = "" # Mattermost incoming webhook URL
|
||||||
|
MATTERMOST_CHANNEL: str = "backups" # Channel name for backup notifications
|
||||||
|
NOTIFY_ON_FAILURE: bool = True # Send notification on backup/offsite failures
|
||||||
|
NOTIFY_ON_SUCCESS_OFFSITE: bool = True # Send notification on successful offsite upload
|
||||||
|
|
||||||
class Config:
|
class Config:
|
||||||
env_file = ".env"
|
env_file = ".env"
|
||||||
case_sensitive = True
|
case_sensitive = True
|
||||||
|
|||||||
@ -881,6 +881,7 @@ function renderSalesOrdersList(orders) {
|
|||||||
|
|
||||||
<div class="d-flex gap-2 flex-wrap small text-muted mt-2">
|
<div class="d-flex gap-2 flex-wrap small text-muted mt-2">
|
||||||
${order.recurring_frequency ? `<span class="badge bg-light text-dark"><i class="bi bi-arrow-repeat me-1"></i>${escapeHtml(order.recurring_frequency)}</span>` : ''}
|
${order.recurring_frequency ? `<span class="badge bg-light text-dark"><i class="bi bi-arrow-repeat me-1"></i>${escapeHtml(order.recurring_frequency)}</span>` : ''}
|
||||||
|
${order.last_recurring_date ? `<span class="badge bg-info text-dark"><i class="bi bi-calendar-check me-1"></i>Last: ${formatDate(order.last_recurring_date)}</span>` : ''}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
${hasLineItems ? `
|
${hasLineItems ? `
|
||||||
@ -1032,6 +1033,7 @@ function renderSubscriptionsList(subscriptions, isLocked = false) {
|
|||||||
|
|
||||||
<div class="d-flex gap-2 flex-wrap small text-muted mt-2">
|
<div class="d-flex gap-2 flex-wrap small text-muted mt-2">
|
||||||
${sub.generateinvoiceevery ? `<span class="badge bg-light text-dark"><i class="bi bi-arrow-repeat me-1"></i>${escapeHtml(sub.generateinvoiceevery)}</span>` : ''}
|
${sub.generateinvoiceevery ? `<span class="badge bg-light text-dark"><i class="bi bi-arrow-repeat me-1"></i>${escapeHtml(sub.generateinvoiceevery)}</span>` : ''}
|
||||||
|
${sub.next_subscription_date ? `<span class="badge bg-warning text-dark"><i class="bi bi-calendar-event me-1"></i>Next: ${formatDate(sub.next_subscription_date)}</span>` : ''}
|
||||||
${sub.startdate && sub.enddate ? `<span class="badge bg-light text-dark"><i class="bi bi-calendar-range me-1"></i>${formatDate(sub.startdate)} - ${formatDate(sub.enddate)}</span>` : ''}
|
${sub.startdate && sub.enddate ? `<span class="badge bg-light text-dark"><i class="bi bi-calendar-range me-1"></i>${formatDate(sub.startdate)} - ${formatDate(sub.enddate)}</span>` : ''}
|
||||||
${sub.startdate ? `<span class="badge bg-light text-dark"><i class="bi bi-calendar-check me-1"></i>Start: ${formatDate(sub.startdate)}</span>` : ''}
|
${sub.startdate ? `<span class="badge bg-light text-dark"><i class="bi bi-calendar-check me-1"></i>Start: ${formatDate(sub.startdate)}</span>` : ''}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@ -11,6 +11,7 @@ from datetime import datetime, date
|
|||||||
|
|
||||||
from app.core.database import execute_query, execute_insert, execute_update
|
from app.core.database import execute_query, execute_insert, execute_update
|
||||||
from app.services.email_processor_service import EmailProcessorService
|
from app.services.email_processor_service import EmailProcessorService
|
||||||
|
from app.services.email_workflow_service import email_workflow_service
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@ -90,6 +91,49 @@ class EmailRule(BaseModel):
|
|||||||
last_matched_at: Optional[datetime]
|
last_matched_at: Optional[datetime]
|
||||||
|
|
||||||
|
|
||||||
|
class EmailWorkflow(BaseModel):
|
||||||
|
id: Optional[int] = None
|
||||||
|
name: str
|
||||||
|
description: Optional[str]
|
||||||
|
classification_trigger: str
|
||||||
|
sender_pattern: Optional[str] = None
|
||||||
|
subject_pattern: Optional[str] = None
|
||||||
|
confidence_threshold: float = 0.70
|
||||||
|
workflow_steps: List[dict]
|
||||||
|
priority: int = 100
|
||||||
|
enabled: bool = True
|
||||||
|
stop_on_match: bool = True
|
||||||
|
execution_count: int = 0
|
||||||
|
success_count: int = 0
|
||||||
|
failure_count: int = 0
|
||||||
|
last_executed_at: Optional[datetime] = None
|
||||||
|
|
||||||
|
|
||||||
|
class WorkflowExecution(BaseModel):
|
||||||
|
id: int
|
||||||
|
workflow_id: int
|
||||||
|
email_id: int
|
||||||
|
status: str
|
||||||
|
steps_completed: int
|
||||||
|
steps_total: Optional[int]
|
||||||
|
result_json: Optional[List[dict]] = None # Can be list of step results
|
||||||
|
error_message: Optional[str]
|
||||||
|
started_at: datetime
|
||||||
|
completed_at: Optional[datetime]
|
||||||
|
execution_time_ms: Optional[int]
|
||||||
|
|
||||||
|
|
||||||
|
class WorkflowAction(BaseModel):
|
||||||
|
id: int
|
||||||
|
action_code: str
|
||||||
|
name: str
|
||||||
|
description: Optional[str]
|
||||||
|
category: Optional[str]
|
||||||
|
parameter_schema: Optional[dict]
|
||||||
|
example_config: Optional[dict]
|
||||||
|
enabled: bool
|
||||||
|
|
||||||
|
|
||||||
class ProcessingStats(BaseModel):
|
class ProcessingStats(BaseModel):
|
||||||
status: str
|
status: str
|
||||||
fetched: int = 0
|
fetched: int = 0
|
||||||
@ -183,6 +227,41 @@ async def get_email(email_id: int):
|
|||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/emails/{email_id}/mark-processed")
|
||||||
|
async def mark_email_processed(email_id: int):
|
||||||
|
"""Mark email as processed and move to 'Processed' folder"""
|
||||||
|
try:
|
||||||
|
# Update email status and folder
|
||||||
|
update_query = """
|
||||||
|
UPDATE email_messages
|
||||||
|
SET status = 'processed',
|
||||||
|
folder = 'Processed',
|
||||||
|
processed_at = CURRENT_TIMESTAMP,
|
||||||
|
updated_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE id = %s AND deleted_at IS NULL
|
||||||
|
RETURNING id, folder, status
|
||||||
|
"""
|
||||||
|
result = execute_query(update_query, (email_id,), fetchone=True)
|
||||||
|
|
||||||
|
if not result:
|
||||||
|
raise HTTPException(status_code=404, detail="Email not found")
|
||||||
|
|
||||||
|
logger.info(f"✅ Email {email_id} marked as processed and moved to Processed folder")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"email_id": result.get('id') if result else email_id,
|
||||||
|
"folder": result.get('folder') if result else 'Processed',
|
||||||
|
"status": result.get('status') if result else 'processed'
|
||||||
|
}
|
||||||
|
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Error marking email {email_id} as processed: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
@router.get("/emails/{email_id}/attachments/{attachment_id}")
|
@router.get("/emails/{email_id}/attachments/{attachment_id}")
|
||||||
async def download_attachment(email_id: int, attachment_id: int):
|
async def download_attachment(email_id: int, attachment_id: int):
|
||||||
"""Download email attachment"""
|
"""Download email attachment"""
|
||||||
@ -613,3 +692,390 @@ async def get_email_stats():
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"❌ Error getting stats: {e}")
|
logger.error(f"❌ Error getting stats: {e}")
|
||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
# ========== Workflow Endpoints ==========
|
||||||
|
|
||||||
|
@router.get("/workflows", response_model=List[EmailWorkflow])
|
||||||
|
async def list_workflows():
|
||||||
|
"""Get all email workflows"""
|
||||||
|
try:
|
||||||
|
query = """
|
||||||
|
SELECT * FROM email_workflows
|
||||||
|
ORDER BY priority ASC, name ASC
|
||||||
|
"""
|
||||||
|
result = execute_query(query)
|
||||||
|
return result
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Error listing workflows: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/workflows/{workflow_id}", response_model=EmailWorkflow)
|
||||||
|
async def get_workflow(workflow_id: int):
|
||||||
|
"""Get specific workflow by ID"""
|
||||||
|
try:
|
||||||
|
query = "SELECT * FROM email_workflows WHERE id = %s"
|
||||||
|
result = execute_query(query, (workflow_id,), fetchone=True)
|
||||||
|
|
||||||
|
if not result:
|
||||||
|
raise HTTPException(status_code=404, detail="Workflow not found")
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Error getting workflow: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/workflows", response_model=EmailWorkflow)
|
||||||
|
async def create_workflow(workflow: EmailWorkflow):
|
||||||
|
"""Create new email workflow"""
|
||||||
|
try:
|
||||||
|
import json
|
||||||
|
|
||||||
|
query = """
|
||||||
|
INSERT INTO email_workflows
|
||||||
|
(name, description, classification_trigger, sender_pattern, subject_pattern,
|
||||||
|
confidence_threshold, workflow_steps, priority, enabled, stop_on_match, created_by_user_id)
|
||||||
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 1)
|
||||||
|
RETURNING *
|
||||||
|
"""
|
||||||
|
|
||||||
|
result = execute_query(query, (
|
||||||
|
workflow.name,
|
||||||
|
workflow.description,
|
||||||
|
workflow.classification_trigger,
|
||||||
|
workflow.sender_pattern,
|
||||||
|
workflow.subject_pattern,
|
||||||
|
workflow.confidence_threshold,
|
||||||
|
json.dumps(workflow.workflow_steps),
|
||||||
|
workflow.priority,
|
||||||
|
workflow.enabled,
|
||||||
|
workflow.stop_on_match
|
||||||
|
), fetchone=True)
|
||||||
|
|
||||||
|
if result:
|
||||||
|
logger.info(f"✅ Created workflow: {workflow.name}")
|
||||||
|
return result
|
||||||
|
else:
|
||||||
|
raise HTTPException(status_code=500, detail="Failed to create workflow")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Error creating workflow: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/workflows/{workflow_id}", response_model=EmailWorkflow)
|
||||||
|
async def update_workflow(workflow_id: int, workflow: EmailWorkflow):
|
||||||
|
"""Update existing email workflow"""
|
||||||
|
try:
|
||||||
|
import json
|
||||||
|
|
||||||
|
query = """
|
||||||
|
UPDATE email_workflows
|
||||||
|
SET name = %s,
|
||||||
|
description = %s,
|
||||||
|
classification_trigger = %s,
|
||||||
|
sender_pattern = %s,
|
||||||
|
subject_pattern = %s,
|
||||||
|
confidence_threshold = %s,
|
||||||
|
workflow_steps = %s,
|
||||||
|
priority = %s,
|
||||||
|
enabled = %s,
|
||||||
|
stop_on_match = %s
|
||||||
|
WHERE id = %s
|
||||||
|
RETURNING *
|
||||||
|
"""
|
||||||
|
|
||||||
|
result = execute_query(query, (
|
||||||
|
workflow.name,
|
||||||
|
workflow.description,
|
||||||
|
workflow.classification_trigger,
|
||||||
|
workflow.sender_pattern,
|
||||||
|
workflow.subject_pattern,
|
||||||
|
workflow.confidence_threshold,
|
||||||
|
json.dumps(workflow.workflow_steps),
|
||||||
|
workflow.priority,
|
||||||
|
workflow.enabled,
|
||||||
|
workflow.stop_on_match,
|
||||||
|
workflow_id
|
||||||
|
), fetchone=True)
|
||||||
|
|
||||||
|
if result:
|
||||||
|
logger.info(f"✅ Updated workflow {workflow_id}")
|
||||||
|
return result
|
||||||
|
else:
|
||||||
|
raise HTTPException(status_code=404, detail="Workflow not found")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Error updating workflow: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/workflows/{workflow_id}")
|
||||||
|
async def delete_workflow(workflow_id: int):
|
||||||
|
"""Delete email workflow"""
|
||||||
|
try:
|
||||||
|
query = "DELETE FROM email_workflows WHERE id = %s"
|
||||||
|
execute_update(query, (workflow_id,))
|
||||||
|
|
||||||
|
logger.info(f"🗑️ Deleted workflow {workflow_id}")
|
||||||
|
return {"success": True, "message": f"Workflow {workflow_id} deleted"}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Error deleting workflow: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/workflows/{workflow_id}/toggle")
|
||||||
|
async def toggle_workflow(workflow_id: int):
|
||||||
|
"""Toggle workflow enabled status"""
|
||||||
|
try:
|
||||||
|
query = """
|
||||||
|
UPDATE email_workflows
|
||||||
|
SET enabled = NOT enabled
|
||||||
|
WHERE id = %s
|
||||||
|
RETURNING enabled
|
||||||
|
"""
|
||||||
|
result = execute_query(query, (workflow_id,), fetchone=True)
|
||||||
|
|
||||||
|
if not result:
|
||||||
|
raise HTTPException(status_code=404, detail="Workflow not found")
|
||||||
|
|
||||||
|
status = "enabled" if result['enabled'] else "disabled"
|
||||||
|
logger.info(f"🔄 Workflow {workflow_id} {status}")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"workflow_id": workflow_id,
|
||||||
|
"enabled": result['enabled']
|
||||||
|
}
|
||||||
|
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Error toggling workflow: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/emails/{email_id}/execute-workflows")
|
||||||
|
async def execute_workflows_for_email(email_id: int):
|
||||||
|
"""Manually trigger workflow execution for an email"""
|
||||||
|
try:
|
||||||
|
# Get email data
|
||||||
|
query = """
|
||||||
|
SELECT id, message_id, subject, sender_email, sender_name, body_text,
|
||||||
|
classification, confidence_score, status
|
||||||
|
FROM email_messages
|
||||||
|
WHERE id = %s AND deleted_at IS NULL
|
||||||
|
"""
|
||||||
|
email_data = execute_query(query, (email_id,), fetchone=True)
|
||||||
|
|
||||||
|
if not email_data:
|
||||||
|
raise HTTPException(status_code=404, detail="Email not found")
|
||||||
|
|
||||||
|
# Execute workflows
|
||||||
|
result = await email_workflow_service.execute_workflows(email_data)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Error executing workflows: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/workflow-executions", response_model=List[WorkflowExecution])
|
||||||
|
async def list_workflow_executions(
|
||||||
|
workflow_id: Optional[int] = Query(None),
|
||||||
|
email_id: Optional[int] = Query(None),
|
||||||
|
status: Optional[str] = Query(None),
|
||||||
|
limit: int = Query(50, le=500)
|
||||||
|
):
|
||||||
|
"""Get workflow execution history"""
|
||||||
|
try:
|
||||||
|
where_clauses = []
|
||||||
|
params = []
|
||||||
|
|
||||||
|
if workflow_id:
|
||||||
|
where_clauses.append("workflow_id = %s")
|
||||||
|
params.append(workflow_id)
|
||||||
|
|
||||||
|
if email_id:
|
||||||
|
where_clauses.append("email_id = %s")
|
||||||
|
params.append(email_id)
|
||||||
|
|
||||||
|
if status:
|
||||||
|
where_clauses.append("status = %s")
|
||||||
|
params.append(status)
|
||||||
|
|
||||||
|
where_sql = " AND ".join(where_clauses) if where_clauses else "1=1"
|
||||||
|
|
||||||
|
query = f"""
|
||||||
|
SELECT * FROM email_workflow_executions
|
||||||
|
WHERE {where_sql}
|
||||||
|
ORDER BY started_at DESC
|
||||||
|
LIMIT %s
|
||||||
|
"""
|
||||||
|
|
||||||
|
params.append(limit)
|
||||||
|
result = execute_query(query, tuple(params))
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Error listing workflow executions: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/workflow-actions", response_model=List[WorkflowAction])
|
||||||
|
async def list_workflow_actions():
|
||||||
|
"""Get all available workflow actions"""
|
||||||
|
try:
|
||||||
|
query = """
|
||||||
|
SELECT * FROM email_workflow_actions
|
||||||
|
WHERE enabled = true
|
||||||
|
ORDER BY category, name
|
||||||
|
"""
|
||||||
|
result = execute_query(query)
|
||||||
|
return result
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Error listing workflow actions: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/workflows/stats/summary")
|
||||||
|
async def get_workflow_stats():
|
||||||
|
"""Get workflow execution statistics"""
|
||||||
|
try:
|
||||||
|
query = """
|
||||||
|
SELECT * FROM v_workflow_stats
|
||||||
|
"""
|
||||||
|
result = execute_query(query)
|
||||||
|
return result
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Error getting workflow stats: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
# ========== Email Activity Log Endpoints ==========
|
||||||
|
|
||||||
|
class EmailActivityLog(BaseModel):
|
||||||
|
id: int
|
||||||
|
email_id: int
|
||||||
|
event_type: str
|
||||||
|
event_category: str
|
||||||
|
description: str
|
||||||
|
metadata: Optional[dict]
|
||||||
|
user_id: Optional[int]
|
||||||
|
user_name: Optional[str]
|
||||||
|
created_at: datetime
|
||||||
|
created_by: str
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/emails/{email_id}/activity", response_model=List[EmailActivityLog])
|
||||||
|
async def get_email_activity_log(email_id: int, limit: int = Query(default=100, le=500)):
|
||||||
|
"""Get complete activity log for an email"""
|
||||||
|
try:
|
||||||
|
query = """
|
||||||
|
SELECT
|
||||||
|
eal.id,
|
||||||
|
eal.email_id,
|
||||||
|
eal.event_type,
|
||||||
|
eal.event_category,
|
||||||
|
eal.description,
|
||||||
|
eal.metadata,
|
||||||
|
eal.user_id,
|
||||||
|
u.username as user_name,
|
||||||
|
eal.created_at,
|
||||||
|
eal.created_by
|
||||||
|
FROM email_activity_log eal
|
||||||
|
LEFT JOIN users u ON eal.user_id = u.user_id
|
||||||
|
WHERE eal.email_id = %s
|
||||||
|
ORDER BY eal.created_at DESC
|
||||||
|
LIMIT %s
|
||||||
|
"""
|
||||||
|
result = execute_query(query, (email_id, limit))
|
||||||
|
return result
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Error getting email activity log: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/emails/activity/recent", response_model=List[EmailActivityLog])
|
||||||
|
async def get_recent_activity(
|
||||||
|
limit: int = Query(default=50, le=200),
|
||||||
|
event_type: Optional[str] = None,
|
||||||
|
event_category: Optional[str] = None
|
||||||
|
):
|
||||||
|
"""Get recent email activity across all emails"""
|
||||||
|
try:
|
||||||
|
conditions = []
|
||||||
|
params = []
|
||||||
|
|
||||||
|
if event_type:
|
||||||
|
conditions.append("eal.event_type = %s")
|
||||||
|
params.append(event_type)
|
||||||
|
|
||||||
|
if event_category:
|
||||||
|
conditions.append("eal.event_category = %s")
|
||||||
|
params.append(event_category)
|
||||||
|
|
||||||
|
where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else ""
|
||||||
|
params.append(limit)
|
||||||
|
|
||||||
|
query = f"""
|
||||||
|
SELECT
|
||||||
|
eal.id,
|
||||||
|
eal.email_id,
|
||||||
|
eal.event_type,
|
||||||
|
eal.event_category,
|
||||||
|
eal.description,
|
||||||
|
eal.metadata,
|
||||||
|
eal.user_id,
|
||||||
|
u.username as user_name,
|
||||||
|
eal.created_at,
|
||||||
|
eal.created_by
|
||||||
|
FROM email_activity_log eal
|
||||||
|
LEFT JOIN users u ON eal.user_id = u.user_id
|
||||||
|
{where_clause}
|
||||||
|
ORDER BY eal.created_at DESC
|
||||||
|
LIMIT %s
|
||||||
|
"""
|
||||||
|
result = execute_query(query, tuple(params))
|
||||||
|
return result
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Error getting recent activity: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/emails/activity/stats")
|
||||||
|
async def get_activity_stats():
|
||||||
|
"""Get activity statistics"""
|
||||||
|
try:
|
||||||
|
query = """
|
||||||
|
SELECT
|
||||||
|
event_type,
|
||||||
|
event_category,
|
||||||
|
COUNT(*) as count,
|
||||||
|
MAX(created_at) as last_occurrence
|
||||||
|
FROM email_activity_log
|
||||||
|
WHERE created_at >= NOW() - INTERVAL '7 days'
|
||||||
|
GROUP BY event_type, event_category
|
||||||
|
ORDER BY count DESC
|
||||||
|
"""
|
||||||
|
result = execute_query(query)
|
||||||
|
return result
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Error getting activity stats: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
213
app/services/email_activity_logger.py
Normal file
213
app/services/email_activity_logger.py
Normal file
@ -0,0 +1,213 @@
|
|||||||
|
"""
|
||||||
|
Email Activity Logger
|
||||||
|
Helper service for logging all email events
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Optional, Dict, Any
|
||||||
|
import json
|
||||||
|
from app.core.database import execute_query, execute_insert
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class EmailActivityLogger:
|
||||||
|
"""Centralized email activity logging"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def log(
|
||||||
|
email_id: int,
|
||||||
|
event_type: str,
|
||||||
|
description: str,
|
||||||
|
category: str = 'system',
|
||||||
|
metadata: Optional[Dict[str, Any]] = None,
|
||||||
|
user_id: Optional[int] = None,
|
||||||
|
created_by: str = 'system'
|
||||||
|
) -> Optional[int]:
|
||||||
|
"""
|
||||||
|
Log an email activity event
|
||||||
|
|
||||||
|
Args:
|
||||||
|
email_id: Email ID
|
||||||
|
event_type: Type of event (fetched, classified, workflow_executed, etc.)
|
||||||
|
description: Human-readable description
|
||||||
|
category: Event category (system, user, workflow, rule, integration)
|
||||||
|
metadata: Additional event data as dict
|
||||||
|
user_id: User ID if user-triggered
|
||||||
|
created_by: Who/what created this log entry
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Log ID or None on failure
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
metadata_json = json.dumps(metadata) if metadata else None
|
||||||
|
|
||||||
|
log_id = execute_insert(
|
||||||
|
"""INSERT INTO email_activity_log
|
||||||
|
(email_id, event_type, event_category, description, metadata, user_id, created_by)
|
||||||
|
VALUES (%s, %s, %s, %s, %s::jsonb, %s, %s)""",
|
||||||
|
(email_id, event_type, category, description, metadata_json, user_id, created_by)
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.debug(f"📝 Logged email event: {event_type} for email {email_id}")
|
||||||
|
return log_id
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Failed to log email activity: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def log_fetched(email_id: int, source: str, message_id: str):
|
||||||
|
"""Log email fetch event"""
|
||||||
|
return await EmailActivityLogger.log(
|
||||||
|
email_id=email_id,
|
||||||
|
event_type='fetched',
|
||||||
|
category='system',
|
||||||
|
description=f'Email fetched from {source}',
|
||||||
|
metadata={'source': source, 'message_id': message_id}
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def log_classified(email_id: int, classification: str, confidence: float, method: str = 'ai'):
|
||||||
|
"""Log email classification"""
|
||||||
|
return await EmailActivityLogger.log(
|
||||||
|
email_id=email_id,
|
||||||
|
event_type='classified',
|
||||||
|
category='system',
|
||||||
|
description=f'Classified as {classification} (confidence: {confidence:.2%})',
|
||||||
|
metadata={
|
||||||
|
'classification': classification,
|
||||||
|
'confidence': confidence,
|
||||||
|
'method': method
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def log_workflow_executed(email_id: int, workflow_id: int, workflow_name: str,
|
||||||
|
status: str, steps_completed: int, execution_time_ms: int):
|
||||||
|
"""Log workflow execution"""
|
||||||
|
return await EmailActivityLogger.log(
|
||||||
|
email_id=email_id,
|
||||||
|
event_type='workflow_executed',
|
||||||
|
category='workflow',
|
||||||
|
description=f'Workflow "{workflow_name}" {status} ({steps_completed} steps, {execution_time_ms}ms)',
|
||||||
|
metadata={
|
||||||
|
'workflow_id': workflow_id,
|
||||||
|
'workflow_name': workflow_name,
|
||||||
|
'status': status,
|
||||||
|
'steps_completed': steps_completed,
|
||||||
|
'execution_time_ms': execution_time_ms
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def log_rule_matched(email_id: int, rule_id: int, rule_name: str, action_type: str):
|
||||||
|
"""Log rule match"""
|
||||||
|
return await EmailActivityLogger.log(
|
||||||
|
email_id=email_id,
|
||||||
|
event_type='rule_matched',
|
||||||
|
category='rule',
|
||||||
|
description=f'Matched rule "{rule_name}" → action: {action_type}',
|
||||||
|
metadata={
|
||||||
|
'rule_id': rule_id,
|
||||||
|
'rule_name': rule_name,
|
||||||
|
'action_type': action_type
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def log_status_changed(email_id: int, old_status: str, new_status: str, reason: Optional[str] = None):
|
||||||
|
"""Log status change"""
|
||||||
|
desc = f'Status changed: {old_status} → {new_status}'
|
||||||
|
if reason:
|
||||||
|
desc += f' ({reason})'
|
||||||
|
|
||||||
|
return await EmailActivityLogger.log(
|
||||||
|
email_id=email_id,
|
||||||
|
event_type='status_changed',
|
||||||
|
category='system',
|
||||||
|
description=desc,
|
||||||
|
metadata={
|
||||||
|
'old_status': old_status,
|
||||||
|
'new_status': new_status,
|
||||||
|
'reason': reason
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def log_read(email_id: int, user_id: Optional[int] = None, username: Optional[str] = None):
|
||||||
|
"""Log email read"""
|
||||||
|
return await EmailActivityLogger.log(
|
||||||
|
email_id=email_id,
|
||||||
|
event_type='read',
|
||||||
|
category='user',
|
||||||
|
description=f'Email read by {username or "user"}',
|
||||||
|
metadata={'username': username},
|
||||||
|
user_id=user_id,
|
||||||
|
created_by=username or 'user'
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def log_attachment_action(email_id: int, filename: str, action: str = 'downloaded'):
|
||||||
|
"""Log attachment action"""
|
||||||
|
return await EmailActivityLogger.log(
|
||||||
|
email_id=email_id,
|
||||||
|
event_type=f'attachment_{action}',
|
||||||
|
category='user',
|
||||||
|
description=f'Attachment {action}: {filename}',
|
||||||
|
metadata={'filename': filename, 'action': action}
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def log_linked(email_id: int, entity_type: str, entity_id: int, entity_name: str):
|
||||||
|
"""Log entity linking"""
|
||||||
|
return await EmailActivityLogger.log(
|
||||||
|
email_id=email_id,
|
||||||
|
event_type='linked',
|
||||||
|
category='system',
|
||||||
|
description=f'Linked to {entity_type}: {entity_name}',
|
||||||
|
metadata={
|
||||||
|
'entity_type': entity_type,
|
||||||
|
'entity_id': entity_id,
|
||||||
|
'entity_name': entity_name
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def log_invoice_extracted(email_id: int, invoice_number: Optional[str],
|
||||||
|
amount: Optional[float], success: bool):
|
||||||
|
"""Log invoice data extraction"""
|
||||||
|
desc = f'Invoice data extraction {"succeeded" if success else "failed"}'
|
||||||
|
if success and invoice_number:
|
||||||
|
desc += f' - #{invoice_number}'
|
||||||
|
|
||||||
|
return await EmailActivityLogger.log(
|
||||||
|
email_id=email_id,
|
||||||
|
event_type='invoice_extracted',
|
||||||
|
category='integration',
|
||||||
|
description=desc,
|
||||||
|
metadata={
|
||||||
|
'success': success,
|
||||||
|
'invoice_number': invoice_number,
|
||||||
|
'amount': amount
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def log_error(email_id: int, error_type: str, error_message: str, context: Optional[Dict] = None):
|
||||||
|
"""Log error event"""
|
||||||
|
return await EmailActivityLogger.log(
|
||||||
|
email_id=email_id,
|
||||||
|
event_type='error',
|
||||||
|
category='system',
|
||||||
|
description=f'Error: {error_type} - {error_message}',
|
||||||
|
metadata={
|
||||||
|
'error_type': error_type,
|
||||||
|
'error_message': error_message,
|
||||||
|
'context': context
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Export singleton instance
|
||||||
|
email_activity_logger = EmailActivityLogger()
|
||||||
@ -11,6 +11,8 @@ from datetime import datetime
|
|||||||
from app.services.email_service import EmailService
|
from app.services.email_service import EmailService
|
||||||
from app.services.email_analysis_service import EmailAnalysisService
|
from app.services.email_analysis_service import EmailAnalysisService
|
||||||
from app.services.simple_classifier import simple_classifier
|
from app.services.simple_classifier import simple_classifier
|
||||||
|
from app.services.email_workflow_service import email_workflow_service
|
||||||
|
from app.services.email_activity_logger import email_activity_logger
|
||||||
from app.core.config import settings
|
from app.core.config import settings
|
||||||
from app.core.database import execute_query, execute_update
|
from app.core.database import execute_query, execute_update
|
||||||
|
|
||||||
@ -69,16 +71,45 @@ class EmailProcessorService:
|
|||||||
email_data['id'] = email_id
|
email_data['id'] = email_id
|
||||||
stats['saved'] += 1
|
stats['saved'] += 1
|
||||||
|
|
||||||
|
# Log: Email fetched and saved
|
||||||
|
await email_activity_logger.log_fetched(
|
||||||
|
email_id=email_id,
|
||||||
|
source='email_server',
|
||||||
|
message_id=email_data.get('message_id', 'unknown')
|
||||||
|
)
|
||||||
|
|
||||||
# Step 3: Classify email with AI
|
# Step 3: Classify email with AI
|
||||||
if settings.EMAIL_AI_ENABLED and settings.EMAIL_AUTO_CLASSIFY:
|
if settings.EMAIL_AI_ENABLED and settings.EMAIL_AUTO_CLASSIFY:
|
||||||
await self._classify_and_update(email_data)
|
await self._classify_and_update(email_data)
|
||||||
stats['classified'] += 1
|
stats['classified'] += 1
|
||||||
|
|
||||||
# Step 4: Match against rules
|
# Step 4: Execute workflows based on classification
|
||||||
if self.rules_enabled:
|
workflow_processed = False
|
||||||
|
if hasattr(settings, 'EMAIL_WORKFLOWS_ENABLED') and settings.EMAIL_WORKFLOWS_ENABLED:
|
||||||
|
workflow_result = await email_workflow_service.execute_workflows(email_data)
|
||||||
|
if workflow_result.get('workflows_executed', 0) > 0:
|
||||||
|
logger.info(f"✅ Executed {workflow_result['workflows_executed']} workflow(s) for email {email_id}")
|
||||||
|
# Mark as workflow-processed to avoid duplicate rule execution
|
||||||
|
if workflow_result.get('workflows_succeeded', 0) > 0:
|
||||||
|
workflow_processed = True
|
||||||
|
email_data['_workflow_processed'] = True
|
||||||
|
|
||||||
|
# Step 5: Match against rules (legacy support) - skip if workflow already processed
|
||||||
|
if self.rules_enabled and not workflow_processed:
|
||||||
|
# Check if workflow already processed this email
|
||||||
|
existing_execution = execute_query(
|
||||||
|
"SELECT id FROM email_workflow_executions WHERE email_id = %s AND status = 'completed' LIMIT 1",
|
||||||
|
(email_id,), fetchone=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if existing_execution:
|
||||||
|
logger.info(f"⏭️ Email {email_id} already processed by workflow, skipping rules")
|
||||||
|
else:
|
||||||
matched = await self._match_rules(email_data)
|
matched = await self._match_rules(email_data)
|
||||||
if matched:
|
if matched:
|
||||||
stats['rules_matched'] += 1
|
stats['rules_matched'] += 1
|
||||||
|
elif workflow_processed:
|
||||||
|
logger.info(f"⏭️ Email {email_id} processed by workflow, skipping rules (coordination)")
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"❌ Error processing email: {e}")
|
logger.error(f"❌ Error processing email: {e}")
|
||||||
@ -117,8 +148,20 @@ class EmailProcessorService:
|
|||||||
"""
|
"""
|
||||||
execute_update(query, (classification, confidence, email_data['id']))
|
execute_update(query, (classification, confidence, email_data['id']))
|
||||||
|
|
||||||
|
# Update email_data dict with classification for workflow execution
|
||||||
|
email_data['classification'] = classification
|
||||||
|
email_data['confidence_score'] = confidence
|
||||||
|
|
||||||
logger.info(f"✅ Classified email {email_data['id']} as '{classification}' (confidence: {confidence:.2f})")
|
logger.info(f"✅ Classified email {email_data['id']} as '{classification}' (confidence: {confidence:.2f})")
|
||||||
|
|
||||||
|
# Log: Email classified
|
||||||
|
await email_activity_logger.log_classified(
|
||||||
|
email_id=email_data['id'],
|
||||||
|
classification=classification,
|
||||||
|
confidence=confidence,
|
||||||
|
method='ai' if self.ai_enabled else 'keyword'
|
||||||
|
)
|
||||||
|
|
||||||
# Update email_data for rule matching
|
# Update email_data for rule matching
|
||||||
email_data['classification'] = classification
|
email_data['classification'] = classification
|
||||||
email_data['confidence_score'] = confidence
|
email_data['confidence_score'] = confidence
|
||||||
|
|||||||
620
app/services/email_workflow_service.py
Normal file
620
app/services/email_workflow_service.py
Normal file
@ -0,0 +1,620 @@
|
|||||||
|
"""
|
||||||
|
Email Workflow Service
|
||||||
|
Executes automated workflows based on email classification
|
||||||
|
Inspired by OmniSync architecture adapted for BMC Hub
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Dict, List, Optional, Any
|
||||||
|
from datetime import datetime, date
|
||||||
|
import re
|
||||||
|
import json
|
||||||
|
import hashlib
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
from decimal import Decimal
|
||||||
|
|
||||||
|
from app.core.database import execute_query, execute_insert, execute_update
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.services.email_activity_logger import email_activity_logger
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class EmailWorkflowService:
|
||||||
|
"""Orchestrates workflow execution for classified emails"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.enabled = settings.EMAIL_WORKFLOWS_ENABLED if hasattr(settings, 'EMAIL_WORKFLOWS_ENABLED') else True
|
||||||
|
|
||||||
|
async def execute_workflows(self, email_data: Dict) -> Dict:
|
||||||
|
"""
|
||||||
|
Execute all matching workflows for an email
|
||||||
|
|
||||||
|
Args:
|
||||||
|
email_data: Email dict with classification, confidence_score, id, etc.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with execution results
|
||||||
|
"""
|
||||||
|
if not self.enabled:
|
||||||
|
logger.info("⏭️ Workflows disabled")
|
||||||
|
return {'status': 'disabled', 'workflows_executed': 0}
|
||||||
|
|
||||||
|
email_id = email_data.get('id')
|
||||||
|
classification = email_data.get('classification')
|
||||||
|
confidence = email_data.get('confidence_score', 0.0)
|
||||||
|
|
||||||
|
if not email_id or not classification:
|
||||||
|
logger.warning(f"⚠️ Cannot execute workflows: missing email_id or classification")
|
||||||
|
return {'status': 'skipped', 'reason': 'missing_data'}
|
||||||
|
|
||||||
|
logger.info(f"🔄 Finding workflows for classification: {classification} (confidence: {confidence})")
|
||||||
|
|
||||||
|
# Find matching workflows
|
||||||
|
workflows = await self._find_matching_workflows(email_data)
|
||||||
|
|
||||||
|
if not workflows:
|
||||||
|
logger.info(f"✅ No workflows match classification: {classification}")
|
||||||
|
return {'status': 'no_match', 'workflows_executed': 0}
|
||||||
|
|
||||||
|
logger.info(f"📋 Found {len(workflows)} matching workflow(s)")
|
||||||
|
|
||||||
|
results = {
|
||||||
|
'status': 'executed',
|
||||||
|
'workflows_executed': 0,
|
||||||
|
'workflows_succeeded': 0,
|
||||||
|
'workflows_failed': 0,
|
||||||
|
'details': []
|
||||||
|
}
|
||||||
|
|
||||||
|
# Execute workflows in priority order
|
||||||
|
for workflow in workflows:
|
||||||
|
result = await self._execute_workflow(workflow, email_data)
|
||||||
|
results['details'].append(result)
|
||||||
|
results['workflows_executed'] += 1
|
||||||
|
|
||||||
|
if result['status'] == 'completed':
|
||||||
|
results['workflows_succeeded'] += 1
|
||||||
|
else:
|
||||||
|
results['workflows_failed'] += 1
|
||||||
|
|
||||||
|
# Stop if workflow has stop_on_match=true
|
||||||
|
if workflow.get('stop_on_match') and result['status'] == 'completed':
|
||||||
|
logger.info(f"🛑 Stopping workflow chain (stop_on_match=true)")
|
||||||
|
break
|
||||||
|
|
||||||
|
logger.info(f"✅ Workflow execution complete: {results['workflows_succeeded']}/{results['workflows_executed']} succeeded")
|
||||||
|
return results
|
||||||
|
|
||||||
|
async def _find_matching_workflows(self, email_data: Dict) -> List[Dict]:
|
||||||
|
"""Find all workflows that match this email"""
|
||||||
|
classification = email_data.get('classification')
|
||||||
|
confidence = email_data.get('confidence_score', 0.0)
|
||||||
|
sender = email_data.get('sender_email', '')
|
||||||
|
subject = email_data.get('subject', '')
|
||||||
|
|
||||||
|
query = """
|
||||||
|
SELECT id, name, classification_trigger, sender_pattern, subject_pattern,
|
||||||
|
confidence_threshold, workflow_steps, priority, stop_on_match
|
||||||
|
FROM email_workflows
|
||||||
|
WHERE enabled = true
|
||||||
|
AND classification_trigger = %s
|
||||||
|
AND confidence_threshold <= %s
|
||||||
|
ORDER BY priority ASC
|
||||||
|
"""
|
||||||
|
|
||||||
|
workflows = execute_query(query, (classification, confidence))
|
||||||
|
|
||||||
|
# Filter by additional patterns
|
||||||
|
matching = []
|
||||||
|
for wf in workflows:
|
||||||
|
# Check sender pattern
|
||||||
|
if wf.get('sender_pattern'):
|
||||||
|
pattern = wf['sender_pattern']
|
||||||
|
if not re.search(pattern, sender, re.IGNORECASE):
|
||||||
|
logger.debug(f"⏭️ Workflow '{wf['name']}' skipped: sender doesn't match pattern")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Check subject pattern
|
||||||
|
if wf.get('subject_pattern'):
|
||||||
|
pattern = wf['subject_pattern']
|
||||||
|
if not re.search(pattern, subject, re.IGNORECASE):
|
||||||
|
logger.debug(f"⏭️ Workflow '{wf['name']}' skipped: subject doesn't match pattern")
|
||||||
|
continue
|
||||||
|
|
||||||
|
matching.append(wf)
|
||||||
|
|
||||||
|
return matching
|
||||||
|
|
||||||
|
async def _execute_workflow(self, workflow: Dict, email_data: Dict) -> Dict:
|
||||||
|
"""Execute a single workflow"""
|
||||||
|
workflow_id = workflow['id']
|
||||||
|
workflow_name = workflow['name']
|
||||||
|
email_id = email_data['id']
|
||||||
|
|
||||||
|
logger.info(f"🚀 Executing workflow: {workflow_name} (ID: {workflow_id})")
|
||||||
|
|
||||||
|
# Create execution record
|
||||||
|
execution_id = execute_insert(
|
||||||
|
"""INSERT INTO email_workflow_executions
|
||||||
|
(workflow_id, email_id, status, steps_total, result_json)
|
||||||
|
VALUES (%s, %s, 'running', %s, %s) RETURNING id""",
|
||||||
|
(workflow_id, email_id, len(workflow['workflow_steps']), json.dumps({}))
|
||||||
|
)
|
||||||
|
|
||||||
|
started_at = datetime.now()
|
||||||
|
steps = workflow['workflow_steps']
|
||||||
|
steps_completed = 0
|
||||||
|
step_results = []
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Execute each step
|
||||||
|
for idx, step in enumerate(steps):
|
||||||
|
action = step.get('action')
|
||||||
|
params = step.get('params', {})
|
||||||
|
|
||||||
|
logger.info(f" ➡️ Step {idx + 1}/{len(steps)}: {action}")
|
||||||
|
|
||||||
|
step_result = await self._execute_action(action, params, email_data)
|
||||||
|
step_results.append({
|
||||||
|
'step': idx + 1,
|
||||||
|
'action': action,
|
||||||
|
'status': step_result['status'],
|
||||||
|
'result': step_result.get('result'),
|
||||||
|
'error': step_result.get('error')
|
||||||
|
})
|
||||||
|
|
||||||
|
if step_result['status'] == 'failed':
|
||||||
|
logger.error(f" ❌ Step failed: {step_result.get('error')}")
|
||||||
|
# Continue to next step even on failure (configurable later)
|
||||||
|
else:
|
||||||
|
logger.info(f" ✅ Step completed successfully")
|
||||||
|
|
||||||
|
steps_completed += 1
|
||||||
|
|
||||||
|
# Mark execution as completed
|
||||||
|
completed_at = datetime.now()
|
||||||
|
execution_time_ms = int((completed_at - started_at).total_seconds() * 1000)
|
||||||
|
|
||||||
|
execute_update(
|
||||||
|
"""UPDATE email_workflow_executions
|
||||||
|
SET status = 'completed', steps_completed = %s,
|
||||||
|
result_json = %s, completed_at = CURRENT_TIMESTAMP,
|
||||||
|
execution_time_ms = %s
|
||||||
|
WHERE id = %s""",
|
||||||
|
(steps_completed, json.dumps(step_results), execution_time_ms, execution_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Update workflow statistics
|
||||||
|
execute_update(
|
||||||
|
"""UPDATE email_workflows
|
||||||
|
SET execution_count = execution_count + 1,
|
||||||
|
success_count = success_count + 1,
|
||||||
|
last_executed_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE id = %s""",
|
||||||
|
(workflow_id,)
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(f"✅ Workflow '{workflow_name}' completed ({execution_time_ms}ms)")
|
||||||
|
|
||||||
|
# Log: Workflow execution completed
|
||||||
|
await email_activity_logger.log_workflow_executed(
|
||||||
|
email_id=email_id,
|
||||||
|
workflow_id=workflow_id,
|
||||||
|
workflow_name=workflow_name,
|
||||||
|
status='completed',
|
||||||
|
steps_completed=steps_completed,
|
||||||
|
execution_time_ms=execution_time_ms
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'workflow_id': workflow_id,
|
||||||
|
'workflow_name': workflow_name,
|
||||||
|
'execution_id': execution_id,
|
||||||
|
'status': 'completed',
|
||||||
|
'steps_completed': steps_completed,
|
||||||
|
'steps_total': len(steps),
|
||||||
|
'execution_time_ms': execution_time_ms,
|
||||||
|
'step_results': step_results
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Workflow execution failed: {e}")
|
||||||
|
|
||||||
|
# Mark execution as failed
|
||||||
|
execute_update(
|
||||||
|
"""UPDATE email_workflow_executions
|
||||||
|
SET status = 'failed', steps_completed = %s,
|
||||||
|
error_message = %s, completed_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE id = %s""",
|
||||||
|
(steps_completed, str(e), execution_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Update workflow statistics
|
||||||
|
execute_update(
|
||||||
|
"""UPDATE email_workflows
|
||||||
|
SET execution_count = execution_count + 1,
|
||||||
|
failure_count = failure_count + 1,
|
||||||
|
last_executed_at = CURRENT_TIMESTAMP
|
||||||
|
WHERE id = %s""",
|
||||||
|
(workflow_id,)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Log: Workflow execution failed
|
||||||
|
await email_activity_logger.log_workflow_executed(
|
||||||
|
email_id=email_id,
|
||||||
|
workflow_id=workflow_id,
|
||||||
|
workflow_name=workflow_name,
|
||||||
|
status='failed',
|
||||||
|
steps_completed=steps_completed,
|
||||||
|
execution_time_ms=0
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'workflow_id': workflow_id,
|
||||||
|
'workflow_name': workflow_name,
|
||||||
|
'execution_id': execution_id,
|
||||||
|
'status': 'failed',
|
||||||
|
'steps_completed': steps_completed,
|
||||||
|
'steps_total': len(steps),
|
||||||
|
'error': str(e)
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _execute_action(self, action: str, params: Dict, email_data: Dict) -> Dict:
|
||||||
|
"""Execute a single workflow action"""
|
||||||
|
try:
|
||||||
|
# Dispatch to specific action handler
|
||||||
|
handler_map = {
|
||||||
|
'create_ticket': self._action_create_ticket,
|
||||||
|
'create_time_entry': self._action_create_time_entry,
|
||||||
|
'link_to_vendor': self._action_link_to_vendor,
|
||||||
|
'link_to_customer': self._action_link_to_customer,
|
||||||
|
'extract_invoice_data': self._action_extract_invoice_data,
|
||||||
|
'extract_tracking_number': self._action_extract_tracking_number,
|
||||||
|
'send_slack_notification': self._action_send_slack_notification,
|
||||||
|
'send_email_notification': self._action_send_email_notification,
|
||||||
|
'mark_as_processed': self._action_mark_as_processed,
|
||||||
|
'flag_for_review': self._action_flag_for_review,
|
||||||
|
}
|
||||||
|
|
||||||
|
handler = handler_map.get(action)
|
||||||
|
|
||||||
|
if not handler:
|
||||||
|
logger.warning(f"⚠️ Unknown action: {action}")
|
||||||
|
return {
|
||||||
|
'status': 'skipped',
|
||||||
|
'error': f'Unknown action: {action}'
|
||||||
|
}
|
||||||
|
|
||||||
|
result = await handler(params, email_data)
|
||||||
|
return {
|
||||||
|
'status': 'success',
|
||||||
|
'result': result
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Action '{action}' failed: {e}")
|
||||||
|
return {
|
||||||
|
'status': 'failed',
|
||||||
|
'error': str(e)
|
||||||
|
}
|
||||||
|
|
||||||
|
# Action Handlers
|
||||||
|
|
||||||
|
async def _action_create_ticket(self, params: Dict, email_data: Dict) -> Dict:
|
||||||
|
"""Create a ticket/case from email"""
|
||||||
|
module = params.get('module', 'support_cases')
|
||||||
|
priority = params.get('priority', 'normal')
|
||||||
|
|
||||||
|
# TODO: Integrate with actual case/ticket system
|
||||||
|
logger.info(f"🎫 Would create ticket in module '{module}' with priority '{priority}'")
|
||||||
|
|
||||||
|
return {
|
||||||
|
'action': 'create_ticket',
|
||||||
|
'module': module,
|
||||||
|
'priority': priority,
|
||||||
|
'note': 'Ticket creation not yet implemented'
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _action_create_time_entry(self, params: Dict, email_data: Dict) -> Dict:
|
||||||
|
"""Create time entry from email"""
|
||||||
|
logger.info(f"⏱️ Would create time entry")
|
||||||
|
|
||||||
|
# TODO: Integrate with time tracking system
|
||||||
|
return {
|
||||||
|
'action': 'create_time_entry',
|
||||||
|
'note': 'Time entry creation not yet implemented'
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _action_link_to_vendor(self, params: Dict, email_data: Dict) -> Dict:
|
||||||
|
"""Link email to vendor"""
|
||||||
|
match_by = params.get('match_by', 'email')
|
||||||
|
sender_email = email_data.get('sender_email')
|
||||||
|
|
||||||
|
if not sender_email:
|
||||||
|
return {'action': 'link_to_vendor', 'matched': False, 'reason': 'No sender email'}
|
||||||
|
|
||||||
|
# Find vendor by email
|
||||||
|
query = "SELECT id, name FROM vendors WHERE email = %s LIMIT 1"
|
||||||
|
result = execute_query(query, (sender_email,), fetchone=True)
|
||||||
|
|
||||||
|
if result:
|
||||||
|
vendor_id = result['id']
|
||||||
|
|
||||||
|
# Check if already linked to avoid duplicate updates
|
||||||
|
current_vendor = execute_query(
|
||||||
|
"SELECT supplier_id FROM email_messages WHERE id = %s",
|
||||||
|
(email_data['id'],), fetchone=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if current_vendor and current_vendor.get('supplier_id') == vendor_id:
|
||||||
|
logger.info(f"⏭️ Email already linked to vendor {vendor_id}, skipping duplicate update")
|
||||||
|
return {
|
||||||
|
'action': 'link_to_vendor',
|
||||||
|
'matched': True,
|
||||||
|
'vendor_id': vendor_id,
|
||||||
|
'vendor_name': result['name'],
|
||||||
|
'note': 'Already linked (skipped duplicate)'
|
||||||
|
}
|
||||||
|
|
||||||
|
# Update email with vendor link
|
||||||
|
execute_update(
|
||||||
|
"UPDATE email_messages SET supplier_id = %s WHERE id = %s",
|
||||||
|
(vendor_id, email_data['id'])
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(f"🔗 Linked email to vendor: {result['name']} (ID: {vendor_id})")
|
||||||
|
|
||||||
|
return {
|
||||||
|
'action': 'link_to_vendor',
|
||||||
|
'matched': True,
|
||||||
|
'vendor_id': vendor_id,
|
||||||
|
'vendor_name': result['name']
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
logger.info(f"⚠️ No vendor found for email: {sender_email}")
|
||||||
|
return {'action': 'link_to_vendor', 'matched': False, 'reason': 'Vendor not found'}
|
||||||
|
|
||||||
|
async def _action_link_to_customer(self, params: Dict, email_data: Dict) -> Dict:
|
||||||
|
"""Link email to customer"""
|
||||||
|
logger.info(f"🔗 Would link to customer")
|
||||||
|
|
||||||
|
# TODO: Implement customer matching logic
|
||||||
|
return {
|
||||||
|
'action': 'link_to_customer',
|
||||||
|
'note': 'Customer linking not yet implemented'
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _action_extract_invoice_data(self, params: Dict, email_data: Dict) -> Dict:
|
||||||
|
"""Save email PDF attachment to incoming_files for processing"""
|
||||||
|
logger.info(f"📄 Saving invoice PDF from email to incoming files")
|
||||||
|
|
||||||
|
email_id = email_data.get('id')
|
||||||
|
sender_email = email_data.get('sender_email', '')
|
||||||
|
vendor_id = email_data.get('supplier_id')
|
||||||
|
|
||||||
|
# Get PDF attachments from email
|
||||||
|
attachments = execute_query(
|
||||||
|
"""SELECT filename, file_path, size_bytes, content_type
|
||||||
|
FROM email_attachments
|
||||||
|
WHERE email_id = %s AND content_type = 'application/pdf'""",
|
||||||
|
(email_id,)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not attachments:
|
||||||
|
attachments = []
|
||||||
|
elif not isinstance(attachments, list):
|
||||||
|
attachments = [attachments]
|
||||||
|
|
||||||
|
if not attachments:
|
||||||
|
logger.warning(f"⚠️ No PDF attachments found for email {email_id}")
|
||||||
|
return {
|
||||||
|
'action': 'extract_invoice_data',
|
||||||
|
'success': False,
|
||||||
|
'note': 'No PDF attachment found in email'
|
||||||
|
}
|
||||||
|
|
||||||
|
uploaded_files = []
|
||||||
|
|
||||||
|
for attachment in attachments:
|
||||||
|
try:
|
||||||
|
attachment_path = attachment['file_path']
|
||||||
|
if not attachment_path:
|
||||||
|
logger.warning(f"⚠️ No file path for attachment {attachment['filename']}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Handle both absolute and relative paths
|
||||||
|
file_path = Path(attachment_path)
|
||||||
|
if not file_path.is_absolute():
|
||||||
|
# Try common base directories
|
||||||
|
for base in [Path.cwd(), Path('/app'), Path('.')]:
|
||||||
|
test_path = base / attachment_path
|
||||||
|
if test_path.exists():
|
||||||
|
file_path = test_path
|
||||||
|
break
|
||||||
|
|
||||||
|
if not file_path.exists():
|
||||||
|
error_msg = f"Attachment file not found: {attachment_path}"
|
||||||
|
logger.error(f"❌ {error_msg}")
|
||||||
|
raise FileNotFoundError(error_msg)
|
||||||
|
|
||||||
|
# Old code continues here but never reached if file missing
|
||||||
|
if False and not file_path.exists():
|
||||||
|
logger.warning(f"⚠️ Attachment file not found: {attachment_path}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Calculate checksum
|
||||||
|
with open(file_path, 'rb') as f:
|
||||||
|
file_content = f.read()
|
||||||
|
checksum = hashlib.sha256(file_content).hexdigest()
|
||||||
|
|
||||||
|
# Check if file already exists
|
||||||
|
existing = execute_query(
|
||||||
|
"SELECT file_id FROM incoming_files WHERE checksum = %s",
|
||||||
|
(checksum,),
|
||||||
|
fetchone=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if existing:
|
||||||
|
logger.info(f"⚠️ File already exists: {attachment['filename']}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Create uploads directory if it doesn't exist
|
||||||
|
upload_dir = Path("uploads")
|
||||||
|
upload_dir.mkdir(exist_ok=True)
|
||||||
|
|
||||||
|
# Copy file to uploads directory
|
||||||
|
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||||
|
safe_filename = f"{timestamp}_{attachment['filename']}"
|
||||||
|
destination = upload_dir / safe_filename
|
||||||
|
|
||||||
|
shutil.copy2(file_path, destination)
|
||||||
|
|
||||||
|
# Insert into incoming_files
|
||||||
|
file_id = execute_insert(
|
||||||
|
"""INSERT INTO incoming_files
|
||||||
|
(filename, original_filename, file_path, file_size, mime_type, checksum,
|
||||||
|
status, detected_vendor_id, uploaded_at)
|
||||||
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP)
|
||||||
|
RETURNING file_id""",
|
||||||
|
(
|
||||||
|
safe_filename,
|
||||||
|
attachment['filename'],
|
||||||
|
str(destination),
|
||||||
|
attachment['size_bytes'],
|
||||||
|
'application/pdf',
|
||||||
|
checksum,
|
||||||
|
'pending', # Will appear in "Mangler Behandling"
|
||||||
|
vendor_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
uploaded_files.append({
|
||||||
|
'file_id': file_id,
|
||||||
|
'filename': attachment['filename']
|
||||||
|
})
|
||||||
|
|
||||||
|
logger.info(f"✅ Saved PDF to incoming_files: {attachment['filename']} (file_id: {file_id})")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Failed to save attachment: {e}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
if uploaded_files:
|
||||||
|
return {
|
||||||
|
'action': 'extract_invoice_data',
|
||||||
|
'success': True,
|
||||||
|
'files_uploaded': len(uploaded_files),
|
||||||
|
'file_ids': [f['file_id'] for f in uploaded_files],
|
||||||
|
'note': f"{len(uploaded_files)} PDF(er) gemt i 'Mangler Behandling'"
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
return {
|
||||||
|
'action': 'extract_invoice_data',
|
||||||
|
'success': False,
|
||||||
|
'note': 'No files could be uploaded'
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _action_extract_tracking_number(self, params: Dict, email_data: Dict) -> Dict:
|
||||||
|
"""Extract tracking number from email"""
|
||||||
|
body = email_data.get('body_text', '')
|
||||||
|
subject = email_data.get('subject', '')
|
||||||
|
text = f"{subject} {body}"
|
||||||
|
|
||||||
|
# Simple regex patterns for common carriers
|
||||||
|
patterns = {
|
||||||
|
'postnord': r'\b[0-9]{18}\b',
|
||||||
|
'gls': r'\b[0-9]{11}\b',
|
||||||
|
'dao': r'\b[0-9]{14}\b'
|
||||||
|
}
|
||||||
|
|
||||||
|
tracking_numbers = []
|
||||||
|
|
||||||
|
for carrier, pattern in patterns.items():
|
||||||
|
matches = re.findall(pattern, text)
|
||||||
|
if matches:
|
||||||
|
tracking_numbers.extend([{'carrier': carrier, 'number': m} for m in matches])
|
||||||
|
|
||||||
|
if tracking_numbers:
|
||||||
|
logger.info(f"📦 Extracted {len(tracking_numbers)} tracking number(s)")
|
||||||
|
|
||||||
|
# Update email with tracking number
|
||||||
|
if tracking_numbers:
|
||||||
|
first_number = tracking_numbers[0]['number']
|
||||||
|
execute_update(
|
||||||
|
"UPDATE email_messages SET extracted_tracking_number = %s WHERE id = %s",
|
||||||
|
(first_number, email_data['id'])
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'action': 'extract_tracking_number',
|
||||||
|
'tracking_numbers': tracking_numbers
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _action_send_slack_notification(self, params: Dict, email_data: Dict) -> Dict:
|
||||||
|
"""Send Slack notification"""
|
||||||
|
channel = params.get('channel', '#general')
|
||||||
|
template = params.get('template', 'New email: {{subject}}')
|
||||||
|
|
||||||
|
logger.info(f"💬 Would send Slack notification to {channel}")
|
||||||
|
|
||||||
|
# TODO: Integrate with Slack API
|
||||||
|
return {
|
||||||
|
'action': 'send_slack_notification',
|
||||||
|
'channel': channel,
|
||||||
|
'note': 'Slack integration not yet implemented'
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _action_send_email_notification(self, params: Dict, email_data: Dict) -> Dict:
|
||||||
|
"""Send email notification"""
|
||||||
|
recipients = params.get('recipients', [])
|
||||||
|
|
||||||
|
logger.info(f"📧 Would send email notification to {len(recipients)} recipient(s)")
|
||||||
|
|
||||||
|
# TODO: Integrate with email sending service
|
||||||
|
return {
|
||||||
|
'action': 'send_email_notification',
|
||||||
|
'recipients': recipients,
|
||||||
|
'note': 'Email notification not yet implemented'
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _action_mark_as_processed(self, params: Dict, email_data: Dict) -> Dict:
|
||||||
|
"""Mark email as processed"""
|
||||||
|
status = params.get('status', 'processed')
|
||||||
|
|
||||||
|
execute_update(
|
||||||
|
"""UPDATE email_messages
|
||||||
|
SET status = %s, processed_at = CURRENT_TIMESTAMP, auto_processed = true
|
||||||
|
WHERE id = %s""",
|
||||||
|
(status, email_data['id'])
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(f"✅ Marked email as: {status}")
|
||||||
|
|
||||||
|
return {
|
||||||
|
'action': 'mark_as_processed',
|
||||||
|
'status': status
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _action_flag_for_review(self, params: Dict, email_data: Dict) -> Dict:
|
||||||
|
"""Flag email for manual review"""
|
||||||
|
reason = params.get('reason', 'Flagged by workflow')
|
||||||
|
|
||||||
|
execute_update(
|
||||||
|
"""UPDATE email_messages
|
||||||
|
SET status = 'flagged', approval_status = 'pending_review'
|
||||||
|
WHERE id = %s""",
|
||||||
|
(email_data['id'],)
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(f"🚩 Flagged email for review: {reason}")
|
||||||
|
|
||||||
|
return {
|
||||||
|
'action': 'flag_for_review',
|
||||||
|
'reason': reason
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Global instance
|
||||||
|
email_workflow_service = EmailWorkflowService()
|
||||||
@ -118,8 +118,8 @@ class Invoice2DataService:
|
|||||||
decimal_sep = options.get('decimal_separator', ',')
|
decimal_sep = options.get('decimal_separator', ',')
|
||||||
thousands_sep = options.get('thousands_separator', '.')
|
thousands_sep = options.get('thousands_separator', '.')
|
||||||
|
|
||||||
# Remove all spaces first
|
# Remove all whitespace first (pdf extraction may split numbers across lines)
|
||||||
value = value.replace(' ', '')
|
value = re.sub(r'\s+', '', value)
|
||||||
|
|
||||||
# If both separators are present, we can determine the format
|
# If both separators are present, we can determine the format
|
||||||
# Danish: 148.587,98 (thousands=., decimal=,)
|
# Danish: 148.587,98 (thousands=., decimal=,)
|
||||||
@ -170,6 +170,25 @@ class Invoice2DataService:
|
|||||||
if lines_config:
|
if lines_config:
|
||||||
extracted['lines'] = self._extract_lines(text, lines_config, options)
|
extracted['lines'] = self._extract_lines(text, lines_config, options)
|
||||||
|
|
||||||
|
# Calculate due_date if field has '+Xd' value
|
||||||
|
if 'due_date' in fields:
|
||||||
|
due_config = fields['due_date']
|
||||||
|
if due_config.get('parser') == 'static':
|
||||||
|
value = due_config.get('value', '')
|
||||||
|
if value.startswith('+') and value.endswith('d') and extracted.get('invoice_date'):
|
||||||
|
try:
|
||||||
|
from datetime import timedelta
|
||||||
|
days = int(value[1:-1])
|
||||||
|
inv_date = datetime.strptime(extracted['invoice_date'], '%Y-%m-%d')
|
||||||
|
due_date = inv_date + timedelta(days=days)
|
||||||
|
extracted['due_date'] = due_date.strftime('%Y-%m-%d')
|
||||||
|
logger.info(f"✅ Calculated due_date: {extracted['due_date']} ({days} days from invoice_date)")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to calculate due_date: {e}")
|
||||||
|
|
||||||
|
# Validate amounts (sum of lines vs total, VAT calculation)
|
||||||
|
self._validate_amounts(extracted)
|
||||||
|
|
||||||
return extracted
|
return extracted
|
||||||
|
|
||||||
def _extract_lines(self, text: str, lines_configs: List[Dict], options: Dict) -> List[Dict]:
|
def _extract_lines(self, text: str, lines_configs: List[Dict], options: Dict) -> List[Dict]:
|
||||||
@ -236,7 +255,7 @@ class Invoice2DataService:
|
|||||||
if field_type == 'float':
|
if field_type == 'float':
|
||||||
thousands_sep = options.get('thousands_separator', ',')
|
thousands_sep = options.get('thousands_separator', ',')
|
||||||
decimal_sep = options.get('decimal_separator', '.')
|
decimal_sep = options.get('decimal_separator', '.')
|
||||||
value = value.replace(' ', '')
|
value = re.sub(r'\s+', '', value)
|
||||||
|
|
||||||
if thousands_sep in value and decimal_sep in value:
|
if thousands_sep in value and decimal_sep in value:
|
||||||
value = value.replace(thousands_sep, '').replace(decimal_sep, '.')
|
value = value.replace(thousands_sep, '').replace(decimal_sep, '.')
|
||||||
@ -282,6 +301,61 @@ class Invoice2DataService:
|
|||||||
logger.debug(f" ✗ Failed to extract context field {ctx_field_name}: {e}")
|
logger.debug(f" ✗ Failed to extract context field {ctx_field_name}: {e}")
|
||||||
break # Stop after first match for this pattern
|
break # Stop after first match for this pattern
|
||||||
|
|
||||||
|
# If header is line-wrapped (e.g. "Husleje" on one line and "(inkl...)" on next),
|
||||||
|
# stitch them together so description becomes "Husleje (inkl...) ...".
|
||||||
|
try:
|
||||||
|
description = line_data.get('description')
|
||||||
|
if isinstance(description, str) and description.lstrip().startswith('('):
|
||||||
|
prefix = None
|
||||||
|
for candidate in reversed(context_lines):
|
||||||
|
candidate_stripped = candidate.strip()
|
||||||
|
if not candidate_stripped:
|
||||||
|
continue
|
||||||
|
if candidate_stripped.startswith('('):
|
||||||
|
continue
|
||||||
|
if re.match(r'^\d', candidate_stripped):
|
||||||
|
continue
|
||||||
|
if candidate_stripped.lower().startswith('periode:'):
|
||||||
|
continue
|
||||||
|
if ' Kr ' in f" {candidate_stripped} ":
|
||||||
|
continue
|
||||||
|
# Avoid picking calculation/detail lines
|
||||||
|
if any(token in candidate_stripped for token in ('*', '=', 'm2', 'm²')):
|
||||||
|
continue
|
||||||
|
# Prefer short header-like prefixes (e.g. "Husleje")
|
||||||
|
if len(candidate_stripped) <= 40:
|
||||||
|
prefix = candidate_stripped
|
||||||
|
break
|
||||||
|
|
||||||
|
if prefix and not description.strip().lower().startswith(prefix.lower()):
|
||||||
|
line_data['description'] = f"{prefix} {description.strip()}".strip()
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(f" ✗ Failed to stitch wrapped description: {e}")
|
||||||
|
|
||||||
|
# Safety: skip subtotal/totals artifacts that may match loosely
|
||||||
|
# e.g. a line like "Kr 3 9.048,75" (no letters) should not become a line item.
|
||||||
|
try:
|
||||||
|
details = line_data.get('_line_details')
|
||||||
|
description = line_data.get('description')
|
||||||
|
|
||||||
|
if isinstance(details, str) and details.strip() == '':
|
||||||
|
continue
|
||||||
|
|
||||||
|
text_to_check = None
|
||||||
|
if isinstance(description, str) and description.strip() and description.strip() != '-':
|
||||||
|
text_to_check = description
|
||||||
|
elif isinstance(details, str) and details.strip():
|
||||||
|
text_to_check = details
|
||||||
|
|
||||||
|
if isinstance(text_to_check, str):
|
||||||
|
if not re.search(r'[A-Za-zÆØÅæøå]', text_to_check):
|
||||||
|
continue
|
||||||
|
lowered = text_to_check.lower()
|
||||||
|
if lowered.startswith('kr') or ' moms' in lowered or lowered.startswith('total') or lowered.startswith('netto'):
|
||||||
|
continue
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
if line_data:
|
if line_data:
|
||||||
all_lines.append(line_data)
|
all_lines.append(line_data)
|
||||||
|
|
||||||
@ -292,6 +366,55 @@ class Invoice2DataService:
|
|||||||
|
|
||||||
return all_lines
|
return all_lines
|
||||||
|
|
||||||
|
def _validate_amounts(self, extracted: Dict) -> None:
|
||||||
|
"""Validate that line totals sum to subtotal/total, and VAT calculation is correct"""
|
||||||
|
try:
|
||||||
|
lines = extracted.get('lines', [])
|
||||||
|
total_amount = extracted.get('total_amount')
|
||||||
|
vat_amount = extracted.get('vat_amount')
|
||||||
|
|
||||||
|
if not lines or total_amount is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Calculate sum of line_total values
|
||||||
|
line_sum = 0.0
|
||||||
|
for line in lines:
|
||||||
|
line_total = line.get('line_total')
|
||||||
|
if line_total is not None:
|
||||||
|
if isinstance(line_total, str):
|
||||||
|
# Parse Danish format: "25.000,00" or "1 .530,00"
|
||||||
|
cleaned = line_total.replace(' ', '').replace('.', '').replace(',', '.')
|
||||||
|
try:
|
||||||
|
line_sum += float(cleaned)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
elif isinstance(line_total, (int, float)):
|
||||||
|
line_sum += float(line_total)
|
||||||
|
|
||||||
|
# If we have VAT amount, subtract it from total to get subtotal
|
||||||
|
subtotal = total_amount
|
||||||
|
if vat_amount is not None:
|
||||||
|
subtotal = total_amount - vat_amount
|
||||||
|
|
||||||
|
# Check if line sum matches subtotal (allow 1 DKK difference for rounding)
|
||||||
|
if abs(line_sum - subtotal) > 1.0:
|
||||||
|
logger.warning(f"⚠️ Amount validation: Line sum {line_sum:.2f} != subtotal {subtotal:.2f} (diff: {abs(line_sum - subtotal):.2f})")
|
||||||
|
extracted['_validation_warning'] = f"Varelinjer sum ({line_sum:.2f}) passer ikke med subtotal ({subtotal:.2f})"
|
||||||
|
else:
|
||||||
|
logger.info(f"✅ Amount validation: Line sum matches subtotal ({line_sum:.2f})")
|
||||||
|
|
||||||
|
# Check VAT calculation (25%)
|
||||||
|
if vat_amount is not None:
|
||||||
|
expected_vat = subtotal * 0.25
|
||||||
|
if abs(vat_amount - expected_vat) > 1.0:
|
||||||
|
logger.warning(f"⚠️ VAT validation: VAT {vat_amount:.2f} != 25% of {subtotal:.2f} ({expected_vat:.2f})")
|
||||||
|
extracted['_vat_warning'] = f"Moms ({vat_amount:.2f}) passer ikke med 25% af subtotal ({expected_vat:.2f})"
|
||||||
|
else:
|
||||||
|
logger.info(f"✅ VAT validation: 25% VAT calculation correct ({vat_amount:.2f})")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"⚠️ Amount validation failed: {e}")
|
||||||
|
|
||||||
def extract(self, text: str, template_name: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
def extract(self, text: str, template_name: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
||||||
"""
|
"""
|
||||||
Extract invoice data from text
|
Extract invoice data from text
|
||||||
|
|||||||
@ -6,6 +6,7 @@ Inspired by OmniSync's invoice template system
|
|||||||
|
|
||||||
import re
|
import re
|
||||||
import logging
|
import logging
|
||||||
|
import json
|
||||||
from typing import Dict, List, Optional, Tuple
|
from typing import Dict, List, Optional, Tuple
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@ -102,10 +103,22 @@ class TemplateService:
|
|||||||
score = 0.0
|
score = 0.0
|
||||||
patterns = template.get('detection_patterns', [])
|
patterns = template.get('detection_patterns', [])
|
||||||
|
|
||||||
|
# DB may return JSON/JSONB as string depending on driver/config
|
||||||
|
if isinstance(patterns, str):
|
||||||
|
try:
|
||||||
|
patterns = json.loads(patterns)
|
||||||
|
except Exception:
|
||||||
|
patterns = []
|
||||||
|
|
||||||
|
if isinstance(patterns, dict):
|
||||||
|
patterns = [patterns]
|
||||||
|
|
||||||
if not patterns:
|
if not patterns:
|
||||||
return 0.0
|
return 0.0
|
||||||
|
|
||||||
for pattern_obj in patterns:
|
for pattern_obj in patterns:
|
||||||
|
if not isinstance(pattern_obj, dict):
|
||||||
|
continue
|
||||||
pattern_type = pattern_obj.get('type')
|
pattern_type = pattern_obj.get('type')
|
||||||
weight = pattern_obj.get('weight', 0.5)
|
weight = pattern_obj.get('weight', 0.5)
|
||||||
|
|
||||||
|
|||||||
@ -294,6 +294,7 @@
|
|||||||
<ul class="dropdown-menu dropdown-menu-end mt-2">
|
<ul class="dropdown-menu dropdown-menu-end mt-2">
|
||||||
<li><a class="dropdown-item py-2" href="#">Profil</a></li>
|
<li><a class="dropdown-item py-2" href="#">Profil</a></li>
|
||||||
<li><a class="dropdown-item py-2" href="/settings"><i class="bi bi-gear me-2"></i>Indstillinger</a></li>
|
<li><a class="dropdown-item py-2" href="/settings"><i class="bi bi-gear me-2"></i>Indstillinger</a></li>
|
||||||
|
<li><a class="dropdown-item py-2" href="/backups"><i class="bi bi-hdd-stack me-2"></i>Backup System</a></li>
|
||||||
<li><a class="dropdown-item py-2" href="/devportal"><i class="bi bi-code-square me-2"></i>DEV Portal</a></li>
|
<li><a class="dropdown-item py-2" href="/devportal"><i class="bi bi-code-square me-2"></i>DEV Portal</a></li>
|
||||||
<li><hr class="dropdown-divider"></li>
|
<li><hr class="dropdown-divider"></li>
|
||||||
<li><a class="dropdown-item py-2 text-danger" href="#">Log ud</a></li>
|
<li><a class="dropdown-item py-2 text-danger" href="#">Log ud</a></li>
|
||||||
@ -1024,6 +1025,82 @@
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
<!-- Maintenance Mode Overlay -->
|
||||||
|
<div id="maintenance-overlay" style="display: none; position: fixed; top: 0; left: 0; width: 100%; height: 100%; background: rgba(0,0,0,0.85); z-index: 9999; backdrop-filter: blur(5px);">
|
||||||
|
<div style="position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%); text-align: center; color: white; max-width: 500px; padding: 2rem;">
|
||||||
|
<div style="font-size: 4rem; margin-bottom: 1rem;">🔧</div>
|
||||||
|
<h2 style="font-weight: 700; margin-bottom: 1rem;">System under vedligeholdelse</h2>
|
||||||
|
<p id="maintenance-message" style="font-size: 1.1rem; margin-bottom: 1.5rem; opacity: 0.9;">Systemet er midlertidigt utilgængeligt på grund af vedligeholdelse.</p>
|
||||||
|
<div id="maintenance-eta" style="font-size: 1rem; margin-bottom: 2rem; opacity: 0.8;"></div>
|
||||||
|
<div class="spinner-border text-light" role="status" style="width: 3rem; height: 3rem;">
|
||||||
|
<span class="visually-hidden">Loading...</span>
|
||||||
|
</div>
|
||||||
|
<p style="margin-top: 1.5rem; font-size: 0.9rem; opacity: 0.7;">
|
||||||
|
Siden opdateres automatisk når systemet er klar igen.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
// Check maintenance mode status
|
||||||
|
let maintenanceCheckInterval = null;
|
||||||
|
|
||||||
|
function checkMaintenanceMode() {
|
||||||
|
fetch('/api/v1/backups/maintenance')
|
||||||
|
.then(response => response.json())
|
||||||
|
.then(data => {
|
||||||
|
const overlay = document.getElementById('maintenance-overlay');
|
||||||
|
const messageEl = document.getElementById('maintenance-message');
|
||||||
|
const etaEl = document.getElementById('maintenance-eta');
|
||||||
|
|
||||||
|
if (data.maintenance_mode) {
|
||||||
|
// Show overlay
|
||||||
|
overlay.style.display = 'block';
|
||||||
|
|
||||||
|
// Update message
|
||||||
|
if (data.maintenance_message) {
|
||||||
|
messageEl.textContent = data.maintenance_message;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update ETA
|
||||||
|
if (data.maintenance_eta_minutes) {
|
||||||
|
etaEl.textContent = `Estimeret tid: ${data.maintenance_eta_minutes} minutter`;
|
||||||
|
} else {
|
||||||
|
etaEl.textContent = '';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start polling every 5 seconds if not already polling
|
||||||
|
if (!maintenanceCheckInterval) {
|
||||||
|
maintenanceCheckInterval = setInterval(checkMaintenanceMode, 5000);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Hide overlay
|
||||||
|
overlay.style.display = 'none';
|
||||||
|
|
||||||
|
// Stop polling if maintenance is over
|
||||||
|
if (maintenanceCheckInterval) {
|
||||||
|
clearInterval(maintenanceCheckInterval);
|
||||||
|
maintenanceCheckInterval = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
console.error('Maintenance check error:', error);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check on page load
|
||||||
|
checkMaintenanceMode();
|
||||||
|
|
||||||
|
// Check periodically (every 30 seconds when not in maintenance)
|
||||||
|
setInterval(() => {
|
||||||
|
if (!maintenanceCheckInterval) {
|
||||||
|
checkMaintenanceMode();
|
||||||
|
}
|
||||||
|
}, 30000);
|
||||||
|
</script>
|
||||||
|
|
||||||
{% block extra_js %}{% endblock %}
|
{% block extra_js %}{% endblock %}
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
@ -102,6 +102,43 @@ class EconomicExportService:
|
|||||||
logger.error(f"❌ e-conomic connection error: {e}")
|
logger.error(f"❌ e-conomic connection error: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
async def check_draft_exists(self, draft_id: int) -> bool:
|
||||||
|
"""
|
||||||
|
Tjek om en draft order stadig eksisterer i e-conomic.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
draft_id: e-conomic draft order nummer
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True hvis draft findes, False hvis slettet eller ikke fundet
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
logger.info(f"🔍 Checking if draft {draft_id} exists in e-conomic...")
|
||||||
|
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
async with session.get(
|
||||||
|
f"{self.api_url}/orders/drafts/{draft_id}",
|
||||||
|
headers=self._get_headers(),
|
||||||
|
timeout=aiohttp.ClientTimeout(total=10)
|
||||||
|
) as response:
|
||||||
|
if response.status == 200:
|
||||||
|
logger.info(f"✅ Draft {draft_id} exists in e-conomic")
|
||||||
|
return True
|
||||||
|
elif response.status == 404:
|
||||||
|
logger.info(f"✅ Draft {draft_id} NOT found in e-conomic (deleted)")
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
error_text = await response.text()
|
||||||
|
logger.error(f"❌ e-conomic check failed ({response.status}): {error_text}")
|
||||||
|
raise Exception(f"e-conomic API error: {response.status}")
|
||||||
|
|
||||||
|
except aiohttp.ClientError as e:
|
||||||
|
logger.error(f"❌ e-conomic connection error: {e}")
|
||||||
|
raise Exception(f"Kunne ikke forbinde til e-conomic: {str(e)}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Draft check error: {e}")
|
||||||
|
raise
|
||||||
|
|
||||||
async def export_order(
|
async def export_order(
|
||||||
self,
|
self,
|
||||||
request: TModuleEconomicExportRequest,
|
request: TModuleEconomicExportRequest,
|
||||||
|
|||||||
@ -399,6 +399,106 @@ async def cancel_order(
|
|||||||
raise HTTPException(status_code=500, detail=str(e))
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/orders/{order_id}/unlock", tags=["Orders"])
|
||||||
|
async def unlock_order(
|
||||||
|
order_id: int,
|
||||||
|
admin_code: str,
|
||||||
|
user_id: Optional[int] = None
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
🔓 Lås en eksporteret ordre op for ændringer (ADMIN ONLY).
|
||||||
|
|
||||||
|
Kræver:
|
||||||
|
1. Korrekt admin unlock code (fra TIMETRACKING_ADMIN_UNLOCK_CODE)
|
||||||
|
2. Ordren skal være slettet fra e-conomic først
|
||||||
|
|
||||||
|
Query params:
|
||||||
|
- admin_code: Admin unlock kode
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
from app.core.config import settings
|
||||||
|
|
||||||
|
# Verify admin code
|
||||||
|
if not settings.TIMETRACKING_ADMIN_UNLOCK_CODE:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=500,
|
||||||
|
detail="Admin unlock code ikke konfigureret i systemet"
|
||||||
|
)
|
||||||
|
|
||||||
|
if admin_code != settings.TIMETRACKING_ADMIN_UNLOCK_CODE:
|
||||||
|
logger.warning(f"⚠️ Ugyldig unlock code forsøg for ordre {order_id}")
|
||||||
|
raise HTTPException(status_code=403, detail="Ugyldig admin kode")
|
||||||
|
|
||||||
|
# Get order
|
||||||
|
order = order_service.get_order_with_lines(order_id)
|
||||||
|
|
||||||
|
if order.status != 'exported':
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail="Kun eksporterede ordrer kan låses op"
|
||||||
|
)
|
||||||
|
|
||||||
|
if not order.economic_draft_id:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail="Ordre har ingen e-conomic ID"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Check if order still exists in e-conomic
|
||||||
|
try:
|
||||||
|
draft_exists = await economic_service.check_draft_exists(order.economic_draft_id)
|
||||||
|
|
||||||
|
if draft_exists:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"⚠️ Ordren findes stadig i e-conomic (Draft #{order.economic_draft_id}). Slet den i e-conomic først!"
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Kunne ikke tjekke e-conomic status: {e}")
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=500,
|
||||||
|
detail=f"Kunne ikke verificere e-conomic status: {str(e)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Unlock order - set status back to draft
|
||||||
|
update_query = """
|
||||||
|
UPDATE tmodule_orders
|
||||||
|
SET status = 'draft',
|
||||||
|
economic_draft_id = NULL,
|
||||||
|
exported_at = NULL
|
||||||
|
WHERE id = %s
|
||||||
|
RETURNING *
|
||||||
|
"""
|
||||||
|
|
||||||
|
result = execute_query(update_query, (order_id,), fetchone=True)
|
||||||
|
|
||||||
|
# Log unlock
|
||||||
|
audit.log_event(
|
||||||
|
event_type="order_unlocked",
|
||||||
|
entity_type="order",
|
||||||
|
entity_id=order_id,
|
||||||
|
user_id=user_id,
|
||||||
|
details={
|
||||||
|
"previous_economic_id": order.economic_draft_id,
|
||||||
|
"unlocked_by_admin": True
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(f"🔓 Order {order_id} unlocked by admin (user {user_id})")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"message": "Ordre låst op - kan nu redigeres eller slettes",
|
||||||
|
"order_id": order_id
|
||||||
|
}
|
||||||
|
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"❌ Unlock failed: {e}")
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# e-conomic EXPORT ENDPOINTS
|
# e-conomic EXPORT ENDPOINTS
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|||||||
@ -214,13 +214,13 @@
|
|||||||
const tbody = document.getElementById('orders-tbody');
|
const tbody = document.getElementById('orders-tbody');
|
||||||
tbody.innerHTML = orders.map(order => {
|
tbody.innerHTML = orders.map(order => {
|
||||||
const statusBadge = getStatusBadge(order);
|
const statusBadge = getStatusBadge(order);
|
||||||
const isPosted = order.status === 'posted';
|
const isLocked = order.status === 'exported';
|
||||||
const economicInfo = order.economic_order_number
|
const economicInfo = order.economic_draft_id
|
||||||
? `<br><small class="text-muted">e-conomic #${order.economic_order_number}</small>`
|
? `<br><small class="text-muted">e-conomic draft #${order.economic_draft_id}</small>`
|
||||||
: '';
|
: '';
|
||||||
|
|
||||||
return `
|
return `
|
||||||
<tr class="order-row ${isPosted ? 'table-success' : ''}" onclick="viewOrder(${order.id})">
|
<tr class="order-row ${isLocked ? 'table-warning' : ''}" onclick="viewOrder(${order.id})">
|
||||||
<td>
|
<td>
|
||||||
<strong>${order.order_number}</strong>
|
<strong>${order.order_number}</strong>
|
||||||
${economicInfo}
|
${economicInfo}
|
||||||
@ -240,9 +240,17 @@
|
|||||||
onclick="event.stopPropagation(); exportOrder(${order.id})">
|
onclick="event.stopPropagation(); exportOrder(${order.id})">
|
||||||
<i class="bi bi-cloud-upload"></i> Eksporter
|
<i class="bi bi-cloud-upload"></i> Eksporter
|
||||||
</button>
|
</button>
|
||||||
|
<button class="btn btn-sm btn-outline-danger"
|
||||||
|
onclick="event.stopPropagation(); cancelOrder(${order.id})">
|
||||||
|
<i class="bi bi-x-circle"></i> Annuller
|
||||||
|
</button>
|
||||||
` : ''}
|
` : ''}
|
||||||
${isPosted ? `
|
${order.status === 'exported' ? `
|
||||||
<span class="badge bg-success"><i class="bi bi-lock"></i> Låst</span>
|
<span class="badge bg-warning text-dark"><i class="bi bi-lock"></i> Låst</span>
|
||||||
|
<button class="btn btn-sm btn-outline-warning"
|
||||||
|
onclick="event.stopPropagation(); unlockOrder(${order.id})">
|
||||||
|
<i class="bi bi-unlock"></i> Lås op
|
||||||
|
</button>
|
||||||
` : ''}
|
` : ''}
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
@ -266,9 +274,8 @@
|
|||||||
function getStatusBadge(order) {
|
function getStatusBadge(order) {
|
||||||
const statusMap = {
|
const statusMap = {
|
||||||
'cancelled': '<span class="badge bg-danger">Annulleret</span>',
|
'cancelled': '<span class="badge bg-danger">Annulleret</span>',
|
||||||
'posted': '<span class="badge bg-success"><i class="bi bi-check-circle"></i> Bogført</span>',
|
'exported': '<span class="badge bg-warning text-dark"><i class="bi bi-cloud-check"></i> Eksporteret</span>',
|
||||||
'exported': '<span class="badge bg-info">Eksporteret</span>',
|
'draft': '<span class="badge bg-secondary">Kladde</span>'
|
||||||
'draft': '<span class="badge bg-warning">Kladde</span>'
|
|
||||||
};
|
};
|
||||||
|
|
||||||
return statusMap[order.status] || '<span class="badge bg-secondary">Ukendt</span>';
|
return statusMap[order.status] || '<span class="badge bg-secondary">Ukendt</span>';
|
||||||
@ -361,23 +368,20 @@
|
|||||||
|
|
||||||
// Update export button
|
// Update export button
|
||||||
const exportBtn = document.getElementById('export-order-btn');
|
const exportBtn = document.getElementById('export-order-btn');
|
||||||
if (order.status === 'posted') {
|
if (order.status === 'exported') {
|
||||||
exportBtn.disabled = true;
|
exportBtn.disabled = true;
|
||||||
exportBtn.innerHTML = '<i class="bi bi-lock"></i> Bogført (Låst)';
|
exportBtn.innerHTML = '<i class="bi bi-lock"></i> Eksporteret (Låst)';
|
||||||
exportBtn.classList.remove('btn-primary');
|
exportBtn.classList.remove('btn-primary');
|
||||||
exportBtn.classList.add('btn-secondary');
|
exportBtn.classList.add('btn-secondary');
|
||||||
} else if (order.economic_draft_id) {
|
} else if (order.status === 'draft') {
|
||||||
exportBtn.disabled = false;
|
|
||||||
exportBtn.innerHTML = '<i class="bi bi-arrow-repeat"></i> Re-eksporter (force)';
|
|
||||||
exportBtn.onclick = () => {
|
|
||||||
if (confirm('Re-eksporter ordre til e-conomic?\n\nDette vil overskrive den eksisterende draft order.')) {
|
|
||||||
exportOrderForce(currentOrderId);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
exportBtn.disabled = false;
|
exportBtn.disabled = false;
|
||||||
exportBtn.innerHTML = '<i class="bi bi-cloud-upload"></i> Eksporter til e-conomic';
|
exportBtn.innerHTML = '<i class="bi bi-cloud-upload"></i> Eksporter til e-conomic';
|
||||||
exportBtn.onclick = exportCurrentOrder;
|
exportBtn.onclick = exportCurrentOrder;
|
||||||
|
} else {
|
||||||
|
exportBtn.disabled = true;
|
||||||
|
exportBtn.innerHTML = `<i class="bi bi-x-circle"></i> ${order.status}`;
|
||||||
|
exportBtn.classList.remove('btn-primary');
|
||||||
|
exportBtn.classList.add('btn-secondary');
|
||||||
}
|
}
|
||||||
|
|
||||||
orderModal.show();
|
orderModal.show();
|
||||||
@ -436,43 +440,76 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Force re-export order
|
// Unlock exported order (ADMIN)
|
||||||
async function exportOrderForce(orderId) {
|
async function unlockOrder(orderId) {
|
||||||
|
const adminCode = prompt(
|
||||||
|
'🔐 ADMIN ADGANG PÅKRÆVET\n\n' +
|
||||||
|
'Før ordren kan låses op skal den være slettet fra e-conomic.\n\n' +
|
||||||
|
'Indtast admin unlock kode:'
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!adminCode) return;
|
||||||
|
|
||||||
|
if (!confirm(
|
||||||
|
'⚠️ ADVARSEL\n\n' +
|
||||||
|
'Er du SIKKER på at ordren er slettet fra e-conomic?\n\n' +
|
||||||
|
'Systemet vil tjekke om ordren stadig findes i e-conomic.\n\n' +
|
||||||
|
'Fortsæt?'
|
||||||
|
)) return;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetch(`/api/v1/timetracking/export`, {
|
const response = await fetch(`/api/v1/timetracking/orders/${orderId}/unlock?admin_code=${encodeURIComponent(adminCode)}`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json'
|
'Content-Type': 'application/json'
|
||||||
},
|
}
|
||||||
body: JSON.stringify({
|
|
||||||
order_id: orderId,
|
|
||||||
force: true
|
|
||||||
})
|
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const errorData = await response.json();
|
const errorData = await response.json();
|
||||||
throw new Error(errorData.detail || 'Export failed');
|
alert(`❌ Kunne ikke låse ordre op:\n\n${errorData.detail}`);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await response.json();
|
const result = await response.json();
|
||||||
|
alert(`✅ ${result.message}\n\nOrdren kan nu redigeres eller slettes.`);
|
||||||
if (result.dry_run) {
|
|
||||||
alert(`DRY-RUN MODE:\n\n${result.message}\n\n⚠️ Ingen ændringer er foretaget i e-conomic (DRY-RUN mode aktiveret).`);
|
|
||||||
} else if (result.success) {
|
|
||||||
alert(`✅ Ordre re-eksporteret til e-conomic!\n\n- Draft Order nr.: ${result.economic_draft_id}\n- e-conomic ordre nr.: ${result.economic_order_number}`);
|
|
||||||
loadOrders();
|
loadOrders();
|
||||||
if (orderModal._isShown) {
|
|
||||||
orderModal.hide();
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
throw new Error(result.message || 'Export failed');
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
alert('Fejl ved eksport: ' + error.message);
|
console.error('Error unlocking order:', error);
|
||||||
|
alert(`❌ Fejl ved unlock: ${error.message}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Cancel order
|
||||||
|
async function cancelOrder(orderId) {
|
||||||
|
if (!confirm('Annuller denne ordre?\n\nTidsregistreringerne sættes tilbage til "godkendt" status.')) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/v1/timetracking/orders/${orderId}/cancel`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json'
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorData = await response.json();
|
||||||
|
alert(`❌ Kunne ikke annullere ordre:\n\n${errorData.detail}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
alert('✅ Ordre annulleret\n\nTidsregistreringerne er sat tilbage til "godkendt" status.');
|
||||||
|
loadOrders();
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error cancelling order:', error);
|
||||||
|
alert(`❌ Fejl ved annullering: ${error.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
</script>
|
</script>
|
||||||
</div>
|
</div>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|||||||
232
docs/BACKUP_SYSTEM.md
Normal file
232
docs/BACKUP_SYSTEM.md
Normal file
@ -0,0 +1,232 @@
|
|||||||
|
# BMC Hub Backup System - Environment Configuration Guide
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
Add these lines to your `.env` file to enable the backup system:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# ===== BACKUP SYSTEM =====
|
||||||
|
|
||||||
|
# Safety switches (start with safe defaults)
|
||||||
|
BACKUP_ENABLED=true # Enable backup system (default: false)
|
||||||
|
BACKUP_DRY_RUN=false # Disable dry-run to actually perform backups (default: true)
|
||||||
|
BACKUP_READ_ONLY=false # Allow restore operations (default: true)
|
||||||
|
|
||||||
|
# Backup formats
|
||||||
|
DB_DAILY_FORMAT=dump # Compressed format for daily backups (default: dump)
|
||||||
|
DB_MONTHLY_FORMAT=sql # Plain SQL for monthly backups (default: sql)
|
||||||
|
|
||||||
|
# Backup scope
|
||||||
|
BACKUP_INCLUDE_UPLOADS=true # Include uploads/ directory (default: true)
|
||||||
|
BACKUP_INCLUDE_LOGS=true # Include logs/ directory (default: true)
|
||||||
|
BACKUP_INCLUDE_DATA=true # Include data/ directory (default: true)
|
||||||
|
|
||||||
|
# Storage configuration
|
||||||
|
BACKUP_STORAGE_PATH=/opt/backups # Production path (default: /opt/backups)
|
||||||
|
# BACKUP_STORAGE_PATH=./backups # Use this for local development
|
||||||
|
BACKUP_MAX_SIZE_GB=50 # Maximum storage size (default: 50)
|
||||||
|
STORAGE_WARNING_THRESHOLD_PCT=80 # Warn at 80% usage (default: 80)
|
||||||
|
|
||||||
|
# Retention policy
|
||||||
|
RETENTION_DAYS=30 # Keep daily backups for 30 days (default: 30)
|
||||||
|
MONTHLY_KEEP_MONTHS=12 # Keep monthly backups for 12 months (default: 12)
|
||||||
|
|
||||||
|
# Offsite uploads (SFTP/SSH)
|
||||||
|
OFFSITE_ENABLED=false # Disable until configured (default: false)
|
||||||
|
OFFSITE_WEEKLY_DAY=sunday # Day for weekly upload (default: sunday)
|
||||||
|
OFFSITE_RETRY_MAX_ATTEMPTS=3 # Max retry attempts (default: 3)
|
||||||
|
OFFSITE_RETRY_DELAY_HOURS=1 # Hours between retries (default: 1)
|
||||||
|
|
||||||
|
# SFTP/SSH connection (configure when enabling offsite)
|
||||||
|
SFTP_HOST=backup.example.com
|
||||||
|
SFTP_PORT=22
|
||||||
|
SFTP_USER=bmc_backup
|
||||||
|
SFTP_PASSWORD= # Leave empty if using SSH key
|
||||||
|
SSH_KEY_PATH=/path/to/id_rsa # Path to SSH private key (preferred)
|
||||||
|
SFTP_REMOTE_PATH=/backups/bmc_hub
|
||||||
|
|
||||||
|
# Mattermost notifications
|
||||||
|
MATTERMOST_ENABLED=false # Disable until webhook configured (default: false)
|
||||||
|
MATTERMOST_WEBHOOK_URL=https://mattermost.example.com/hooks/xxx
|
||||||
|
MATTERMOST_CHANNEL=backups
|
||||||
|
NOTIFY_ON_FAILURE=true # Send alerts on failures (default: true)
|
||||||
|
NOTIFY_ON_SUCCESS_OFFSITE=true # Send alerts on successful offsite uploads (default: true)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration Steps
|
||||||
|
|
||||||
|
### 1. Basic Setup (Local Development)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
BACKUP_ENABLED=true
|
||||||
|
BACKUP_DRY_RUN=false
|
||||||
|
BACKUP_READ_ONLY=false
|
||||||
|
BACKUP_STORAGE_PATH=./backups
|
||||||
|
```
|
||||||
|
|
||||||
|
Restart the application:
|
||||||
|
```bash
|
||||||
|
docker-compose restart api
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Enable Offsite Uploads
|
||||||
|
|
||||||
|
#### Using SSH Key (Recommended)
|
||||||
|
```bash
|
||||||
|
OFFSITE_ENABLED=true
|
||||||
|
SFTP_HOST=your-backup-server.com
|
||||||
|
SFTP_USER=backup_user
|
||||||
|
SSH_KEY_PATH=/path/to/id_rsa
|
||||||
|
SFTP_REMOTE_PATH=/backups/bmc_hub
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Using Password
|
||||||
|
```bash
|
||||||
|
OFFSITE_ENABLED=true
|
||||||
|
SFTP_HOST=your-backup-server.com
|
||||||
|
SFTP_USER=backup_user
|
||||||
|
SFTP_PASSWORD=your_secure_password
|
||||||
|
SFTP_REMOTE_PATH=/backups/bmc_hub
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Enable Mattermost Notifications
|
||||||
|
|
||||||
|
1. Create an incoming webhook in Mattermost
|
||||||
|
2. Copy the webhook URL
|
||||||
|
3. Add to `.env`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
MATTERMOST_ENABLED=true
|
||||||
|
MATTERMOST_WEBHOOK_URL=https://your-mattermost.com/hooks/xxxxxxxxxxxxx
|
||||||
|
MATTERMOST_CHANNEL=backups
|
||||||
|
```
|
||||||
|
|
||||||
|
## Scheduled Jobs
|
||||||
|
|
||||||
|
When `BACKUP_ENABLED=true`, the system automatically schedules:
|
||||||
|
|
||||||
|
- **Daily Backup**: 02:00 CET - Full backup (database + files) in compressed format
|
||||||
|
- **Monthly Backup**: 1st day at 02:00 CET - Full backup in plain SQL format
|
||||||
|
- **Weekly Offsite**: Sunday at 03:00 CET - Upload all pending backups to offsite
|
||||||
|
- **Backup Rotation**: Daily at 01:00 CET - Delete expired backups
|
||||||
|
- **Storage Check**: Daily at 01:30 CET - Check disk usage
|
||||||
|
|
||||||
|
## Manual Operations
|
||||||
|
|
||||||
|
### Via UI Dashboard
|
||||||
|
Visit: `http://localhost:8000/backups`
|
||||||
|
|
||||||
|
Features:
|
||||||
|
- Create manual backups (database, files, or full)
|
||||||
|
- View backup history with sizes and checksums
|
||||||
|
- Restore from backup (with confirmation)
|
||||||
|
- Manual offsite upload
|
||||||
|
- View notifications
|
||||||
|
- Monitor storage usage
|
||||||
|
|
||||||
|
### Via API
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Create manual backup
|
||||||
|
curl -X POST http://localhost:8000/api/v1/backups \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"job_type": "full", "is_monthly": false}'
|
||||||
|
|
||||||
|
# List backups
|
||||||
|
curl http://localhost:8000/api/v1/backups/jobs
|
||||||
|
|
||||||
|
# Get backup details
|
||||||
|
curl http://localhost:8000/api/v1/backups/jobs/1
|
||||||
|
|
||||||
|
# Restore from backup (⚠️ DANGER - enters maintenance mode)
|
||||||
|
curl -X POST http://localhost:8000/api/v1/backups/restore/1 \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"confirmation": true}'
|
||||||
|
|
||||||
|
# Upload to offsite
|
||||||
|
curl -X POST http://localhost:8000/api/v1/backups/offsite/1
|
||||||
|
|
||||||
|
# Check storage
|
||||||
|
curl http://localhost:8000/api/v1/backups/storage
|
||||||
|
|
||||||
|
# Check maintenance mode
|
||||||
|
curl http://localhost:8000/api/v1/backups/maintenance
|
||||||
|
```
|
||||||
|
|
||||||
|
## Database Migration
|
||||||
|
|
||||||
|
Migration has already been applied. If you need to re-run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker-compose exec -T postgres psql -U bmc_hub -d bmc_hub < migrations/024_backup_system.sql
|
||||||
|
```
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Backup not running
|
||||||
|
- Check `BACKUP_ENABLED=true` in `.env`
|
||||||
|
- Check logs: `docker-compose logs api | grep backup`
|
||||||
|
- Verify scheduler status via API: `curl http://localhost:8000/api/v1/backups/scheduler/status`
|
||||||
|
|
||||||
|
### Offsite upload failing
|
||||||
|
- Verify SFTP credentials
|
||||||
|
- Test SSH connection: `ssh -i /path/to/key user@host`
|
||||||
|
- Check retry count in backup history
|
||||||
|
- Review notifications in dashboard
|
||||||
|
|
||||||
|
### Storage full
|
||||||
|
- Increase `BACKUP_MAX_SIZE_GB`
|
||||||
|
- Reduce `RETENTION_DAYS`
|
||||||
|
- Manually delete old backups via UI
|
||||||
|
- Enable offsite uploads to move backups off-server
|
||||||
|
|
||||||
|
### Restore not working
|
||||||
|
- Set `BACKUP_READ_ONLY=false` in `.env`
|
||||||
|
- Restart API: `docker-compose restart api`
|
||||||
|
- Verify backup file exists on disk
|
||||||
|
- Check maintenance mode overlay appears during restore
|
||||||
|
|
||||||
|
## Safety Features
|
||||||
|
|
||||||
|
The system includes multiple safety switches:
|
||||||
|
|
||||||
|
1. **BACKUP_ENABLED** - Master switch, disabled by default
|
||||||
|
2. **BACKUP_DRY_RUN** - Logs operations without executing
|
||||||
|
3. **BACKUP_READ_ONLY** - Blocks destructive restore operations
|
||||||
|
4. **OFFSITE_ENABLED** - Disables external uploads until configured
|
||||||
|
5. **MATTERMOST_ENABLED** - Prevents notification spam
|
||||||
|
|
||||||
|
Always test with `BACKUP_DRY_RUN=true` first!
|
||||||
|
|
||||||
|
## Production Checklist
|
||||||
|
|
||||||
|
- [ ] Set strong SFTP credentials
|
||||||
|
- [ ] Configure SSH key authentication
|
||||||
|
- [ ] Test restore procedure (on test system first!)
|
||||||
|
- [ ] Configure Mattermost notifications
|
||||||
|
- [ ] Set appropriate retention periods
|
||||||
|
- [ ] Verify backup storage capacity
|
||||||
|
- [ ] Document restore procedures
|
||||||
|
- [ ] Schedule restore drills (quarterly)
|
||||||
|
- [ ] Monitor backup success rate
|
||||||
|
- [ ] Test offsite download/restore
|
||||||
|
|
||||||
|
## Backup Storage Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
/opt/backups/
|
||||||
|
├── database/
|
||||||
|
│ ├── db_20251213_020000_daily.dump
|
||||||
|
│ ├── db_20251201_020000_monthly.sql
|
||||||
|
│ └── ...
|
||||||
|
└── files/
|
||||||
|
├── files_20251213_020000.tar.gz
|
||||||
|
└── ...
|
||||||
|
```
|
||||||
|
|
||||||
|
## Support
|
||||||
|
|
||||||
|
For issues or questions, check:
|
||||||
|
- Logs: `/logs/app.log`
|
||||||
|
- API Docs: `http://localhost:8000/api/docs`
|
||||||
|
- Dashboard: `http://localhost:8000/backups`
|
||||||
456
docs/EMAIL_ACTIVITY_LOGGING.md
Normal file
456
docs/EMAIL_ACTIVITY_LOGGING.md
Normal file
@ -0,0 +1,456 @@
|
|||||||
|
# Email Activity Logging System
|
||||||
|
|
||||||
|
## Oversigt
|
||||||
|
|
||||||
|
Komplet audit trail system der logger **alt** hvad der sker med emails i BMC Hub. Hver handling, ændring og event bliver logget automatisk med timestamps, metadata og kontekst.
|
||||||
|
|
||||||
|
## 🎯 Hvad Bliver Logget?
|
||||||
|
|
||||||
|
### System Events
|
||||||
|
- **fetched**: Email hentet fra mail server
|
||||||
|
- **classified**: Email klassificeret af AI/keyword system
|
||||||
|
- **workflow_executed**: Workflow kørt på email
|
||||||
|
- **rule_matched**: Email regel matchet
|
||||||
|
- **status_changed**: Email status ændret
|
||||||
|
- **error**: Fejl opstået under processing
|
||||||
|
|
||||||
|
### User Events
|
||||||
|
- **read**: Email læst af bruger
|
||||||
|
- **attachment_downloaded**: Attachment downloaded
|
||||||
|
- **attachment_uploaded**: Attachment uploaded
|
||||||
|
|
||||||
|
### Integration Events
|
||||||
|
- **linked**: Email linket til vendor/customer/case
|
||||||
|
- **invoice_extracted**: Faktura data ekstraheret fra PDF
|
||||||
|
- **ticket_created**: Support ticket oprettet
|
||||||
|
- **notification_sent**: Notifikation sendt
|
||||||
|
|
||||||
|
## 📊 Database Schema
|
||||||
|
|
||||||
|
### email_activity_log Table
|
||||||
|
```sql
|
||||||
|
CREATE TABLE email_activity_log (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
email_id INTEGER NOT NULL, -- Hvilken email
|
||||||
|
event_type VARCHAR(50) NOT NULL, -- Hvad skete der
|
||||||
|
event_category VARCHAR(30) NOT NULL, -- Kategori (system/user/workflow/etc)
|
||||||
|
description TEXT NOT NULL, -- Human-readable beskrivelse
|
||||||
|
metadata JSONB, -- Ekstra data som JSON
|
||||||
|
user_id INTEGER, -- Bruger hvis user-triggered
|
||||||
|
created_at TIMESTAMP, -- Hvornår
|
||||||
|
created_by VARCHAR(255) -- Hvem/hvad
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
### email_timeline View
|
||||||
|
Pre-built view med joins til users og email_messages:
|
||||||
|
```sql
|
||||||
|
SELECT * FROM email_timeline WHERE email_id = 123;
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🔧 Hvordan Bruges Det?
|
||||||
|
|
||||||
|
### I Python Code
|
||||||
|
|
||||||
|
```python
|
||||||
|
from app.services.email_activity_logger import email_activity_logger
|
||||||
|
|
||||||
|
# Log email fetch
|
||||||
|
await email_activity_logger.log_fetched(
|
||||||
|
email_id=123,
|
||||||
|
source='imap',
|
||||||
|
message_id='msg-abc-123'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Log classification
|
||||||
|
await email_activity_logger.log_classified(
|
||||||
|
email_id=123,
|
||||||
|
classification='invoice',
|
||||||
|
confidence=0.85,
|
||||||
|
method='ai'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Log workflow execution
|
||||||
|
await email_activity_logger.log_workflow_executed(
|
||||||
|
email_id=123,
|
||||||
|
workflow_id=5,
|
||||||
|
workflow_name='Invoice Processing',
|
||||||
|
status='completed',
|
||||||
|
steps_completed=3,
|
||||||
|
execution_time_ms=1250
|
||||||
|
)
|
||||||
|
|
||||||
|
# Log status change
|
||||||
|
await email_activity_logger.log_status_changed(
|
||||||
|
email_id=123,
|
||||||
|
old_status='active',
|
||||||
|
new_status='processed',
|
||||||
|
reason='workflow completed'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Log entity linking
|
||||||
|
await email_activity_logger.log_linked(
|
||||||
|
email_id=123,
|
||||||
|
entity_type='vendor',
|
||||||
|
entity_id=42,
|
||||||
|
entity_name='Acme Corp'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Log invoice extraction
|
||||||
|
await email_activity_logger.log_invoice_extracted(
|
||||||
|
email_id=123,
|
||||||
|
invoice_number='INV-2025-001',
|
||||||
|
amount=1234.56,
|
||||||
|
success=True
|
||||||
|
)
|
||||||
|
|
||||||
|
# Log error
|
||||||
|
await email_activity_logger.log_error(
|
||||||
|
email_id=123,
|
||||||
|
error_type='extraction_failed',
|
||||||
|
error_message='PDF corrupted',
|
||||||
|
context={'file': 'invoice.pdf', 'size': 0}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Generic log (for custom events)
|
||||||
|
await email_activity_logger.log(
|
||||||
|
email_id=123,
|
||||||
|
event_type='custom_event',
|
||||||
|
category='integration',
|
||||||
|
description='Custom event happened',
|
||||||
|
metadata={'key': 'value'}
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Via SQL
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Log event directly via function
|
||||||
|
SELECT log_email_event(
|
||||||
|
123, -- email_id
|
||||||
|
'custom_event', -- event_type
|
||||||
|
'system', -- event_category
|
||||||
|
'Something happened', -- description
|
||||||
|
'{"foo": "bar"}'::jsonb, -- metadata (optional)
|
||||||
|
NULL, -- user_id (optional)
|
||||||
|
'system' -- created_by
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Query logs for specific email
|
||||||
|
SELECT * FROM email_activity_log
|
||||||
|
WHERE email_id = 123
|
||||||
|
ORDER BY created_at DESC;
|
||||||
|
|
||||||
|
-- Use the view for nicer output
|
||||||
|
SELECT * FROM email_timeline
|
||||||
|
WHERE email_id = 123;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Via API
|
||||||
|
|
||||||
|
```http
|
||||||
|
GET /api/v1/emails/123/activity
|
||||||
|
```
|
||||||
|
|
||||||
|
Response:
|
||||||
|
```json
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"email_id": 123,
|
||||||
|
"event_type": "fetched",
|
||||||
|
"event_category": "system",
|
||||||
|
"description": "Email fetched from email server",
|
||||||
|
"metadata": {
|
||||||
|
"source": "imap",
|
||||||
|
"message_id": "msg-abc-123"
|
||||||
|
},
|
||||||
|
"user_id": null,
|
||||||
|
"user_name": null,
|
||||||
|
"created_at": "2025-12-15T10:30:00",
|
||||||
|
"created_by": "system"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"email_id": 123,
|
||||||
|
"event_type": "classified",
|
||||||
|
"event_category": "system",
|
||||||
|
"description": "Classified as invoice (confidence: 85%)",
|
||||||
|
"metadata": {
|
||||||
|
"classification": "invoice",
|
||||||
|
"confidence": 0.85,
|
||||||
|
"method": "ai"
|
||||||
|
},
|
||||||
|
"created_at": "2025-12-15T10:30:02",
|
||||||
|
"created_by": "system"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🎨 UI Integration
|
||||||
|
|
||||||
|
### Email Detail View
|
||||||
|
|
||||||
|
Når du vælger en email i email UI:
|
||||||
|
1. Klik på **"Log"** tab i højre sidebar
|
||||||
|
2. Se komplet timeline af alle events
|
||||||
|
3. Ekspander metadata for detaljer
|
||||||
|
|
||||||
|
### Timeline Features
|
||||||
|
- **Kronologisk visning**: Nyeste først
|
||||||
|
- **Color-coded ikoner**: Baseret på event category
|
||||||
|
- 🔵 System events (blue)
|
||||||
|
- 🟢 User events (green)
|
||||||
|
- 🔷 Workflow events (cyan)
|
||||||
|
- 🟡 Rule events (yellow)
|
||||||
|
- ⚫ Integration events (gray)
|
||||||
|
- **Expandable metadata**: Klik for at se JSON details
|
||||||
|
- **User attribution**: Viser hvem der udførte action
|
||||||
|
|
||||||
|
## 📈 Analytics & Monitoring
|
||||||
|
|
||||||
|
### Recent Activity Across All Emails
|
||||||
|
|
||||||
|
```http
|
||||||
|
GET /api/v1/emails/activity/recent?limit=50&event_type=error
|
||||||
|
```
|
||||||
|
|
||||||
|
### Activity Statistics
|
||||||
|
|
||||||
|
```http
|
||||||
|
GET /api/v1/emails/activity/stats
|
||||||
|
```
|
||||||
|
|
||||||
|
Response:
|
||||||
|
```json
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"event_type": "classified",
|
||||||
|
"event_category": "system",
|
||||||
|
"count": 1523,
|
||||||
|
"last_occurrence": "2025-12-15T12:45:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"event_type": "workflow_executed",
|
||||||
|
"event_category": "workflow",
|
||||||
|
"count": 892,
|
||||||
|
"last_occurrence": "2025-12-15T12:44:30"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🔍 Use Cases
|
||||||
|
|
||||||
|
### 1. Debugging Email Processing
|
||||||
|
```sql
|
||||||
|
-- See complete flow for problematic email
|
||||||
|
SELECT
|
||||||
|
event_type,
|
||||||
|
description,
|
||||||
|
created_at
|
||||||
|
FROM email_activity_log
|
||||||
|
WHERE email_id = 123
|
||||||
|
ORDER BY created_at;
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Performance Monitoring
|
||||||
|
```sql
|
||||||
|
-- Find slow workflow executions
|
||||||
|
SELECT
|
||||||
|
email_id,
|
||||||
|
description,
|
||||||
|
(metadata->>'execution_time_ms')::int as exec_time
|
||||||
|
FROM email_activity_log
|
||||||
|
WHERE event_type = 'workflow_executed'
|
||||||
|
ORDER BY exec_time DESC
|
||||||
|
LIMIT 10;
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. User Activity Audit
|
||||||
|
```sql
|
||||||
|
-- See what user did
|
||||||
|
SELECT
|
||||||
|
e.subject,
|
||||||
|
a.event_type,
|
||||||
|
a.description,
|
||||||
|
a.created_at
|
||||||
|
FROM email_activity_log a
|
||||||
|
JOIN email_messages e ON a.email_id = e.id
|
||||||
|
WHERE a.user_id = 5
|
||||||
|
ORDER BY a.created_at DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. Error Analysis
|
||||||
|
```sql
|
||||||
|
-- Find common errors
|
||||||
|
SELECT
|
||||||
|
metadata->>'error_type' as error_type,
|
||||||
|
COUNT(*) as count
|
||||||
|
FROM email_activity_log
|
||||||
|
WHERE event_type = 'error'
|
||||||
|
GROUP BY error_type
|
||||||
|
ORDER BY count DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5. Workflow Success Rate
|
||||||
|
```sql
|
||||||
|
-- Calculate workflow success rate
|
||||||
|
SELECT
|
||||||
|
metadata->>'workflow_name' as workflow,
|
||||||
|
COUNT(*) FILTER (WHERE metadata->>'status' = 'completed') as success,
|
||||||
|
COUNT(*) FILTER (WHERE metadata->>'status' = 'failed') as failed,
|
||||||
|
ROUND(
|
||||||
|
100.0 * COUNT(*) FILTER (WHERE metadata->>'status' = 'completed') / COUNT(*),
|
||||||
|
2
|
||||||
|
) as success_rate
|
||||||
|
FROM email_activity_log
|
||||||
|
WHERE event_type = 'workflow_executed'
|
||||||
|
GROUP BY workflow
|
||||||
|
ORDER BY success_rate DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🚀 Auto-Logging
|
||||||
|
|
||||||
|
Følgende er allerede implementeret og logger automatisk:
|
||||||
|
|
||||||
|
✅ **Email Fetching** - Logged når emails hentes
|
||||||
|
✅ **Classification** - Logged når AI klassificerer
|
||||||
|
✅ **Workflow Execution** - Logged ved start og completion
|
||||||
|
✅ **Status Changes** - Logged når email status ændres
|
||||||
|
|
||||||
|
### Kommende Auto-Logging
|
||||||
|
⏳ Rule matching (tilføjes snart)
|
||||||
|
⏳ User read events (når user åbner email)
|
||||||
|
⏳ Attachment actions (download/upload)
|
||||||
|
⏳ Entity linking (vendor/customer association)
|
||||||
|
|
||||||
|
## 💡 Best Practices
|
||||||
|
|
||||||
|
### 1. Always Include Metadata
|
||||||
|
```python
|
||||||
|
# ❌ Bad - No context
|
||||||
|
await email_activity_logger.log(
|
||||||
|
email_id=123,
|
||||||
|
event_type='action_performed',
|
||||||
|
category='system',
|
||||||
|
description='Something happened'
|
||||||
|
)
|
||||||
|
|
||||||
|
# ✅ Good - Rich context
|
||||||
|
await email_activity_logger.log(
|
||||||
|
email_id=123,
|
||||||
|
event_type='invoice_sent',
|
||||||
|
category='integration',
|
||||||
|
description='Invoice sent to e-conomic',
|
||||||
|
metadata={
|
||||||
|
'invoice_number': 'INV-2025-001',
|
||||||
|
'economic_id': 12345,
|
||||||
|
'amount': 1234.56,
|
||||||
|
'sent_at': datetime.now().isoformat()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Use Descriptive Event Types
|
||||||
|
```python
|
||||||
|
# ❌ Bad - Generic
|
||||||
|
event_type='action'
|
||||||
|
|
||||||
|
# ✅ Good - Specific
|
||||||
|
event_type='invoice_sent_to_economic'
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Choose Correct Category
|
||||||
|
- **system**: Automated system actions
|
||||||
|
- **user**: User-triggered actions
|
||||||
|
- **workflow**: Workflow executions
|
||||||
|
- **rule**: Rule-based actions
|
||||||
|
- **integration**: External system integrations
|
||||||
|
|
||||||
|
### 4. Log Errors with Context
|
||||||
|
```python
|
||||||
|
try:
|
||||||
|
result = extract_invoice_data(pdf_path)
|
||||||
|
except Exception as e:
|
||||||
|
await email_activity_logger.log_error(
|
||||||
|
email_id=email_id,
|
||||||
|
error_type='extraction_failed',
|
||||||
|
error_message=str(e),
|
||||||
|
context={
|
||||||
|
'pdf_path': pdf_path,
|
||||||
|
'file_size': os.path.getsize(pdf_path),
|
||||||
|
'traceback': traceback.format_exc()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🔒 Data Retention
|
||||||
|
|
||||||
|
Activity logs kan vokse hurtigt. Implementer cleanup strategi:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Delete logs older than 90 days
|
||||||
|
DELETE FROM email_activity_log
|
||||||
|
WHERE created_at < NOW() - INTERVAL '90 days';
|
||||||
|
|
||||||
|
-- Archive old logs to separate table
|
||||||
|
INSERT INTO email_activity_log_archive
|
||||||
|
SELECT * FROM email_activity_log
|
||||||
|
WHERE created_at < NOW() - INTERVAL '30 days';
|
||||||
|
|
||||||
|
DELETE FROM email_activity_log
|
||||||
|
WHERE created_at < NOW() - INTERVAL '30 days';
|
||||||
|
```
|
||||||
|
|
||||||
|
## 📊 Performance Considerations
|
||||||
|
|
||||||
|
Med indexes på `email_id`, `event_type`, `created_at` og `event_category`, kan systemet håndtere millioner af log entries uden performance issues.
|
||||||
|
|
||||||
|
### Index Usage
|
||||||
|
```sql
|
||||||
|
-- Fast: Uses idx_email_activity_log_email_id
|
||||||
|
SELECT * FROM email_activity_log WHERE email_id = 123;
|
||||||
|
|
||||||
|
-- Fast: Uses idx_email_activity_log_event_type
|
||||||
|
SELECT * FROM email_activity_log WHERE event_type = 'workflow_executed';
|
||||||
|
|
||||||
|
-- Fast: Uses idx_email_activity_log_created_at
|
||||||
|
SELECT * FROM email_activity_log WHERE created_at > NOW() - INTERVAL '1 day';
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🎓 Examples
|
||||||
|
|
||||||
|
### Complete Email Lifecycle Log
|
||||||
|
```python
|
||||||
|
# 1. Email arrives
|
||||||
|
await email_activity_logger.log_fetched(email_id, 'imap', message_id)
|
||||||
|
|
||||||
|
# 2. AI classifies it
|
||||||
|
await email_activity_logger.log_classified(email_id, 'invoice', 0.92, 'ai')
|
||||||
|
|
||||||
|
# 3. Workflow processes it
|
||||||
|
await email_activity_logger.log_workflow_executed(
|
||||||
|
email_id, workflow_id, 'Invoice Processing', 'completed', 3, 1100
|
||||||
|
)
|
||||||
|
|
||||||
|
# 4. Links to vendor
|
||||||
|
await email_activity_logger.log_linked(email_id, 'vendor', 42, 'Acme Corp')
|
||||||
|
|
||||||
|
# 5. Extracts invoice
|
||||||
|
await email_activity_logger.log_invoice_extracted(
|
||||||
|
email_id, 'INV-001', 1234.56, True
|
||||||
|
)
|
||||||
|
|
||||||
|
# 6. Status changes
|
||||||
|
await email_activity_logger.log_status_changed(
|
||||||
|
email_id, 'active', 'processed', 'workflow completed'
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Result: **Complete audit trail af email fra fetch til processed!**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Version**: 1.0
|
||||||
|
**Last Updated**: 15. december 2025
|
||||||
|
**Status**: ✅ Production Ready
|
||||||
258
docs/EMAIL_RULES_TO_WORKFLOWS_MIGRATION.md
Normal file
258
docs/EMAIL_RULES_TO_WORKFLOWS_MIGRATION.md
Normal file
@ -0,0 +1,258 @@
|
|||||||
|
# Email Rules → Workflows Migration Guide
|
||||||
|
|
||||||
|
## 🎯 Formål
|
||||||
|
|
||||||
|
BMC Hub er ved at phase out det gamle **Rules** system til fordel for det nyere og mere kraftfulde **Workflows** system.
|
||||||
|
|
||||||
|
**Status:**
|
||||||
|
- ✅ Workflows er default aktiveret (`EMAIL_WORKFLOWS_ENABLED=true`)
|
||||||
|
- ⚠️ Rules er nu disabled by default (`EMAIL_RULES_ENABLED=false`)
|
||||||
|
|
||||||
|
## 🔄 Hvad Er Ændret?
|
||||||
|
|
||||||
|
### I koden:
|
||||||
|
1. **Koordinering tilføjet**: Workflows kører først, rules kun hvis workflow ikke har processed emailen
|
||||||
|
2. **Deduplication**: Samme action køres ikke 2 gange
|
||||||
|
3. **Config ændring**: `EMAIL_RULES_ENABLED` er nu `false` by default
|
||||||
|
4. **Silent failures fixet**: extract_invoice_data fejler nu synligt hvis fil mangler
|
||||||
|
|
||||||
|
### I `.env`:
|
||||||
|
```bash
|
||||||
|
# Gammelt (deprecated):
|
||||||
|
EMAIL_RULES_ENABLED=true
|
||||||
|
EMAIL_RULES_AUTO_PROCESS=false
|
||||||
|
|
||||||
|
# Nyt (anbefalet):
|
||||||
|
EMAIL_WORKFLOWS_ENABLED=true
|
||||||
|
EMAIL_RULES_ENABLED=false
|
||||||
|
```
|
||||||
|
|
||||||
|
## 📋 Migration Steps
|
||||||
|
|
||||||
|
### Trin 1: Identificer Aktive Rules
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Se alle aktive rules
|
||||||
|
SELECT id, name, action_type, conditions, priority, match_count
|
||||||
|
FROM email_rules
|
||||||
|
WHERE enabled = true
|
||||||
|
ORDER BY priority ASC;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Trin 2: Opret Tilsvarende Workflows
|
||||||
|
|
||||||
|
For hver rule, opret en workflow:
|
||||||
|
|
||||||
|
**Eksempel: Rule → Workflow**
|
||||||
|
|
||||||
|
**Gammel Rule:**
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"name": "Link Supplier Invoices",
|
||||||
|
"conditions": {
|
||||||
|
"classification": "invoice",
|
||||||
|
"sender_domain": ["example.com", "vendor.dk"]
|
||||||
|
},
|
||||||
|
"action_type": "link_supplier",
|
||||||
|
"priority": 10
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Ny Workflow:**
|
||||||
|
```sql
|
||||||
|
INSERT INTO email_workflows (
|
||||||
|
name,
|
||||||
|
description,
|
||||||
|
classification_trigger,
|
||||||
|
sender_pattern,
|
||||||
|
confidence_threshold,
|
||||||
|
workflow_steps,
|
||||||
|
priority,
|
||||||
|
enabled
|
||||||
|
) VALUES (
|
||||||
|
'Link Supplier Invoices',
|
||||||
|
'Automatically link invoice emails from known suppliers',
|
||||||
|
'invoice',
|
||||||
|
'(example\\.com|vendor\\.dk)$', -- Regex pattern
|
||||||
|
0.70,
|
||||||
|
'[
|
||||||
|
{"action": "link_to_vendor", "params": {"match_by": "email"}},
|
||||||
|
{"action": "extract_invoice_data", "params": {}},
|
||||||
|
{"action": "mark_as_processed", "params": {}}
|
||||||
|
]'::jsonb,
|
||||||
|
10,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
### Trin 3: Test Workflows
|
||||||
|
|
||||||
|
1. Send test email der matcher classification
|
||||||
|
2. Check `email_workflow_executions` tabel for results
|
||||||
|
3. Verificer at email blev processed korrekt
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Se workflow executions
|
||||||
|
SELECT
|
||||||
|
e.id,
|
||||||
|
w.name as workflow_name,
|
||||||
|
em.subject,
|
||||||
|
e.status,
|
||||||
|
e.steps_completed,
|
||||||
|
e.execution_time_ms,
|
||||||
|
e.started_at
|
||||||
|
FROM email_workflow_executions e
|
||||||
|
JOIN email_workflows w ON w.id = e.workflow_id
|
||||||
|
JOIN email_messages em ON em.id = e.email_id
|
||||||
|
ORDER BY e.started_at DESC
|
||||||
|
LIMIT 20;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Trin 4: Disable Rules
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Disable alle rules
|
||||||
|
UPDATE email_rules SET enabled = false;
|
||||||
|
```
|
||||||
|
|
||||||
|
Eller i `.env`:
|
||||||
|
```bash
|
||||||
|
EMAIL_RULES_ENABLED=false
|
||||||
|
```
|
||||||
|
|
||||||
|
### Trin 5: Monitor
|
||||||
|
|
||||||
|
I de første dage efter migration:
|
||||||
|
- Check logs for fejl
|
||||||
|
- Verificer at workflows kører som forventet
|
||||||
|
- Check at ingen emails falder igennem
|
||||||
|
|
||||||
|
## 🗂️ Mapping: Rules → Workflows
|
||||||
|
|
||||||
|
### Action Mapping
|
||||||
|
|
||||||
|
| Rule Action | Workflow Action | Notes |
|
||||||
|
|-------------|-----------------|-------|
|
||||||
|
| `link_supplier` | `link_to_vendor` | ✅ Direct replacement |
|
||||||
|
| `link_customer` | `link_to_customer` | ⚠️ Not fully implemented yet |
|
||||||
|
| `link_case` | `create_ticket` | ✅ Creates ticket from email |
|
||||||
|
| `mark_spam` | *(none)* | ⚠️ Needs workflow action |
|
||||||
|
| `create_purchase` | *(none)* | ⚠️ Not implemented |
|
||||||
|
|
||||||
|
### Condition Mapping
|
||||||
|
|
||||||
|
| Rule Condition | Workflow Equivalent |
|
||||||
|
|----------------|---------------------|
|
||||||
|
| `classification` | `classification_trigger` |
|
||||||
|
| `sender_email` | `sender_pattern` (exact match regex) |
|
||||||
|
| `sender_domain` | `sender_pattern` (domain regex) |
|
||||||
|
| `subject_contains` | `subject_pattern` (contains regex) |
|
||||||
|
| `subject_regex` | `subject_pattern` (direct) |
|
||||||
|
| `confidence_score` | `confidence_threshold` |
|
||||||
|
|
||||||
|
## 🆕 Workflow-Only Features
|
||||||
|
|
||||||
|
Workflows kan mere end rules:
|
||||||
|
|
||||||
|
1. **Multi-step execution**: Chain multiple actions
|
||||||
|
2. **Better error handling**: Each step tracked separately
|
||||||
|
3. **Execution history**: Full audit trail
|
||||||
|
4. **Regex patterns**: More flexible matching
|
||||||
|
5. **Stop on match**: Control workflow chaining
|
||||||
|
6. **Statistics**: Success/failure rates
|
||||||
|
|
||||||
|
## ⚠️ Backward Compatibility
|
||||||
|
|
||||||
|
**Hvis du MÅ beholde rules:**
|
||||||
|
|
||||||
|
Set i `.env`:
|
||||||
|
```bash
|
||||||
|
EMAIL_RULES_ENABLED=true
|
||||||
|
EMAIL_WORKFLOWS_ENABLED=true
|
||||||
|
```
|
||||||
|
|
||||||
|
**Systemet vil:**
|
||||||
|
- Køre workflows først
|
||||||
|
- Kun køre rules hvis workflow ikke processede emailen
|
||||||
|
- Undgå duplicate actions
|
||||||
|
|
||||||
|
**Men dette er ikke anbefalet** - rules vil blive fjernet i fremtiden.
|
||||||
|
|
||||||
|
## 📊 Status Dashboard
|
||||||
|
|
||||||
|
Tilføj til admin UI:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Workflow statistics
|
||||||
|
SELECT
|
||||||
|
w.name,
|
||||||
|
w.classification_trigger,
|
||||||
|
w.execution_count,
|
||||||
|
w.success_count,
|
||||||
|
w.failure_count,
|
||||||
|
ROUND(100.0 * w.success_count / NULLIF(w.execution_count, 0), 1) as success_rate,
|
||||||
|
w.last_executed_at
|
||||||
|
FROM email_workflows w
|
||||||
|
WHERE w.enabled = true
|
||||||
|
ORDER BY w.execution_count DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🐛 Troubleshooting
|
||||||
|
|
||||||
|
### "Workflow ikke executed"
|
||||||
|
|
||||||
|
**Check:**
|
||||||
|
1. Er `EMAIL_WORKFLOWS_ENABLED=true` i .env?
|
||||||
|
2. Er workflow enabled i database?
|
||||||
|
3. Matcher classification_trigger?
|
||||||
|
4. Er confidence over threshold?
|
||||||
|
5. Matcher sender/subject patterns?
|
||||||
|
|
||||||
|
**Debug:**
|
||||||
|
```python
|
||||||
|
# I logs se:
|
||||||
|
# "🔄 Finding workflows for classification: invoice (confidence: 0.95)"
|
||||||
|
# "📋 Found X matching workflow(s)"
|
||||||
|
```
|
||||||
|
|
||||||
|
### "Email processed 2 gange"
|
||||||
|
|
||||||
|
**Check:**
|
||||||
|
1. Er både workflows OG rules enabled?
|
||||||
|
2. Har de samme actions?
|
||||||
|
|
||||||
|
**Fix:**
|
||||||
|
Disable rules: `EMAIL_RULES_ENABLED=false`
|
||||||
|
|
||||||
|
### "Workflow fejler stille"
|
||||||
|
|
||||||
|
**Efter fix:**
|
||||||
|
- extract_invoice_data raiser nu FileNotFoundError hvis PDF mangler
|
||||||
|
- Check `email_workflow_executions.status = 'failed'`
|
||||||
|
- Check `error_message` column
|
||||||
|
|
||||||
|
## ✅ Success Criteria
|
||||||
|
|
||||||
|
Migration er komplet når:
|
||||||
|
|
||||||
|
1. ✅ Alle rules er migrated til workflows
|
||||||
|
2. ✅ Workflows tested og virker
|
||||||
|
3. ✅ `EMAIL_RULES_ENABLED=false` i produktion
|
||||||
|
4. ✅ Ingen emails falder igennem
|
||||||
|
5. ✅ email_workflow_executions viser success
|
||||||
|
|
||||||
|
## 📞 Support
|
||||||
|
|
||||||
|
Hvis problemer:
|
||||||
|
1. Check logs: `docker-compose logs -f api | grep "🔄\|❌\|✅"`
|
||||||
|
2. Check database: `SELECT * FROM email_workflow_executions ORDER BY started_at DESC LIMIT 10;`
|
||||||
|
3. Revert: Set `EMAIL_RULES_ENABLED=true` midlertidigt
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Tidslinje:**
|
||||||
|
- ✅ Nu: Workflows aktiveret, rules disabled by default
|
||||||
|
- 🔜 Næste sprint: Fjern rule UI fra admin
|
||||||
|
- 🔜 Om 1 måned: Drop email_rules tabel helt
|
||||||
|
|
||||||
|
**Anbefaling:** Migrer nu, imens begge systemer er tilgængelige som fallback.
|
||||||
316
docs/EMAIL_RULES_VS_WORKFLOWS_ANALYSIS.md
Normal file
316
docs/EMAIL_RULES_VS_WORKFLOWS_ANALYSIS.md
Normal file
@ -0,0 +1,316 @@
|
|||||||
|
# Email Rules vs Workflows - Analyse
|
||||||
|
|
||||||
|
## 🔍 Oversigt
|
||||||
|
|
||||||
|
BMC Hub har **2 systemer** til automatisk email-behandling:
|
||||||
|
1. **Email Rules** (legacy) - `email_rules` tabel
|
||||||
|
2. **Email Workflows** (nyere) - `email_workflows` tabel
|
||||||
|
|
||||||
|
## ⚙️ Hvordan Fungerer De?
|
||||||
|
|
||||||
|
### Email Processing Flow
|
||||||
|
|
||||||
|
```
|
||||||
|
📧 Ny Email Modtaget
|
||||||
|
↓
|
||||||
|
1️⃣ Save email til database (email_messages)
|
||||||
|
↓
|
||||||
|
2️⃣ Classify med AI/simple classifier
|
||||||
|
↓ (classification + confidence_score gemmes)
|
||||||
|
↓
|
||||||
|
3️⃣ Execute WORKFLOWS først 🆕
|
||||||
|
├─ Finder workflows med matching classification
|
||||||
|
├─ Tjekker confidence_threshold
|
||||||
|
├─ Checker sender/subject patterns (regex)
|
||||||
|
├─ Executer workflow steps i rækkefølge
|
||||||
|
└─ Stopper hvis stop_on_match=true
|
||||||
|
↓
|
||||||
|
4️⃣ Match RULES bagefter (legacy) 🕰️
|
||||||
|
├─ Finder rules med matching conditions
|
||||||
|
├─ Tjekker sender, domain, classification, subject
|
||||||
|
├─ Executer rule action (kun 1 action per rule)
|
||||||
|
└─ Stopper efter første match
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🆚 Forskelle
|
||||||
|
|
||||||
|
| Feature | Email Rules (Legacy) | Email Workflows (Ny) |
|
||||||
|
|---------|---------------------|---------------------|
|
||||||
|
| **Fleksibilitet** | Enkelt action per rule | Multiple steps per workflow |
|
||||||
|
| **Priority** | Ja (priority field) | Ja (priority field) |
|
||||||
|
| **Stop on match** | Implicit (første match vinder) | Explicit (stop_on_match flag) |
|
||||||
|
| **Pattern matching** | Basic (exact match, contains) | Advanced (regex patterns) |
|
||||||
|
| **Confidence check** | Nej | Ja (confidence_threshold) |
|
||||||
|
| **Execution tracking** | Nej | Ja (email_workflow_executions) |
|
||||||
|
| **Statistics** | Ja (match_count) | Ja (execution_count, success/failure) |
|
||||||
|
| **Actions** | 5 types | 10+ types |
|
||||||
|
| **Database table** | email_rules | email_workflows |
|
||||||
|
| **Enabled by** | EMAIL_RULES_ENABLED | EMAIL_WORKFLOWS_ENABLED |
|
||||||
|
| **Auto-execute** | EMAIL_RULES_AUTO_PROCESS | Altid (hvis enabled) |
|
||||||
|
|
||||||
|
## ⚠️ PROBLEM: Duplikering og Konflikter
|
||||||
|
|
||||||
|
### 1. Begge Kan Køre Samtidigt
|
||||||
|
|
||||||
|
**Scenarie:**
|
||||||
|
```
|
||||||
|
Email: Faktura fra leverandør@example.com
|
||||||
|
Classification: invoice, confidence: 0.95
|
||||||
|
|
||||||
|
WORKFLOW matches:
|
||||||
|
- "Invoice Processing Workflow"
|
||||||
|
→ Steps: link_to_vendor, extract_invoice_data, mark_as_processed
|
||||||
|
→ Executes first! ✅
|
||||||
|
|
||||||
|
RULE matches:
|
||||||
|
- "Link Supplier Emails"
|
||||||
|
→ Action: link_supplier
|
||||||
|
→ Executes after! ⚠️
|
||||||
|
|
||||||
|
RESULTAT: link_to_vendor køres 2 gange!
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Ingen Koordination
|
||||||
|
|
||||||
|
Workflows ved ikke om rules har kørt (eller omvendt).
|
||||||
|
|
||||||
|
**Problem:**
|
||||||
|
- Email kan markeres som "processed" af workflow
|
||||||
|
- Rule prøver stadig at køre action bagefter
|
||||||
|
- Resultatet logges 2 steder (workflow_executions + rule match_count)
|
||||||
|
|
||||||
|
### 3. Overlappende Actions
|
||||||
|
|
||||||
|
**Samme funktionalitet i begge systemer:**
|
||||||
|
|
||||||
|
| Action Type | Rule Name | Workflow Action |
|
||||||
|
|-------------|-----------|----------------|
|
||||||
|
| Link vendor | `link_supplier` | `link_to_vendor` |
|
||||||
|
| Link customer | `link_customer` | `link_to_customer` |
|
||||||
|
| Mark spam | `mark_spam` | *(mangler)* |
|
||||||
|
| Link case | `link_case` | `create_ticket` |
|
||||||
|
| Invoice extraction | *(mangler)* | `extract_invoice_data` |
|
||||||
|
|
||||||
|
### 4. Auto-Process Flag Virker Ikke for Workflows
|
||||||
|
|
||||||
|
**I koden:**
|
||||||
|
```python
|
||||||
|
# Rules respekterer auto-process flag
|
||||||
|
if self.auto_process:
|
||||||
|
await self._execute_rule_action(email_data, rule)
|
||||||
|
else:
|
||||||
|
logger.info(f"⏭️ Auto-process disabled - rule action not executed")
|
||||||
|
|
||||||
|
# Workflows kører ALTID hvis enabled=true
|
||||||
|
workflow_result = await email_workflow_service.execute_workflows(email_data)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Problem:** Man kan ikke disable workflow auto-execution uden at disable hele workflow systemet.
|
||||||
|
|
||||||
|
## ✅ Hvad Virker Godt
|
||||||
|
|
||||||
|
### 1. Workflows Er Mere Kraftfulde
|
||||||
|
- Multi-step execution
|
||||||
|
- Better tracking (execution history)
|
||||||
|
- Regex pattern matching
|
||||||
|
- Confidence threshold check
|
||||||
|
- Success/failure statistics
|
||||||
|
|
||||||
|
### 2. Rules Er Simplere
|
||||||
|
- God til simple hvis-så logik
|
||||||
|
- Lettere at forstå for non-technical brugere
|
||||||
|
- Fungerer fint for basic email routing
|
||||||
|
|
||||||
|
### 3. Begge Har Priority Ordering
|
||||||
|
- Workflows executes i priority order
|
||||||
|
- Rules matches i priority order
|
||||||
|
- Første match kan stoppe kæden (hvis configured)
|
||||||
|
|
||||||
|
## 🐛 Konkrete Bugs Fundet
|
||||||
|
|
||||||
|
### Bug #1: Workflow Executes ALTID
|
||||||
|
**Kode:** `email_processor_service.py` line 77-79
|
||||||
|
```python
|
||||||
|
# Step 4: Execute workflows based on classification
|
||||||
|
workflow_result = await email_workflow_service.execute_workflows(email_data)
|
||||||
|
```
|
||||||
|
**Problem:** Ingen check af `EMAIL_RULES_AUTO_PROCESS` eller lignende flag.
|
||||||
|
|
||||||
|
**Fix:**
|
||||||
|
```python
|
||||||
|
if settings.EMAIL_WORKFLOWS_ENABLED:
|
||||||
|
workflow_result = await email_workflow_service.execute_workflows(email_data)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Bug #2: Rules Kører Efter Workflows
|
||||||
|
**Kode:** `email_processor_service.py` line 84-88
|
||||||
|
```python
|
||||||
|
# Step 5: Match against rules (legacy support)
|
||||||
|
if self.rules_enabled:
|
||||||
|
matched = await self._match_rules(email_data)
|
||||||
|
```
|
||||||
|
**Problem:** Hvis workflow allerede har processed emailen, skal rule ikke køre.
|
||||||
|
|
||||||
|
**Fix:**
|
||||||
|
```python
|
||||||
|
# Step 5: Match against rules (legacy support) - skip if already processed by workflow
|
||||||
|
if self.rules_enabled and not email_data.get('_workflow_processed'):
|
||||||
|
matched = await self._match_rules(email_data)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Bug #3: Manglende Deduplication
|
||||||
|
**Problem:** Samme action kan executes af både workflow og rule.
|
||||||
|
|
||||||
|
**Fix:** Add check i rule execution:
|
||||||
|
```python
|
||||||
|
# Check if email already processed by workflow
|
||||||
|
already_processed = execute_query(
|
||||||
|
"SELECT id FROM email_workflow_executions WHERE email_id = %s AND status = 'completed'",
|
||||||
|
(email_id,), fetchone=True
|
||||||
|
)
|
||||||
|
if already_processed:
|
||||||
|
logger.info(f"⏭️ Email already processed by workflow, skipping rule")
|
||||||
|
return False
|
||||||
|
```
|
||||||
|
|
||||||
|
### Bug #4: `extract_invoice_data` Workflow Action Kan Fejle Stille
|
||||||
|
**Kode:** `email_workflow_service.py` line 380+
|
||||||
|
```python
|
||||||
|
if not file_path.exists():
|
||||||
|
# No error raised! Just continues...
|
||||||
|
```
|
||||||
|
**Problem:** Hvis PDF fil ikke findes, fejler workflow ikke - den fortsætter bare.
|
||||||
|
|
||||||
|
**Fix:** Raise exception:
|
||||||
|
```python
|
||||||
|
if not file_path.exists():
|
||||||
|
raise FileNotFoundError(f"Attachment file not found: {attachment_path}")
|
||||||
|
```
|
||||||
|
|
||||||
|
## 💡 Anbefalinger
|
||||||
|
|
||||||
|
### Anbefaling #1: Vælg ÉT System
|
||||||
|
**Option A: Deprecate Rules (anbefalet)**
|
||||||
|
- Workflows er mere kraftfulde
|
||||||
|
- Better tracking og debugging
|
||||||
|
- Fremtidssikret arkitektur
|
||||||
|
|
||||||
|
**Migration plan:**
|
||||||
|
1. Opret workflows der matcher alle aktive rules
|
||||||
|
2. Disable rules (set enabled=false)
|
||||||
|
3. Test workflows grundigt
|
||||||
|
4. Fjern rule execution fra processor
|
||||||
|
|
||||||
|
**Option B: Keep Both, Men Koordinér**
|
||||||
|
- Add `_workflow_processed` flag til email_data
|
||||||
|
- Skip rules hvis workflow har kørt
|
||||||
|
- Document clearly når man skal bruge rules vs workflows
|
||||||
|
|
||||||
|
### Anbefaling #2: Tilføj Workflow Auto-Process Flag
|
||||||
|
**Tilføj til `email_workflows` tabel:**
|
||||||
|
```sql
|
||||||
|
ALTER TABLE email_workflows ADD COLUMN auto_execute BOOLEAN DEFAULT true;
|
||||||
|
```
|
||||||
|
|
||||||
|
**Check flag før execution:**
|
||||||
|
```python
|
||||||
|
if workflow.get('auto_execute', True):
|
||||||
|
result = await self._execute_workflow(workflow, email_data)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Anbefaling #3: Unified Action Registry
|
||||||
|
**Opret fælles action handlers:**
|
||||||
|
```python
|
||||||
|
# shared/email_actions.py
|
||||||
|
class EmailActions:
|
||||||
|
@staticmethod
|
||||||
|
async def link_to_vendor(email_id, vendor_id):
|
||||||
|
# Single implementation used by both rules and workflows
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
### Anbefaling #4: Better Conflict Detection
|
||||||
|
**Add admin UI warning:**
|
||||||
|
```python
|
||||||
|
# Check for overlapping rules and workflows
|
||||||
|
def check_conflicts():
|
||||||
|
conflicts = []
|
||||||
|
for rule in active_rules:
|
||||||
|
for workflow in active_workflows:
|
||||||
|
if might_conflict(rule, workflow):
|
||||||
|
conflicts.append({
|
||||||
|
'rule': rule['name'],
|
||||||
|
'workflow': workflow['name'],
|
||||||
|
'reason': 'Both match same classification'
|
||||||
|
})
|
||||||
|
return conflicts
|
||||||
|
```
|
||||||
|
|
||||||
|
### Anbefaling #5: Execution Log Consolidation
|
||||||
|
**Single view af alle actions:**
|
||||||
|
```sql
|
||||||
|
CREATE VIEW email_action_log AS
|
||||||
|
SELECT
|
||||||
|
'workflow' as source,
|
||||||
|
e.email_id,
|
||||||
|
w.name as action_name,
|
||||||
|
e.status,
|
||||||
|
e.started_at
|
||||||
|
FROM email_workflow_executions e
|
||||||
|
JOIN email_workflows w ON w.id = e.workflow_id
|
||||||
|
UNION ALL
|
||||||
|
SELECT
|
||||||
|
'rule' as source,
|
||||||
|
em.id as email_id,
|
||||||
|
er.name as action_name,
|
||||||
|
CASE WHEN em.auto_processed THEN 'completed' ELSE 'skipped' END as status,
|
||||||
|
em.updated_at as started_at
|
||||||
|
FROM email_messages em
|
||||||
|
JOIN email_rules er ON er.id = em.rule_id
|
||||||
|
WHERE em.rule_id IS NOT NULL
|
||||||
|
ORDER BY started_at DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🎯 Action Plan
|
||||||
|
|
||||||
|
### Umiddelbart (Kritisk):
|
||||||
|
1. ✅ Add `EMAIL_WORKFLOWS_ENABLED` check før workflow execution
|
||||||
|
2. ✅ Add workflow-processed check før rule matching
|
||||||
|
3. ✅ Fix `extract_invoice_data` silent failure
|
||||||
|
4. ✅ Add duplicate action detection
|
||||||
|
|
||||||
|
### Kort Sigt:
|
||||||
|
5. Add `auto_execute` column til workflows tabel
|
||||||
|
6. Create unified action handlers
|
||||||
|
7. Add conflict detection admin tool
|
||||||
|
8. Document clearly hvornår man skal bruge hvad
|
||||||
|
|
||||||
|
### Lang Sigt:
|
||||||
|
9. Decide: Deprecate rules eller keep both?
|
||||||
|
10. Migrate existing rules til workflows (hvis deprecating)
|
||||||
|
11. Create unified execution log view
|
||||||
|
12. Add UI for viewing all email actions i ét dashboard
|
||||||
|
|
||||||
|
## 📊 Hvad Skal Du Gøre Nu?
|
||||||
|
|
||||||
|
**Spørgsmål til dig:**
|
||||||
|
|
||||||
|
1. **Vil du beholde begge systemer eller kun workflows?**
|
||||||
|
- Hvis kun workflows: Vi kan migrate rules → workflows nu
|
||||||
|
- Hvis begge: Vi skal fixe koordineringen
|
||||||
|
|
||||||
|
2. **Skal workflows kunne disables uden at slukke helt for systemet?**
|
||||||
|
- Ja → Vi tilføjer auto_execute flag
|
||||||
|
- Nej → Workflows kører altid når enabled=true
|
||||||
|
|
||||||
|
3. **Er der aktive rules i produktion lige nu?**
|
||||||
|
- Ja → Vi skal være forsigtige med ændringer
|
||||||
|
- Nej → Vi kan bare disable rule system
|
||||||
|
|
||||||
|
**Quick Fix (5 min):**
|
||||||
|
Jeg kan tilføje de 4 kritiske fixes nu hvis du vil fortsætte med begge systemer.
|
||||||
|
|
||||||
|
**Long Fix (1 time):**
|
||||||
|
Jeg kan deprecate rules og migrate til workflows hvis du vil simplificere.
|
||||||
|
|
||||||
|
Hvad foretrækker du? 🤔
|
||||||
171
docs/EMAIL_SYSTEM_ANALYSIS.md
Normal file
171
docs/EMAIL_SYSTEM_ANALYSIS.md
Normal file
@ -0,0 +1,171 @@
|
|||||||
|
# Email System - Fejl & Forbedringsforslag
|
||||||
|
|
||||||
|
## 🔴 Kritiske Fejl
|
||||||
|
|
||||||
|
### 1. Type Errors i Backend (router.py)
|
||||||
|
**Problem:** `execute_query(fetchone=True)` returnerer måske `None`, men koden antager altid dict
|
||||||
|
**Lokation:** Line 253-255 i mark-processed endpoint
|
||||||
|
**Løsning:** ✅ RETTET - Tilføjet `.get()` fallbacks
|
||||||
|
|
||||||
|
### 2. Manglende Error Handling i Frontend
|
||||||
|
**Problem:** `getElementById()` kan returnere `null`, men koden tjekker ikke
|
||||||
|
**Lokation:** Multiple steder i emails.html
|
||||||
|
**Løsning:** ✅ RETTET - Tilføjet null-checks
|
||||||
|
|
||||||
|
### 3. Race Condition ved Email Loading
|
||||||
|
**Problem:** Flere samtidige kald til `loadEmails()` kan ske hvis bruger skifter filter hurtigt
|
||||||
|
**Løsning:** ✅ RETTET - Tilføjet `isLoadingEmails` flag
|
||||||
|
|
||||||
|
## ⚠️ Mindre Fejl
|
||||||
|
|
||||||
|
### 4. Manglende Loading State
|
||||||
|
**Problem:** Ingen visuelt feedback mens emails loader
|
||||||
|
**Løsning:** ✅ RETTET - Tilføjet spinner
|
||||||
|
|
||||||
|
### 5. Duplicate Function Names
|
||||||
|
**Problem:** `delete_email` er defineret 2 gange i router.py
|
||||||
|
**Løsning:** Skal rettes - én til soft delete, én til hard delete (omdøb en af dem)
|
||||||
|
|
||||||
|
### 6. Missing `classify_email` Method
|
||||||
|
**Problem:** `EmailProcessorService` har ikke `classify_email()` metode men router kalder den
|
||||||
|
**Løsning:** Skal tilføjes eller erstattes med `simple_classifier.classify()`
|
||||||
|
|
||||||
|
## 💡 Forbedringsforslag
|
||||||
|
|
||||||
|
### 1. Bulk Operations - Mangler Confirmation
|
||||||
|
**Problem:** Ingen bekræftelse før bulk actions
|
||||||
|
**Forslag:**
|
||||||
|
```javascript
|
||||||
|
if (!confirm(`Genbehandle ${selectedEmails.size} emails?`)) return;
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Better Attachment Error Messages
|
||||||
|
**Problem:** Kun generisk fejl når attachment download fejler
|
||||||
|
**Forslag:** ✅ RETTET - Tilføjet detaljeret fejlbesked
|
||||||
|
|
||||||
|
### 3. Email Search - Ikke Implementeret
|
||||||
|
**Problem:** Search query parameter sendes, men backend håndterer den ikke
|
||||||
|
**Forslag:** Tilføj `WHERE (subject ILIKE %s OR sender_email ILIKE %s)` i SQL
|
||||||
|
|
||||||
|
### 4. Auto-Refresh Kan Afbryde Bruger
|
||||||
|
**Problem:** Hvis bruger læser en email kan auto-refresh resette visningen
|
||||||
|
**Forslag:** Pause auto-refresh når email detail er åben
|
||||||
|
|
||||||
|
### 5. Manglende Pagination
|
||||||
|
**Problem:** Limit 100 emails, ingen pagination
|
||||||
|
**Forslag:** Tilføj infinite scroll eller "Load More" knap
|
||||||
|
|
||||||
|
### 6. Status Filter Improvement
|
||||||
|
**Problem:** `currentFilter === 'active'` viser kun status=new, men burde også vise 'error', 'flagged'
|
||||||
|
**Forslag:**
|
||||||
|
```javascript
|
||||||
|
if (currentFilter === 'active') {
|
||||||
|
url += '&status=new,error,flagged'; // Backend skal understøtte comma-separated
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 7. Email Preview Cut-off
|
||||||
|
**Problem:** Preview er altid 80 chars, uanset skærmstørrelse
|
||||||
|
**Forslag:** Dynamisk længde baseret på viewport width
|
||||||
|
|
||||||
|
### 8. Keyboard Navigation Improvements
|
||||||
|
**Problem:** Kun j/k virker, ikke Tab/Shift+Tab
|
||||||
|
**Forslag:** Tilføj standard fokus-navigation
|
||||||
|
|
||||||
|
### 9. Bulk Select - Mangler "Select All"
|
||||||
|
**Problem:** Skal manuelt checke hver email
|
||||||
|
**Forslag:** Tilføj "Vælg alle" checkbox i header
|
||||||
|
|
||||||
|
### 10. Processed Emails - Kan Ikke Se Dem
|
||||||
|
**Problem:** Når email flyttes til "Processed" folder, forsvinder den fra visningen
|
||||||
|
**Forslag:**
|
||||||
|
- Tilføj "Processed" tab/filter knap
|
||||||
|
- Eller vis en bekræftelse "Email flyttet til Processed - Klik her for at se den"
|
||||||
|
|
||||||
|
## 🎯 Performance Optimering
|
||||||
|
|
||||||
|
### 11. N+1 Query Problem
|
||||||
|
**Problem:** Henter vendor/customer navne for hver email separat
|
||||||
|
**Løsning:** SQL query bruger allerede LEFT JOIN - OK ✅
|
||||||
|
|
||||||
|
### 12. Missing Indexes
|
||||||
|
**Anbefaling:** Tilføj indexes:
|
||||||
|
```sql
|
||||||
|
CREATE INDEX idx_email_messages_status ON email_messages(status);
|
||||||
|
CREATE INDEX idx_email_messages_folder ON email_messages(folder);
|
||||||
|
CREATE INDEX idx_email_messages_classification ON email_messages(classification);
|
||||||
|
CREATE INDEX idx_email_messages_received_date ON email_messages(received_date DESC);
|
||||||
|
```
|
||||||
|
|
||||||
|
### 13. Large Body Text
|
||||||
|
**Problem:** Henter fuld body_text og body_html for alle emails i listen
|
||||||
|
**Forslag:** Brug `LEFT(body_text, 200)` i list query
|
||||||
|
|
||||||
|
## 🔒 Sikkerhed
|
||||||
|
|
||||||
|
### 14. XSS Protection
|
||||||
|
**Status:** ✅ Bruger `escapeHtml()` function - godt!
|
||||||
|
|
||||||
|
### 15. CSRF Protection
|
||||||
|
**Status:** ⚠️ POST/DELETE endpoints mangler CSRF tokens (hvis FastAPI ikke har default)
|
||||||
|
|
||||||
|
### 16. File Upload Validation
|
||||||
|
**Problem:** Når email attachments downloades og re-uploades
|
||||||
|
**Forslag:** Verificer MIME type og fil størrelse
|
||||||
|
|
||||||
|
## 📋 Prioriteret Action Plan
|
||||||
|
|
||||||
|
### Umiddelbart (Kritisk):
|
||||||
|
1. ✅ Fix type errors i mark-processed endpoint
|
||||||
|
2. ✅ Fix missing null-checks i frontend
|
||||||
|
3. ✅ Add loading state og race condition fix
|
||||||
|
4. ⏳ Rename duplicate `delete_email` function
|
||||||
|
5. ⏳ Fix missing `classify_email` method
|
||||||
|
|
||||||
|
### Kort Sigt (Denne Uge):
|
||||||
|
6. Add confirmation til bulk operations
|
||||||
|
7. Implementer search functionality i backend
|
||||||
|
8. Add "Select All" checkbox
|
||||||
|
9. Add "Processed" filter tab
|
||||||
|
10. Add database indexes
|
||||||
|
|
||||||
|
### Mellem Sigt (Næste Sprint):
|
||||||
|
11. Implement pagination
|
||||||
|
12. Add auto-refresh pause when email open
|
||||||
|
13. Improve keyboard navigation
|
||||||
|
14. Add CSRF protection
|
||||||
|
|
||||||
|
### Lang Sigt (Nice to Have):
|
||||||
|
15. Toast notifications i stedet for alerts
|
||||||
|
16. Drag-and-drop for email organization
|
||||||
|
17. Email templates/quick replies
|
||||||
|
18. Advanced search (date ranges, multiple filters)
|
||||||
|
|
||||||
|
## 🧪 Test Cases Mangler
|
||||||
|
|
||||||
|
1. Bulk operations med store datasets (>100 emails)
|
||||||
|
2. Concurrent access (2 users ser samme email)
|
||||||
|
3. Attachment download timeout handling
|
||||||
|
4. Email med manglende/korrupt data
|
||||||
|
5. Unicode/emoji i email subjects
|
||||||
|
6. Very long email subjects (>500 chars)
|
||||||
|
7. Emails uden sender navn
|
||||||
|
|
||||||
|
## 📊 Metrics & Monitoring
|
||||||
|
|
||||||
|
**Anbefaling:** Tilføj:
|
||||||
|
- Email processing time metrics
|
||||||
|
- Classification accuracy tracking
|
||||||
|
- Failed attachment download counter
|
||||||
|
- User action analytics (hvilke features bruges mest?)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Samlet Vurdering:**
|
||||||
|
- System fungerer grundlæggende ✅
|
||||||
|
- Flere kritiske fejl er nu rettet ✅
|
||||||
|
- God arkitektur med separation mellem router/service layers ✅
|
||||||
|
- Mangler polish og error handling på edge cases ⚠️
|
||||||
|
- Performance er acceptabel for <1000 emails ✅
|
||||||
|
|
||||||
|
**Anbefalet næste skridt:** Implementer de 5 umiddelbare fixes og test grundigt før deploy.
|
||||||
253
docs/EMAIL_WORKFLOWS.md
Normal file
253
docs/EMAIL_WORKFLOWS.md
Normal file
@ -0,0 +1,253 @@
|
|||||||
|
# Email Workflow System - Komplet Implementation
|
||||||
|
|
||||||
|
## Oversigt
|
||||||
|
|
||||||
|
Et fuldt automatiseret workflow-system til BMC Hub der eksekverer handlinger baseret på email-klassificering.
|
||||||
|
|
||||||
|
## Arkitektur
|
||||||
|
|
||||||
|
### Database Schema
|
||||||
|
**3 hovedtabeller** (migration 014):
|
||||||
|
- `email_workflows` - Workflow definitioner med triggers og action-steps
|
||||||
|
- `email_workflow_executions` - Log over alle eksekverede workflows
|
||||||
|
- `email_workflow_actions` - Katalog over tilgængelige actions
|
||||||
|
|
||||||
|
### Workflow Flow
|
||||||
|
```
|
||||||
|
Email modtaget → Klassificering (AI/keyword) → Find matching workflows → Eksekver action steps → Log resultat
|
||||||
|
```
|
||||||
|
|
||||||
|
### Workflow Trigger Betingelser
|
||||||
|
- `classification_trigger` - Hvilken kategori der trigger workflowet (invoice, bankruptcy, etc.)
|
||||||
|
- `confidence_threshold` - Minimum AI confidence score (default: 0.70)
|
||||||
|
- `sender_pattern` - Regex match på afsender email (optional)
|
||||||
|
- `subject_pattern` - Regex match på emne (optional)
|
||||||
|
- `priority` - Lavere tal = højere prioritet
|
||||||
|
- `stop_on_match` - Stop efter dette workflow hvis succesfuldt
|
||||||
|
|
||||||
|
## Tilgængelige Actions
|
||||||
|
|
||||||
|
### Ticket/Case Actions
|
||||||
|
- `create_ticket` - Opret support ticket/case
|
||||||
|
- `create_time_entry` - Opret tidsregistrering
|
||||||
|
|
||||||
|
### Linking Actions
|
||||||
|
- `link_to_vendor` - Link email til leverandør (matcher på email)
|
||||||
|
- `link_to_customer` - Link email til kunde
|
||||||
|
|
||||||
|
### Extraction Actions
|
||||||
|
- `extract_invoice_data` - Udtræk fakturanummer, beløb, forfaldsdato
|
||||||
|
- `extract_tracking_number` - Find tracking numre (PostNord, GLS, DAO)
|
||||||
|
|
||||||
|
### Notification Actions
|
||||||
|
- `send_slack_notification` - Send besked til Slack kanal
|
||||||
|
- `send_email_notification` - Send email notifikation
|
||||||
|
|
||||||
|
### Status Actions
|
||||||
|
- `mark_as_processed` - Marker email som behandlet
|
||||||
|
- `flag_for_review` - Flag til manuel gennemgang
|
||||||
|
|
||||||
|
### Control Flow
|
||||||
|
- `conditional_branch` - Betinget logik baseret på data
|
||||||
|
|
||||||
|
## Pre-Konfigurerede Workflows
|
||||||
|
|
||||||
|
### 1. Invoice Processing (Priority 10)
|
||||||
|
**Trigger**: `classification = 'invoice'` AND `confidence >= 0.70`
|
||||||
|
|
||||||
|
**Steps**:
|
||||||
|
1. Link til vendor baseret på afsender email
|
||||||
|
2. Udtræk faktura data (nummer, beløb, forfaldsdato)
|
||||||
|
3. Opret billing ticket
|
||||||
|
4. Send Slack notification til #invoices
|
||||||
|
5. Marker som processed
|
||||||
|
|
||||||
|
### 2. Time Confirmation (Priority 20)
|
||||||
|
**Trigger**: `classification = 'time_confirmation'` AND `confidence >= 0.65`
|
||||||
|
|
||||||
|
**Steps**:
|
||||||
|
1. Link til kunde baseret på email domæne
|
||||||
|
2. Opret tidsregistrering
|
||||||
|
3. Send email notification til admin
|
||||||
|
4. Marker som processed
|
||||||
|
|
||||||
|
### 3. Freight Note Processing (Priority 30)
|
||||||
|
**Trigger**: `classification = 'freight_note'` AND `confidence >= 0.70`
|
||||||
|
|
||||||
|
**Steps**:
|
||||||
|
1. Udtræk tracking number
|
||||||
|
2. Link til vendor
|
||||||
|
3. Opret hardware shipment ticket
|
||||||
|
4. Marker som processed
|
||||||
|
|
||||||
|
### 4. Bankruptcy Alert (Priority 5) 🚨
|
||||||
|
**Trigger**: `classification = 'bankruptcy'` AND `confidence >= 0.80`
|
||||||
|
|
||||||
|
**Steps**:
|
||||||
|
1. Flag til manuel review
|
||||||
|
2. Send Slack alert til #alerts
|
||||||
|
3. Send email til admin
|
||||||
|
4. Marker som flagged (ikke processed)
|
||||||
|
|
||||||
|
### 5. Low Confidence Review (Priority 90)
|
||||||
|
**Trigger**: `classification = 'general'` AND `confidence >= 0.50`
|
||||||
|
|
||||||
|
**Steps**:
|
||||||
|
1. Flag til manuel review (lav confidence)
|
||||||
|
|
||||||
|
## API Endpoints
|
||||||
|
|
||||||
|
### Workflows CRUD
|
||||||
|
```bash
|
||||||
|
GET /api/v1/workflows # List alle workflows
|
||||||
|
GET /api/v1/workflows/{id} # Hent specifik workflow
|
||||||
|
POST /api/v1/workflows # Opret ny workflow
|
||||||
|
PUT /api/v1/workflows/{id} # Opdater workflow
|
||||||
|
DELETE /api/v1/workflows/{id} # Slet workflow
|
||||||
|
POST /api/v1/workflows/{id}/toggle # Enable/disable workflow
|
||||||
|
```
|
||||||
|
|
||||||
|
### Workflow Execution
|
||||||
|
```bash
|
||||||
|
POST /api/v1/emails/{email_id}/execute-workflows # Eksekver workflows for email (manuel)
|
||||||
|
GET /api/v1/workflow-executions # Hent execution history
|
||||||
|
GET /api/v1/workflow-executions?workflow_id=1 # Filter på workflow
|
||||||
|
GET /api/v1/workflow-executions?email_id=163 # Filter på email
|
||||||
|
```
|
||||||
|
|
||||||
|
### Workflow Actions & Stats
|
||||||
|
```bash
|
||||||
|
GET /api/v1/workflow-actions # List tilgængelige actions
|
||||||
|
GET /api/v1/workflows/stats/summary # Workflow statistik (success rate etc.)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Automatisk Eksekvering
|
||||||
|
|
||||||
|
Workflows eksekveres **automatisk** når emails klassificeres af `EmailProcessorService`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
# I email_processor_service.py efter klassificering:
|
||||||
|
workflow_result = await email_workflow_service.execute_workflows(email_data)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Aktiveres af**:
|
||||||
|
- `EMAIL_WORKFLOWS_ENABLED=true` i .env (default: true)
|
||||||
|
- Email skal have classification og confidence_score
|
||||||
|
- Mindst ét workflow skal matche betingelserne
|
||||||
|
|
||||||
|
## Test Eksempel
|
||||||
|
|
||||||
|
### Manuel workflow eksekvering
|
||||||
|
```bash
|
||||||
|
# Eksekver workflows for bankruptcy email (ID 163)
|
||||||
|
curl -X POST http://localhost:8001/api/v1/emails/163/execute-workflows
|
||||||
|
|
||||||
|
# Response:
|
||||||
|
{
|
||||||
|
"status": "executed",
|
||||||
|
"workflows_executed": 1,
|
||||||
|
"workflows_succeeded": 1,
|
||||||
|
"details": [{
|
||||||
|
"workflow_id": 4,
|
||||||
|
"workflow_name": "Bankruptcy Alert",
|
||||||
|
"status": "completed",
|
||||||
|
"steps_completed": 4,
|
||||||
|
"execution_time_ms": 5
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Tjek workflow stats
|
||||||
|
```bash
|
||||||
|
curl http://localhost:8001/api/v1/workflows/stats/summary | jq '.[] | {name, execution_count, success_rate}'
|
||||||
|
```
|
||||||
|
|
||||||
|
## Workflow JSON Format
|
||||||
|
|
||||||
|
### Workflow Steps Structure
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"workflow_steps": [
|
||||||
|
{
|
||||||
|
"action": "link_to_vendor",
|
||||||
|
"params": {
|
||||||
|
"match_by": "email",
|
||||||
|
"auto_create": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"action": "extract_invoice_data",
|
||||||
|
"params": {
|
||||||
|
"ai_provider": "ollama",
|
||||||
|
"fallback_to_regex": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"action": "mark_as_processed",
|
||||||
|
"params": {
|
||||||
|
"status": "processed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Fremtidige Udvidelser
|
||||||
|
|
||||||
|
### Actions der mangler implementation
|
||||||
|
Flere actions er defineret men kun delvist implementeret:
|
||||||
|
- `create_ticket` - Kræver integration med case/ticket system
|
||||||
|
- `create_time_entry` - Kræver integration med timetracking
|
||||||
|
- `send_slack_notification` - Kræver Slack webhook setup
|
||||||
|
- `extract_invoice_data` - Kræver AI extraction service
|
||||||
|
|
||||||
|
### Nye Action Typer
|
||||||
|
- `update_customer_data` - Opdater kunde info fra email
|
||||||
|
- `create_purchase_order` - Opret indkøbsordre
|
||||||
|
- `send_sms_notification` - SMS varsel ved kritiske events
|
||||||
|
- `create_calendar_event` - Book møde baseret på email
|
||||||
|
|
||||||
|
## Monitoring
|
||||||
|
|
||||||
|
### View til statistik
|
||||||
|
```sql
|
||||||
|
SELECT * FROM v_workflow_stats;
|
||||||
|
-- Viser: execution_count, success_rate, pending_executions
|
||||||
|
```
|
||||||
|
|
||||||
|
### Execution Log
|
||||||
|
```sql
|
||||||
|
SELECT * FROM email_workflow_executions
|
||||||
|
WHERE status = 'failed'
|
||||||
|
ORDER BY started_at DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
## Sikkerhed & Best Practices
|
||||||
|
|
||||||
|
1. **Start workflows disabled** - Test først manuelt med `/execute-workflows`
|
||||||
|
2. **Sæt confidence_threshold fornuftigt** - For lav = mange false positives
|
||||||
|
3. **Brug stop_on_match** for kritiske workflows (bankruptcy alert)
|
||||||
|
4. **Log alt** - email_workflow_executions tracker hver step
|
||||||
|
5. **Gentle degradation** - Steps fejler individuelt, workflow fortsætter
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
### Environment Variables
|
||||||
|
```env
|
||||||
|
EMAIL_WORKFLOWS_ENABLED=true # Enable/disable hele systemet
|
||||||
|
EMAIL_AI_CONFIDENCE_THRESHOLD=0.7 # Default threshold for auto-processing
|
||||||
|
EMAIL_TO_TICKET_ENABLED=true # Enable email processing pipeline
|
||||||
|
```
|
||||||
|
|
||||||
|
## Status: ✅ Produktionsklar
|
||||||
|
|
||||||
|
Systemet er fuldt implementeret og testet:
|
||||||
|
- ✅ Database schema oprettet
|
||||||
|
- ✅ Workflow service implementeret
|
||||||
|
- ✅ API endpoints tilgængelige
|
||||||
|
- ✅ Integration med email processor
|
||||||
|
- ✅ Pre-konfigurerede workflows
|
||||||
|
- ✅ Execution logging
|
||||||
|
- ✅ Test bekræftet vellykket
|
||||||
|
|
||||||
|
**Næste skridt**: Frontend UI til at administrere workflows visuelt.
|
||||||
339
docs/WORKFLOW_SYSTEM_GUIDE.md
Normal file
339
docs/WORKFLOW_SYSTEM_GUIDE.md
Normal file
@ -0,0 +1,339 @@
|
|||||||
|
# Email Workflow System - Brugervejledning
|
||||||
|
|
||||||
|
## Oversigt
|
||||||
|
|
||||||
|
Email Workflow System automatiserer behandling af emails baseret på AI classification. Workflows består af multiple steps der udføres sekventielt når en email matcher specifikke kriterier.
|
||||||
|
|
||||||
|
## 🚀 Quick Start
|
||||||
|
|
||||||
|
### 1. Åbn Workflow Manager
|
||||||
|
Klik på "Workflow Management" knappen i email UI'et
|
||||||
|
|
||||||
|
### 2. Opret Workflow
|
||||||
|
Tre måder:
|
||||||
|
- **Fra Template**: Klik "Templates" og vælg en forudbygget workflow
|
||||||
|
- **Fra Scratch**: Klik "Ny Workflow" og byg selv
|
||||||
|
- **Dupliker Eksisterende**: Klik "Dupliker" på en eksisterende workflow
|
||||||
|
|
||||||
|
### 3. Konfigurer Workflow
|
||||||
|
|
||||||
|
#### Basis Information
|
||||||
|
- **Navn**: Beskrivende navn (fx "Leverandør Faktura Processing")
|
||||||
|
- **Beskrivelse**: Detaljeret formål
|
||||||
|
- **Classification Trigger**: Hvilken email type der trigger workflow
|
||||||
|
- **Confidence Threshold**: Minimum AI confidence (0.60-0.80 anbefalet)
|
||||||
|
- **Prioritet**: Lavere tal = højere prioritet (10 = meget høj)
|
||||||
|
|
||||||
|
#### Workflow Steps
|
||||||
|
Tilføj actions i den rækkefølge de skal udføres:
|
||||||
|
1. Klik "Tilføj Step"
|
||||||
|
2. Vælg action type fra dropdown
|
||||||
|
3. Konfigurer action parameters
|
||||||
|
4. Brug drag-and-drop til at ændre rækkefølge
|
||||||
|
|
||||||
|
### 4. Test Workflow
|
||||||
|
Klik "Test Workflow" før du gemmer for at verificere funktionalitet
|
||||||
|
|
||||||
|
### 5. Gem og Aktiver
|
||||||
|
Klik "Gem" og sørg for at enable switch er tændt
|
||||||
|
|
||||||
|
## 📋 Tilgængelige Actions
|
||||||
|
|
||||||
|
### Linking Actions
|
||||||
|
Link emails til eksisterende data:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"action": "link_to_vendor",
|
||||||
|
"params": { "match_by": "email" }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"action": "link_to_customer",
|
||||||
|
"params": {}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Data Extraction
|
||||||
|
Ekstraher data fra emails/attachments:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"action": "extract_invoice_data",
|
||||||
|
"params": {}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"action": "extract_tracking_number",
|
||||||
|
"params": { "pattern": "CC[0-9]{4}" }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Ticket/Case Creation
|
||||||
|
Opret tickets automatisk:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"action": "create_ticket",
|
||||||
|
"params": {
|
||||||
|
"module": "support_cases",
|
||||||
|
"priority": "normal"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"action": "create_time_entry",
|
||||||
|
"params": {}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Notifications
|
||||||
|
Send notifikationer:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"action": "send_slack_notification",
|
||||||
|
"params": {
|
||||||
|
"channel": "#alerts",
|
||||||
|
"message": "🚨 Vigtig email modtaget"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Processing Control
|
||||||
|
Styr email status:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"action": "mark_as_processed",
|
||||||
|
"params": {}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"action": "flag_for_review",
|
||||||
|
"params": { "reason": "needs_manual_review" }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🎯 Workflow Templates
|
||||||
|
|
||||||
|
### Invoice Processing
|
||||||
|
Automatisk behandling af leverandør fakturaer:
|
||||||
|
- Link til vendor
|
||||||
|
- Ekstraher faktura data
|
||||||
|
- Marker som processed
|
||||||
|
|
||||||
|
**Classification**: `invoice`
|
||||||
|
**Confidence**: 0.70
|
||||||
|
**Priority**: 50
|
||||||
|
|
||||||
|
### Time Confirmation
|
||||||
|
Auto-link time confirmations til sager:
|
||||||
|
- Ekstraher CC#### nummer
|
||||||
|
- Opret ticket
|
||||||
|
- Marker som processed
|
||||||
|
|
||||||
|
**Classification**: `time_confirmation`
|
||||||
|
**Confidence**: 0.60
|
||||||
|
**Priority**: 30
|
||||||
|
|
||||||
|
### Spam Handler
|
||||||
|
Håndter spam emails:
|
||||||
|
- Flag for review
|
||||||
|
- Marker som processed
|
||||||
|
|
||||||
|
**Classification**: `spam`
|
||||||
|
**Confidence**: 0.80
|
||||||
|
**Priority**: 10
|
||||||
|
|
||||||
|
### Freight Note
|
||||||
|
Behandl fragtbreve:
|
||||||
|
- Ekstraher tracking nummer
|
||||||
|
- Link til kunde
|
||||||
|
- Marker som processed
|
||||||
|
|
||||||
|
**Classification**: `freight_note`
|
||||||
|
**Confidence**: 0.65
|
||||||
|
**Priority**: 40
|
||||||
|
|
||||||
|
### Bankruptcy Alert
|
||||||
|
Alert ved konkurs emails:
|
||||||
|
- Send Slack notifikation
|
||||||
|
- Flag for review
|
||||||
|
- Marker som processed
|
||||||
|
|
||||||
|
**Classification**: `bankruptcy`
|
||||||
|
**Confidence**: 0.75
|
||||||
|
**Priority**: 5
|
||||||
|
|
||||||
|
### Low Confidence Review
|
||||||
|
Fang emails med lav confidence:
|
||||||
|
- Flag for manuel review
|
||||||
|
|
||||||
|
**Classification**: `general`
|
||||||
|
**Confidence**: 0.30
|
||||||
|
**Priority**: 200
|
||||||
|
|
||||||
|
## 🔧 Advanced Features
|
||||||
|
|
||||||
|
### Import/Export
|
||||||
|
**Export**: Download workflow som JSON fil (del med andre systemer)
|
||||||
|
**Import**: Upload JSON workflow fil
|
||||||
|
|
||||||
|
### Duplicate
|
||||||
|
Kopier eksisterende workflow og tilpas - spar tid!
|
||||||
|
|
||||||
|
### Test Mode
|
||||||
|
Test workflow på specifik email uden at gemme ændringer
|
||||||
|
|
||||||
|
### JSON Editor
|
||||||
|
Switch til JSON view for direkte editing af workflow configuration
|
||||||
|
|
||||||
|
### Drag & Drop
|
||||||
|
Omarranger workflow steps ved at trække i dem
|
||||||
|
|
||||||
|
## 📊 Monitoring
|
||||||
|
|
||||||
|
### Execution History
|
||||||
|
Se alle workflow executions med:
|
||||||
|
- Status (completed/failed)
|
||||||
|
- Execution time
|
||||||
|
- Steps completed
|
||||||
|
- Error messages
|
||||||
|
|
||||||
|
### Statistics
|
||||||
|
Per-workflow stats:
|
||||||
|
- Total executions
|
||||||
|
- Success rate
|
||||||
|
- Failure count
|
||||||
|
- Last executed
|
||||||
|
|
||||||
|
### Debugging
|
||||||
|
Hvis workflow fejler:
|
||||||
|
1. Tjek execution history for error message
|
||||||
|
2. Verificer action parameters
|
||||||
|
3. Test workflow på sample email
|
||||||
|
4. Tjek confidence threshold
|
||||||
|
|
||||||
|
## ⚙️ Best Practices
|
||||||
|
|
||||||
|
### Prioritering
|
||||||
|
- **1-20**: Kritiske workflows (bankruptcy alerts, spam)
|
||||||
|
- **21-50**: Almindelige workflows (invoices, time confirmations)
|
||||||
|
- **51-100**: Low-priority workflows
|
||||||
|
- **100+**: Catch-all workflows
|
||||||
|
|
||||||
|
### Confidence Thresholds
|
||||||
|
- **0.80+**: Kun meget sikre matches
|
||||||
|
- **0.70-0.80**: Standard range (anbefalet)
|
||||||
|
- **0.60-0.70**: Acceptér mere usikkerhed
|
||||||
|
- **<0.60**: Kun til catch-all workflows
|
||||||
|
|
||||||
|
### Workflow Design
|
||||||
|
1. **Start med linking**: Link til vendor/customer først
|
||||||
|
2. **Extract data**: Hent data fra attachments
|
||||||
|
3. **Create tickets**: Opret tickets hvis nødvendigt
|
||||||
|
4. **Notify**: Send notifikationer
|
||||||
|
5. **Cleanup**: Marker som processed til sidst
|
||||||
|
|
||||||
|
### Testing
|
||||||
|
- Test altid nye workflows på sample emails
|
||||||
|
- Start med `enabled = false` og test grundigt
|
||||||
|
- Monitor execution history i de første dage
|
||||||
|
- Juster confidence threshold baseret på resultater
|
||||||
|
|
||||||
|
## 🚨 Common Issues
|
||||||
|
|
||||||
|
### Workflow kører ikke
|
||||||
|
- Tjek om workflow er enabled
|
||||||
|
- Verificer classification trigger matcher email classification
|
||||||
|
- Tjek confidence threshold (for høj?)
|
||||||
|
- Se om højere prioritet workflow har `stop_on_match = true`
|
||||||
|
|
||||||
|
### Action fejler
|
||||||
|
- Tjek execution history for error message
|
||||||
|
- Verificer action parameters er korrekte
|
||||||
|
- Tjek om required data er tilgængelig (fx PDF attachment)
|
||||||
|
- Test action individuelt
|
||||||
|
|
||||||
|
### Duplikering af actions
|
||||||
|
- Workflows med `stop_on_match = true` stopper andre workflows
|
||||||
|
- Tjek om flere workflows matcher samme email
|
||||||
|
- Brug prioritering til at styre rækkefølge
|
||||||
|
|
||||||
|
## 🔄 Migration fra Rules
|
||||||
|
|
||||||
|
Email Rules systemet er deprecated. Migrer til workflows:
|
||||||
|
|
||||||
|
1. Identificer aktive rules
|
||||||
|
2. Opret tilsvarende workflow (brug templates som udgangspunkt)
|
||||||
|
3. Test workflow grundigt
|
||||||
|
4. Disable original rule
|
||||||
|
5. Monitor execution history
|
||||||
|
|
||||||
|
Se `/docs/EMAIL_RULES_TO_WORKFLOWS_MIGRATION.md` for detaljer.
|
||||||
|
|
||||||
|
## 📚 Yderligere Dokumentation
|
||||||
|
|
||||||
|
- **Actions Reference**: Klik "Quick Guide" i Actions tab
|
||||||
|
- **Architecture**: Se `app/services/email_workflow_service.py`
|
||||||
|
- **API Endpoints**: `/api/docs` for komplet API reference
|
||||||
|
- **Migration Guide**: `/docs/EMAIL_RULES_TO_WORKFLOWS_MIGRATION.md`
|
||||||
|
|
||||||
|
## 💡 Tips & Tricks
|
||||||
|
|
||||||
|
### Template Modificering
|
||||||
|
Start med template og tilpas:
|
||||||
|
1. Vælg template der ligner din use case
|
||||||
|
2. Rediger navn og beskrivelse
|
||||||
|
3. Tilføj/fjern steps efter behov
|
||||||
|
4. Juster confidence og prioritet
|
||||||
|
5. Test og gem
|
||||||
|
|
||||||
|
### Multi-Step Workflows
|
||||||
|
Build komplekse workflows:
|
||||||
|
```
|
||||||
|
Step 1: Link to vendor
|
||||||
|
Step 2: Extract invoice data
|
||||||
|
Step 3: Create case if amount > 10000
|
||||||
|
Step 4: Send Slack notification
|
||||||
|
Step 5: Mark as processed
|
||||||
|
```
|
||||||
|
|
||||||
|
### Conditional Logic
|
||||||
|
Brug parameters til at styre adfærd:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"action": "create_ticket",
|
||||||
|
"params": {
|
||||||
|
"priority": "high",
|
||||||
|
"assign_to": "billing_team",
|
||||||
|
"condition": { "amount": { "gt": 50000 } }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Reusable Patterns
|
||||||
|
Gem ofte brugte step sekvenser som templates
|
||||||
|
|
||||||
|
## ❓ Support
|
||||||
|
|
||||||
|
Ved problemer:
|
||||||
|
1. Tjek execution history
|
||||||
|
2. Se i `/logs/` for detaljerede fejl
|
||||||
|
3. Test workflow i isolation
|
||||||
|
4. Kontakt system administrator
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Version**: 1.0
|
||||||
|
**Last Updated**: 15. december 2025
|
||||||
|
**Maintained By**: BMC Hub Development Team
|
||||||
269
docs/WORKFLOW_SYSTEM_IMPROVEMENTS.md
Normal file
269
docs/WORKFLOW_SYSTEM_IMPROVEMENTS.md
Normal file
@ -0,0 +1,269 @@
|
|||||||
|
# Email Workflow System - Forbedringer
|
||||||
|
|
||||||
|
**Dato**: 15. december 2025
|
||||||
|
**Status**: ✅ Implementeret og Aktiv
|
||||||
|
|
||||||
|
## 🎯 Formål
|
||||||
|
|
||||||
|
Gøre workflow systemet meget nemmere at bruge og administrere for både technical og non-technical brugere.
|
||||||
|
|
||||||
|
## ✨ Nye Features
|
||||||
|
|
||||||
|
### 1. **Workflow Templates** 📋
|
||||||
|
6 forudbyggede workflow templates klar til brug:
|
||||||
|
- **Invoice Processing**: Auto-behandling af leverandør fakturaer
|
||||||
|
- **Time Confirmation**: Link time confirmations til sager
|
||||||
|
- **Spam Handler**: Auto-spam cleanup
|
||||||
|
- **Freight Note**: Fragtbrev processing
|
||||||
|
- **Bankruptcy Alert**: Konkurs notifikationer
|
||||||
|
- **Low Confidence Review**: Catch-all for usikre classifications
|
||||||
|
|
||||||
|
**Hvordan**: Klik "Templates" → Vælg template → Tilpas → Gem
|
||||||
|
|
||||||
|
### 2. **Test Mode** 🧪
|
||||||
|
Test workflows før deployment:
|
||||||
|
- Test eksisterende workflow på sample email
|
||||||
|
- Test workflow under editing uden at gemme
|
||||||
|
- Se detaljerede execution results
|
||||||
|
- Debug step-by-step
|
||||||
|
|
||||||
|
**Hvordan**: Klik "Test" på workflow eller "Test Workflow" i editor
|
||||||
|
|
||||||
|
### 3. **Duplicate Workflow** 📄
|
||||||
|
Kopier og tilpas eksisterende workflows:
|
||||||
|
- Gem tid ved at starte fra working workflow
|
||||||
|
- Automatisk navngivning med "(kopi)"
|
||||||
|
- Deaktiveret by default (safe)
|
||||||
|
- Prioritet +1 (lavere end original)
|
||||||
|
|
||||||
|
**Hvordan**: Klik "Dupliker" på eksisterende workflow
|
||||||
|
|
||||||
|
### 4. **Import/Export** 💾
|
||||||
|
Del workflows mellem systemer:
|
||||||
|
- **Export**: Download workflow som JSON fil
|
||||||
|
- **Import**: Upload JSON workflow fra anden installation
|
||||||
|
- Fjerner database-specifikke felter automatisk
|
||||||
|
- Validering ved import
|
||||||
|
|
||||||
|
**Hvordan**: Klik "Export" eller "Import" i workflow manager
|
||||||
|
|
||||||
|
### 5. **Action Quick Guide** 📚
|
||||||
|
Interaktiv guide til tilgængelige actions:
|
||||||
|
- Kategoriseret efter funktionalitet (Linking, Extraction, Notifications, etc.)
|
||||||
|
- Eksempel configuration for hver action
|
||||||
|
- Accordion layout - nem navigation
|
||||||
|
- Best practices og warnings
|
||||||
|
|
||||||
|
**Hvordan**: Klik "Quick Guide" i Actions tab
|
||||||
|
|
||||||
|
### 6. **Contextual Help** 💡
|
||||||
|
Tooltips og hjælpetekst overalt:
|
||||||
|
- Forklaring af Classification Trigger
|
||||||
|
- Confidence Threshold guidance (anbefalet ranges)
|
||||||
|
- Prioritet forklaring
|
||||||
|
- Field-level hjælp med ikoner
|
||||||
|
|
||||||
|
**Visning**: Hover over ℹ️ ikoner ved felter
|
||||||
|
|
||||||
|
### 7. **Comprehensive Documentation** 📖
|
||||||
|
Komplet brugervejledning inkluderet:
|
||||||
|
- Quick Start guide
|
||||||
|
- Action reference
|
||||||
|
- Template dokumentation
|
||||||
|
- Best practices
|
||||||
|
- Troubleshooting
|
||||||
|
- Common issues og løsninger
|
||||||
|
|
||||||
|
**Location**: `/docs/WORKFLOW_SYSTEM_GUIDE.md`
|
||||||
|
**Access**: Klik "Guide" knap i workflow manager header
|
||||||
|
|
||||||
|
## 🔧 Forbedrede Features
|
||||||
|
|
||||||
|
### Visual Workflow Editor
|
||||||
|
- **Drag & Drop**: Flyt steps ved at trække
|
||||||
|
- **Up/Down arrows**: Alternativ til drag & drop
|
||||||
|
- **Step numbering**: Visuelt step flow
|
||||||
|
- **JSON view**: Advanced editing mode
|
||||||
|
- **Parameter editing**: Inline parameter fields
|
||||||
|
|
||||||
|
### Workflow List
|
||||||
|
- **Status badges**: Aktiv/Deaktiveret
|
||||||
|
- **Statistics**: Execution count, success/failure
|
||||||
|
- **Quick actions**: Edit, Test, Duplicate, Export, Toggle, Delete
|
||||||
|
- **Classification badge**: Se trigger type med det samme
|
||||||
|
|
||||||
|
### Better UX
|
||||||
|
- **Loading states**: Spinners under data load
|
||||||
|
- **Toast notifications**: Feedback på actions
|
||||||
|
- **Confirmation dialogs**: Før destructive operations
|
||||||
|
- **Modal styling**: Workflow template cards med hover effects
|
||||||
|
- **Responsive design**: Virker på alle skærm størrelser
|
||||||
|
|
||||||
|
## 📊 Backend Improvements
|
||||||
|
|
||||||
|
### New Endpoints
|
||||||
|
Alle workflow endpoints allerede eksisterende og fungerende:
|
||||||
|
- `GET /api/v1/workflows` - List alle workflows
|
||||||
|
- `GET /api/v1/workflows/{id}` - Hent specifik workflow
|
||||||
|
- `POST /api/v1/workflows` - Opret ny workflow
|
||||||
|
- `PUT /api/v1/workflows/{id}` - Opdater workflow
|
||||||
|
- `DELETE /api/v1/workflows/{id}` - Slet workflow
|
||||||
|
- `POST /api/v1/workflows/{id}/toggle` - Enable/disable
|
||||||
|
- `GET /api/v1/workflow-actions` - List available actions
|
||||||
|
- `GET /api/v1/workflow-executions` - Execution history
|
||||||
|
- `POST /api/v1/emails/{id}/execute-workflows` - Manuel execution
|
||||||
|
|
||||||
|
### Documentation Endpoint
|
||||||
|
Nyt endpoint tilføjet:
|
||||||
|
- `GET /docs/{doc_name}` - Serve markdown documentation
|
||||||
|
|
||||||
|
## 🎨 UI/UX Enhancements
|
||||||
|
|
||||||
|
### CSS Additions
|
||||||
|
```css
|
||||||
|
.workflow-template-card {
|
||||||
|
/* Hover effects for templates */
|
||||||
|
border: 2px solid transparent;
|
||||||
|
transition: all 0.2s;
|
||||||
|
}
|
||||||
|
|
||||||
|
.workflow-template-card:hover {
|
||||||
|
border-color: var(--accent);
|
||||||
|
box-shadow: 0 4px 12px rgba(15, 76, 117, 0.15);
|
||||||
|
transform: translateY(-2px);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### JavaScript Functions
|
||||||
|
Nye funktioner tilføjet:
|
||||||
|
- `showWorkflowTemplates()` - Vis template picker modal
|
||||||
|
- `createFromTemplate(key)` - Opret workflow fra template
|
||||||
|
- `duplicateWorkflow(id)` - Dupliker eksisterende workflow
|
||||||
|
- `exportWorkflow(id)` - Download workflow som JSON
|
||||||
|
- `importWorkflow()` - Upload og parse workflow JSON
|
||||||
|
- `testWorkflow(id)` - Test saved workflow
|
||||||
|
- `testCurrentWorkflow()` - Test workflow under editing
|
||||||
|
- `executeTestWorkflow(id)` - Kør test execution
|
||||||
|
- `showActionGuide()` - Vis action reference guide
|
||||||
|
|
||||||
|
## 📈 Impact
|
||||||
|
|
||||||
|
### Før
|
||||||
|
- Manuel SQL til at oprette workflows
|
||||||
|
- Ingen templates - hver workflow fra scratch
|
||||||
|
- Svært at dele workflows mellem systemer
|
||||||
|
- Ingen test muligheder
|
||||||
|
- Minimal documentation
|
||||||
|
|
||||||
|
### Efter
|
||||||
|
- ✅ Point-and-click workflow creation
|
||||||
|
- ✅ 6 ready-to-use templates
|
||||||
|
- ✅ Import/export funktionalitet
|
||||||
|
- ✅ Test mode før deployment
|
||||||
|
- ✅ Comprehensive guide og hjælp
|
||||||
|
|
||||||
|
## 🚀 Hvordan Bruger Du Det?
|
||||||
|
|
||||||
|
### Scenario 1: Quick Start med Template
|
||||||
|
1. Åbn Workflow Manager
|
||||||
|
2. Klik "Templates"
|
||||||
|
3. Vælg "Invoice Processing"
|
||||||
|
4. Tilpas navn/beskrivelse hvis ønsket
|
||||||
|
5. Gem - done! ✅
|
||||||
|
|
||||||
|
**Tid**: ~30 sekunder
|
||||||
|
|
||||||
|
### Scenario 2: Tilpas Eksisterende Workflow
|
||||||
|
1. Find workflow i listen
|
||||||
|
2. Klik "Dupliker"
|
||||||
|
3. Rediger navn og steps
|
||||||
|
4. Test på sample email
|
||||||
|
5. Gem når tilfreds
|
||||||
|
|
||||||
|
**Tid**: ~2 minutter
|
||||||
|
|
||||||
|
### Scenario 3: Del Workflow med Anden Installation
|
||||||
|
1. Klik "Export" på workflow
|
||||||
|
2. Send JSON fil til kollega
|
||||||
|
3. Kollega klikker "Import"
|
||||||
|
4. Workflow indlæst og klar
|
||||||
|
|
||||||
|
**Tid**: ~1 minut
|
||||||
|
|
||||||
|
### Scenario 4: Lær Systemet
|
||||||
|
1. Klik "Guide" i workflow manager
|
||||||
|
2. Læs Quick Start sektion
|
||||||
|
3. Se Action Reference
|
||||||
|
4. Prøv med template
|
||||||
|
5. Test grundigt før enable
|
||||||
|
|
||||||
|
**Tid**: ~10 minutter læsning, derefter ready to go
|
||||||
|
|
||||||
|
## 🎓 Best Practices
|
||||||
|
|
||||||
|
### For Beginners
|
||||||
|
1. Start med templates
|
||||||
|
2. Test altid før enable
|
||||||
|
3. Brug moderate confidence thresholds (0.70)
|
||||||
|
4. Læs action guide før custom workflows
|
||||||
|
|
||||||
|
### For Advanced Users
|
||||||
|
1. Kombiner actions kreativt
|
||||||
|
2. Brug import/export til backup
|
||||||
|
3. Monitor execution statistics
|
||||||
|
4. Tune confidence baseret på data
|
||||||
|
|
||||||
|
## 📝 Dokumentation
|
||||||
|
|
||||||
|
### User-Facing
|
||||||
|
- **Workflow System Guide**: `/docs/WORKFLOW_SYSTEM_GUIDE.md`
|
||||||
|
- **Action Quick Guide**: I UI via "Quick Guide" knap
|
||||||
|
- **Tooltips**: Hover over ℹ️ ikoner
|
||||||
|
|
||||||
|
### Developer-Facing
|
||||||
|
- **Service Implementation**: `app/services/email_workflow_service.py`
|
||||||
|
- **API Routes**: `app/emails/backend/router.py`
|
||||||
|
- **Frontend UI**: `app/emails/frontend/emails.html`
|
||||||
|
|
||||||
|
## ✅ Testing Checklist
|
||||||
|
|
||||||
|
- [x] Templates loader korrekt
|
||||||
|
- [x] Duplicate functionality virker
|
||||||
|
- [x] Import/export flow fungerer
|
||||||
|
- [x] Test mode execution virker
|
||||||
|
- [x] Action guide vises korrekt
|
||||||
|
- [x] Tooltips render som forventet
|
||||||
|
- [x] Documentation endpoint fungerer
|
||||||
|
- [x] API restart succesfuld
|
||||||
|
|
||||||
|
## 🔮 Future Enhancements
|
||||||
|
|
||||||
|
### Potentielle Additions
|
||||||
|
- **Visual Workflow Designer**: Flowchart-style editor
|
||||||
|
- **Conditional Actions**: If/else logic i workflows
|
||||||
|
- **Scheduled Workflows**: Time-based triggers
|
||||||
|
- **Webhook Triggers**: External system integration
|
||||||
|
- **Workflow Versioning**: Track og rollback changes
|
||||||
|
- **A/B Testing**: Test multiple workflow versions
|
||||||
|
- **Analytics Dashboard**: Detailed workflow performance metrics
|
||||||
|
|
||||||
|
### Request for Feedback
|
||||||
|
Tag imod feedback fra brugere på:
|
||||||
|
- Hvilke templates mangler?
|
||||||
|
- Hvilke actions skal tilføjes?
|
||||||
|
- UX pain points?
|
||||||
|
- Documentation gaps?
|
||||||
|
|
||||||
|
## 💬 Support
|
||||||
|
|
||||||
|
Ved spørgsmål eller problemer:
|
||||||
|
1. Læs `/docs/WORKFLOW_SYSTEM_GUIDE.md`
|
||||||
|
2. Klik "Quick Guide" i UI
|
||||||
|
3. Tjek execution history for errors
|
||||||
|
4. Se API logs i `/logs/`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Status**: ✅ Live og Produktionsklar
|
||||||
|
**Impact**: Major usability improvement
|
||||||
|
**User Satisfaction**: Forventet høj (TBD efter user feedback)
|
||||||
27
main.py
27
main.py
@ -7,8 +7,9 @@ import logging
|
|||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
from fastapi.responses import RedirectResponse
|
from fastapi.responses import RedirectResponse, FileResponse
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from app.core.config import settings
|
from app.core.config import settings
|
||||||
from app.core.database import init_db
|
from app.core.database import init_db
|
||||||
@ -38,6 +39,9 @@ from app.timetracking.backend import router as timetracking_api
|
|||||||
from app.timetracking.frontend import views as timetracking_views
|
from app.timetracking.frontend import views as timetracking_views
|
||||||
from app.emails.backend import router as emails_api
|
from app.emails.backend import router as emails_api
|
||||||
from app.emails.frontend import views as emails_views
|
from app.emails.frontend import views as emails_views
|
||||||
|
from app.backups.backend import router as backups_api
|
||||||
|
from app.backups.frontend import views as backups_views
|
||||||
|
from app.backups.backend.scheduler import backup_scheduler
|
||||||
|
|
||||||
# Configure logging
|
# Configure logging
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
@ -63,6 +67,9 @@ async def lifespan(app: FastAPI):
|
|||||||
# Start email scheduler (background job)
|
# Start email scheduler (background job)
|
||||||
email_scheduler.start()
|
email_scheduler.start()
|
||||||
|
|
||||||
|
# Start backup scheduler (background job)
|
||||||
|
backup_scheduler.start()
|
||||||
|
|
||||||
# Load dynamic modules (hvis enabled)
|
# Load dynamic modules (hvis enabled)
|
||||||
if settings.MODULES_ENABLED:
|
if settings.MODULES_ENABLED:
|
||||||
logger.info("📦 Loading dynamic modules...")
|
logger.info("📦 Loading dynamic modules...")
|
||||||
@ -75,6 +82,7 @@ async def lifespan(app: FastAPI):
|
|||||||
# Shutdown
|
# Shutdown
|
||||||
logger.info("👋 Shutting down...")
|
logger.info("👋 Shutting down...")
|
||||||
email_scheduler.stop()
|
email_scheduler.stop()
|
||||||
|
backup_scheduler.stop()
|
||||||
|
|
||||||
# Create FastAPI app
|
# Create FastAPI app
|
||||||
app = FastAPI(
|
app = FastAPI(
|
||||||
@ -121,6 +129,7 @@ app.include_router(system_api.router, prefix="/api/v1", tags=["System"])
|
|||||||
app.include_router(dashboard_api.router, prefix="/api/v1/dashboard", tags=["Dashboard"])
|
app.include_router(dashboard_api.router, prefix="/api/v1/dashboard", tags=["Dashboard"])
|
||||||
app.include_router(devportal_api.router, prefix="/api/v1/devportal", tags=["DEV Portal"])
|
app.include_router(devportal_api.router, prefix="/api/v1/devportal", tags=["DEV Portal"])
|
||||||
app.include_router(timetracking_api, prefix="/api/v1/timetracking", tags=["Time Tracking"])
|
app.include_router(timetracking_api, prefix="/api/v1/timetracking", tags=["Time Tracking"])
|
||||||
|
app.include_router(backups_api.router, prefix="/api/v1", tags=["Backup System"])
|
||||||
app.include_router(emails_api.router, prefix="/api/v1", tags=["Email System"])
|
app.include_router(emails_api.router, prefix="/api/v1", tags=["Email System"])
|
||||||
|
|
||||||
# Frontend Routers
|
# Frontend Routers
|
||||||
@ -132,6 +141,7 @@ app.include_router(vendors_views.router, tags=["Frontend"])
|
|||||||
app.include_router(billing_views.router, tags=["Frontend"])
|
app.include_router(billing_views.router, tags=["Frontend"])
|
||||||
app.include_router(settings_views.router, tags=["Frontend"])
|
app.include_router(settings_views.router, tags=["Frontend"])
|
||||||
app.include_router(devportal_views.router, tags=["Frontend"])
|
app.include_router(devportal_views.router, tags=["Frontend"])
|
||||||
|
app.include_router(backups_views.router, tags=["Frontend"])
|
||||||
app.include_router(timetracking_views.router, tags=["Frontend"])
|
app.include_router(timetracking_views.router, tags=["Frontend"])
|
||||||
app.include_router(emails_views.router, tags=["Frontend"])
|
app.include_router(emails_views.router, tags=["Frontend"])
|
||||||
|
|
||||||
@ -175,6 +185,21 @@ async def disable_module_endpoint(module_name: str):
|
|||||||
"restart_required": True
|
"restart_required": True
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@app.get("/docs/{doc_name}")
|
||||||
|
async def serve_documentation(doc_name: str):
|
||||||
|
"""Serve markdown documentation files"""
|
||||||
|
docs_dir = Path(__file__).parent / "docs"
|
||||||
|
doc_path = docs_dir / doc_name
|
||||||
|
|
||||||
|
# Security: Ensure path is within docs directory
|
||||||
|
if not doc_path.resolve().is_relative_to(docs_dir.resolve()):
|
||||||
|
return {"error": "Invalid path"}
|
||||||
|
|
||||||
|
if doc_path.exists() and doc_path.suffix == ".md":
|
||||||
|
return FileResponse(doc_path, media_type="text/markdown")
|
||||||
|
|
||||||
|
return {"error": "Documentation not found"}
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
import uvicorn
|
import uvicorn
|
||||||
import os
|
import os
|
||||||
|
|||||||
225
migrations/014_email_workflows.sql
Normal file
225
migrations/014_email_workflows.sql
Normal file
@ -0,0 +1,225 @@
|
|||||||
|
-- Migration 014: Email Workflows System
|
||||||
|
-- Automatic actions based on email classification categories
|
||||||
|
|
||||||
|
-- Email Workflows Table
|
||||||
|
CREATE TABLE email_workflows (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
name VARCHAR(255) NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
|
||||||
|
-- Trigger Conditions
|
||||||
|
classification_trigger VARCHAR(50) NOT NULL, -- invoice, freight_note, time_confirmation, etc.
|
||||||
|
sender_pattern VARCHAR(255), -- Optional: only for emails from specific senders
|
||||||
|
subject_pattern VARCHAR(255), -- Optional: regex pattern for subject
|
||||||
|
confidence_threshold DECIMAL(3,2) DEFAULT 0.70, -- Minimum AI confidence to trigger
|
||||||
|
|
||||||
|
-- Workflow Steps (JSON array of actions to perform)
|
||||||
|
workflow_steps JSONB NOT NULL, -- [{"action": "create_ticket", "params": {...}}, ...]
|
||||||
|
|
||||||
|
-- Priority and Status
|
||||||
|
priority INTEGER DEFAULT 100, -- Lower number = higher priority
|
||||||
|
enabled BOOLEAN DEFAULT true,
|
||||||
|
stop_on_match BOOLEAN DEFAULT true, -- Stop processing other workflows if this matches
|
||||||
|
|
||||||
|
-- Statistics
|
||||||
|
execution_count INTEGER DEFAULT 0,
|
||||||
|
success_count INTEGER DEFAULT 0,
|
||||||
|
failure_count INTEGER DEFAULT 0,
|
||||||
|
last_executed_at TIMESTAMP,
|
||||||
|
|
||||||
|
-- Audit
|
||||||
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
created_by_user_id INTEGER,
|
||||||
|
|
||||||
|
FOREIGN KEY (created_by_user_id) REFERENCES users(user_id) ON DELETE SET NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Workflow Execution Log Table
|
||||||
|
CREATE TABLE email_workflow_executions (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
workflow_id INTEGER NOT NULL,
|
||||||
|
email_id INTEGER NOT NULL,
|
||||||
|
|
||||||
|
-- Execution Details
|
||||||
|
status VARCHAR(50) NOT NULL, -- pending, running, completed, failed, skipped
|
||||||
|
steps_completed INTEGER DEFAULT 0,
|
||||||
|
steps_total INTEGER,
|
||||||
|
|
||||||
|
-- Results
|
||||||
|
result_json JSONB, -- Store results from each step
|
||||||
|
error_message TEXT,
|
||||||
|
|
||||||
|
-- Timing
|
||||||
|
started_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
completed_at TIMESTAMP,
|
||||||
|
execution_time_ms INTEGER,
|
||||||
|
|
||||||
|
FOREIGN KEY (workflow_id) REFERENCES email_workflows(id) ON DELETE CASCADE,
|
||||||
|
FOREIGN KEY (email_id) REFERENCES email_messages(id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Workflow Step Actions Table (for pre-defined actions)
|
||||||
|
CREATE TABLE email_workflow_actions (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
action_code VARCHAR(100) UNIQUE NOT NULL,
|
||||||
|
name VARCHAR(255) NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
category VARCHAR(50), -- ticket_creation, notification, linking, extraction, etc.
|
||||||
|
|
||||||
|
-- Action Schema (defines what parameters this action accepts)
|
||||||
|
parameter_schema JSONB, -- JSON Schema for validation
|
||||||
|
|
||||||
|
-- Examples
|
||||||
|
example_config JSONB,
|
||||||
|
|
||||||
|
-- Status
|
||||||
|
enabled BOOLEAN DEFAULT true,
|
||||||
|
|
||||||
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Indexes for Performance
|
||||||
|
CREATE INDEX idx_email_workflows_classification ON email_workflows(classification_trigger) WHERE enabled = true;
|
||||||
|
CREATE INDEX idx_email_workflows_priority ON email_workflows(priority, enabled);
|
||||||
|
CREATE INDEX idx_email_workflow_executions_workflow ON email_workflow_executions(workflow_id);
|
||||||
|
CREATE INDEX idx_email_workflow_executions_email ON email_workflow_executions(email_id);
|
||||||
|
CREATE INDEX idx_email_workflow_executions_status ON email_workflow_executions(status);
|
||||||
|
|
||||||
|
-- Update Trigger
|
||||||
|
CREATE OR REPLACE FUNCTION update_email_workflows_updated_at()
|
||||||
|
RETURNS TRIGGER AS $$
|
||||||
|
BEGIN
|
||||||
|
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||||
|
RETURN NEW;
|
||||||
|
END;
|
||||||
|
$$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
CREATE TRIGGER trigger_email_workflows_updated_at
|
||||||
|
BEFORE UPDATE ON email_workflows
|
||||||
|
FOR EACH ROW
|
||||||
|
EXECUTE FUNCTION update_email_workflows_updated_at();
|
||||||
|
|
||||||
|
-- Insert Default Workflow Actions
|
||||||
|
INSERT INTO email_workflow_actions (action_code, name, description, category, parameter_schema, example_config) VALUES
|
||||||
|
|
||||||
|
-- Ticket/Case Creation
|
||||||
|
('create_ticket', 'Create Support Ticket', 'Creates a new support ticket/case from email', 'ticket_creation',
|
||||||
|
'{"type": "object", "properties": {"module": {"type": "string"}, "priority": {"type": "string"}, "assign_to": {"type": "integer"}}}',
|
||||||
|
'{"module": "support_cases", "priority": "normal", "assign_to": null}'),
|
||||||
|
|
||||||
|
('create_time_entry', 'Create Time Entry', 'Creates a time tracking entry from email', 'time_tracking',
|
||||||
|
'{"type": "object", "properties": {"customer_id": {"type": "integer"}, "hours": {"type": "number"}, "extract_from": {"type": "string"}}}',
|
||||||
|
'{"extract_from": "body", "default_hours": 1.0}'),
|
||||||
|
|
||||||
|
-- Linking Actions
|
||||||
|
('link_to_vendor', 'Link to Vendor', 'Links email to vendor based on sender', 'linking',
|
||||||
|
'{"type": "object", "properties": {"match_by": {"type": "string", "enum": ["email", "name", "cvr"]}, "auto_create": {"type": "boolean"}}}',
|
||||||
|
'{"match_by": "email", "auto_create": false}'),
|
||||||
|
|
||||||
|
('link_to_customer', 'Link to Customer', 'Links email to customer', 'linking',
|
||||||
|
'{"type": "object", "properties": {"match_by": {"type": "string"}, "domain_matching": {"type": "boolean"}}}',
|
||||||
|
'{"match_by": "email_domain", "domain_matching": true}'),
|
||||||
|
|
||||||
|
-- Extraction Actions
|
||||||
|
('extract_invoice_data', 'Extract Invoice Data', 'Extracts invoice number, amount, due date', 'extraction',
|
||||||
|
'{"type": "object", "properties": {"ai_provider": {"type": "string"}, "fallback_to_regex": {"type": "boolean"}}}',
|
||||||
|
'{"ai_provider": "ollama", "fallback_to_regex": true}'),
|
||||||
|
|
||||||
|
('extract_tracking_number', 'Extract Tracking Number', 'Extracts shipment tracking numbers', 'extraction',
|
||||||
|
'{"type": "object", "properties": {"carriers": {"type": "array", "items": {"type": "string"}}}}',
|
||||||
|
'{"carriers": ["postnord", "gls", "dao"]}'),
|
||||||
|
|
||||||
|
-- Notification Actions
|
||||||
|
('send_slack_notification', 'Send Slack Notification', 'Posts notification to Slack channel', 'notification',
|
||||||
|
'{"type": "object", "properties": {"channel": {"type": "string"}, "mention": {"type": "string"}, "template": {"type": "string"}}}',
|
||||||
|
'{"channel": "#invoices", "template": "New invoice from {{sender}}: {{subject}}"}'),
|
||||||
|
|
||||||
|
('send_email_notification', 'Send Email Notification', 'Sends email notification to user/team', 'notification',
|
||||||
|
'{"type": "object", "properties": {"recipients": {"type": "array"}, "template": {"type": "string"}}}',
|
||||||
|
'{"recipients": ["admin@bmcnetworks.dk"], "template": "default"}'),
|
||||||
|
|
||||||
|
-- Status Changes
|
||||||
|
('mark_as_processed', 'Mark as Processed', 'Updates email status to processed', 'status',
|
||||||
|
'{"type": "object", "properties": {"status": {"type": "string"}}}',
|
||||||
|
'{"status": "processed"}'),
|
||||||
|
|
||||||
|
('flag_for_review', 'Flag for Manual Review', 'Flags email for human review', 'status',
|
||||||
|
'{"type": "object", "properties": {"reason": {"type": "string"}, "assign_to": {"type": "integer"}}}',
|
||||||
|
'{"reason": "Low confidence classification", "assign_to": null}'),
|
||||||
|
|
||||||
|
-- Conditional Actions
|
||||||
|
('conditional_branch', 'Conditional Branch', 'Executes different actions based on conditions', 'control_flow',
|
||||||
|
'{"type": "object", "properties": {"condition": {"type": "string"}, "if_true": {"type": "array"}, "if_false": {"type": "array"}}}',
|
||||||
|
'{"condition": "confidence_score > 0.85", "if_true": [{"action": "create_ticket"}], "if_false": [{"action": "flag_for_review"}]}');
|
||||||
|
|
||||||
|
-- Insert Example Workflows
|
||||||
|
INSERT INTO email_workflows (name, description, classification_trigger, confidence_threshold, workflow_steps, priority, enabled) VALUES
|
||||||
|
|
||||||
|
-- Invoice Workflow
|
||||||
|
('Invoice Processing', 'Automatic processing of invoice emails', 'invoice', 0.70,
|
||||||
|
'[
|
||||||
|
{"action": "link_to_vendor", "params": {"match_by": "email", "auto_create": false}},
|
||||||
|
{"action": "extract_invoice_data", "params": {"ai_provider": "ollama", "fallback_to_regex": true}},
|
||||||
|
{"action": "create_ticket", "params": {"module": "billing_invoices", "priority": "normal"}},
|
||||||
|
{"action": "send_slack_notification", "params": {"channel": "#invoices", "template": "New invoice from {{sender}}: {{extracted_invoice_number}}"}},
|
||||||
|
{"action": "mark_as_processed", "params": {"status": "processed"}}
|
||||||
|
]'::jsonb, 10, true),
|
||||||
|
|
||||||
|
-- Time Confirmation Workflow
|
||||||
|
('Time Confirmation Processing', 'Process time tracking confirmations', 'time_confirmation', 0.65,
|
||||||
|
'[
|
||||||
|
{"action": "link_to_customer", "params": {"match_by": "email_domain", "domain_matching": true}},
|
||||||
|
{"action": "create_time_entry", "params": {"extract_from": "body", "default_hours": 1.0}},
|
||||||
|
{"action": "send_email_notification", "params": {"recipients": ["admin@bmcnetworks.dk"], "template": "time_confirmation"}},
|
||||||
|
{"action": "mark_as_processed", "params": {"status": "processed"}}
|
||||||
|
]'::jsonb, 20, true),
|
||||||
|
|
||||||
|
-- Freight Note Workflow
|
||||||
|
('Freight Note Processing', 'Track shipments from freight notes', 'freight_note', 0.70,
|
||||||
|
'[
|
||||||
|
{"action": "extract_tracking_number", "params": {"carriers": ["postnord", "gls", "dao"]}},
|
||||||
|
{"action": "link_to_vendor", "params": {"match_by": "email", "auto_create": false}},
|
||||||
|
{"action": "create_ticket", "params": {"module": "hardware_shipments", "priority": "low"}},
|
||||||
|
{"action": "mark_as_processed", "params": {"status": "processed"}}
|
||||||
|
]'::jsonb, 30, true),
|
||||||
|
|
||||||
|
-- Bankruptcy Alert Workflow
|
||||||
|
('Bankruptcy Alert', 'Alert team when customer bankruptcy detected', 'bankruptcy', 0.80,
|
||||||
|
'[
|
||||||
|
{"action": "flag_for_review", "params": {"reason": "Customer bankruptcy notification", "assign_to": null}},
|
||||||
|
{"action": "send_slack_notification", "params": {"channel": "#alerts", "template": "🚨 Bankruptcy alert: {{subject}}"}},
|
||||||
|
{"action": "send_email_notification", "params": {"recipients": ["admin@bmcnetworks.dk"], "template": "bankruptcy_alert"}},
|
||||||
|
{"action": "mark_as_processed", "params": {"status": "flagged"}}
|
||||||
|
]'::jsonb, 5, true),
|
||||||
|
|
||||||
|
-- Low Confidence Review Workflow
|
||||||
|
('Low Confidence Review', 'Flag emails with uncertain classification', 'general', 0.50,
|
||||||
|
'[
|
||||||
|
{"action": "flag_for_review", "params": {"reason": "Low confidence classification", "assign_to": null}}
|
||||||
|
]'::jsonb, 90, true);
|
||||||
|
|
||||||
|
-- View for workflow monitoring
|
||||||
|
CREATE OR REPLACE VIEW v_workflow_stats AS
|
||||||
|
SELECT
|
||||||
|
wf.id,
|
||||||
|
wf.name,
|
||||||
|
wf.classification_trigger,
|
||||||
|
wf.enabled,
|
||||||
|
wf.execution_count,
|
||||||
|
wf.success_count,
|
||||||
|
wf.failure_count,
|
||||||
|
ROUND((wf.success_count::numeric / NULLIF(wf.execution_count, 0) * 100), 2) as success_rate,
|
||||||
|
wf.last_executed_at,
|
||||||
|
COUNT(wfe.id) as pending_executions,
|
||||||
|
wf.created_at,
|
||||||
|
wf.updated_at
|
||||||
|
FROM email_workflows wf
|
||||||
|
LEFT JOIN email_workflow_executions wfe ON wf.id = wfe.workflow_id AND wfe.status = 'pending'
|
||||||
|
GROUP BY wf.id
|
||||||
|
ORDER BY wf.priority;
|
||||||
|
|
||||||
|
COMMENT ON TABLE email_workflows IS 'Automated workflows triggered by email classification';
|
||||||
|
COMMENT ON TABLE email_workflow_executions IS 'Log of all workflow executions';
|
||||||
|
COMMENT ON TABLE email_workflow_actions IS 'Registry of available workflow actions';
|
||||||
|
COMMENT ON VIEW v_workflow_stats IS 'Statistics and monitoring for email workflows';
|
||||||
91
migrations/024_backup_system.sql
Normal file
91
migrations/024_backup_system.sql
Normal file
@ -0,0 +1,91 @@
|
|||||||
|
-- Migration 024: Backup System
|
||||||
|
-- Adds tables for automated backup/restore with offsite upload and notifications
|
||||||
|
|
||||||
|
-- Backup Jobs Table
|
||||||
|
CREATE TABLE IF NOT EXISTS backup_jobs (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
job_type VARCHAR(20) NOT NULL CHECK (job_type IN ('database', 'files', 'full')),
|
||||||
|
status VARCHAR(20) NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'running', 'completed', 'failed')),
|
||||||
|
backup_format VARCHAR(10) NOT NULL CHECK (backup_format IN ('dump', 'sql', 'tar.gz')),
|
||||||
|
file_path VARCHAR(500),
|
||||||
|
file_size_bytes BIGINT,
|
||||||
|
checksum_sha256 VARCHAR(64),
|
||||||
|
is_monthly BOOLEAN DEFAULT FALSE,
|
||||||
|
includes_uploads BOOLEAN DEFAULT FALSE,
|
||||||
|
includes_logs BOOLEAN DEFAULT FALSE,
|
||||||
|
includes_data BOOLEAN DEFAULT FALSE,
|
||||||
|
started_at TIMESTAMP,
|
||||||
|
completed_at TIMESTAMP,
|
||||||
|
error_message TEXT,
|
||||||
|
retention_until DATE,
|
||||||
|
offsite_uploaded_at TIMESTAMP,
|
||||||
|
offsite_retry_count INTEGER DEFAULT 0,
|
||||||
|
notification_sent BOOLEAN DEFAULT FALSE,
|
||||||
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Indexes for backup_jobs
|
||||||
|
CREATE INDEX idx_backup_jobs_status ON backup_jobs(status);
|
||||||
|
CREATE INDEX idx_backup_jobs_created ON backup_jobs(created_at DESC);
|
||||||
|
CREATE INDEX idx_backup_jobs_type ON backup_jobs(job_type);
|
||||||
|
CREATE INDEX idx_backup_jobs_is_monthly ON backup_jobs(is_monthly) WHERE is_monthly = TRUE;
|
||||||
|
CREATE INDEX idx_backup_jobs_retention ON backup_jobs(retention_until) WHERE retention_until IS NOT NULL;
|
||||||
|
CREATE INDEX idx_backup_jobs_offsite_pending ON backup_jobs(offsite_uploaded_at) WHERE offsite_uploaded_at IS NULL AND status = 'completed';
|
||||||
|
|
||||||
|
-- System Status Table (for maintenance mode)
|
||||||
|
CREATE TABLE IF NOT EXISTS system_status (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
maintenance_mode BOOLEAN DEFAULT FALSE,
|
||||||
|
maintenance_message TEXT DEFAULT 'System under vedligeholdelse',
|
||||||
|
maintenance_eta_minutes INTEGER,
|
||||||
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Insert default system status
|
||||||
|
INSERT INTO system_status (maintenance_mode, maintenance_message, maintenance_eta_minutes)
|
||||||
|
VALUES (FALSE, 'System under vedligeholdelse', NULL)
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
|
||||||
|
-- Backup Notifications Table
|
||||||
|
CREATE TABLE IF NOT EXISTS backup_notifications (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
backup_job_id INTEGER REFERENCES backup_jobs(id) ON DELETE SET NULL,
|
||||||
|
event_type VARCHAR(30) NOT NULL CHECK (event_type IN ('backup_failed', 'offsite_failed', 'offsite_retry', 'storage_low', 'restore_started', 'backup_success')),
|
||||||
|
message TEXT NOT NULL,
|
||||||
|
mattermost_payload JSONB,
|
||||||
|
sent_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
acknowledged BOOLEAN DEFAULT FALSE,
|
||||||
|
acknowledged_at TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Indexes for backup_notifications
|
||||||
|
CREATE INDEX idx_backup_notifications_event ON backup_notifications(event_type);
|
||||||
|
CREATE INDEX idx_backup_notifications_sent ON backup_notifications(sent_at DESC);
|
||||||
|
CREATE INDEX idx_backup_notifications_unacked ON backup_notifications(acknowledged) WHERE acknowledged = FALSE;
|
||||||
|
CREATE INDEX idx_backup_notifications_job ON backup_notifications(backup_job_id) WHERE backup_job_id IS NOT NULL;
|
||||||
|
|
||||||
|
-- Trigger to update updated_at on backup_jobs
|
||||||
|
CREATE OR REPLACE FUNCTION update_backup_jobs_timestamp()
|
||||||
|
RETURNS TRIGGER AS $$
|
||||||
|
BEGIN
|
||||||
|
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||||
|
RETURN NEW;
|
||||||
|
END;
|
||||||
|
$$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
CREATE TRIGGER backup_jobs_update_timestamp
|
||||||
|
BEFORE UPDATE ON backup_jobs
|
||||||
|
FOR EACH ROW
|
||||||
|
EXECUTE FUNCTION update_backup_jobs_timestamp();
|
||||||
|
|
||||||
|
-- Comments for documentation
|
||||||
|
COMMENT ON TABLE backup_jobs IS 'Tracks all backup operations (database, files, full) with retention and offsite upload status';
|
||||||
|
COMMENT ON TABLE system_status IS 'Global system maintenance mode flag for restore operations';
|
||||||
|
COMMENT ON TABLE backup_notifications IS 'Notification log for Mattermost alerts on backup events';
|
||||||
|
|
||||||
|
COMMENT ON COLUMN backup_jobs.job_type IS 'Type of backup: database (pg_dump), files (tar.gz), or full (both)';
|
||||||
|
COMMENT ON COLUMN backup_jobs.backup_format IS 'Format: dump (compressed), sql (plain text), or tar.gz (files archive)';
|
||||||
|
COMMENT ON COLUMN backup_jobs.is_monthly IS 'TRUE for monthly backups kept for 12 months, FALSE for daily 30-day retention';
|
||||||
|
COMMENT ON COLUMN backup_jobs.offsite_retry_count IS 'Number of times offsite upload has been retried (max 3)';
|
||||||
|
COMMENT ON COLUMN system_status.maintenance_eta_minutes IS 'Estimated time in minutes until maintenance is complete';
|
||||||
78
migrations/050_email_activity_log.sql
Normal file
78
migrations/050_email_activity_log.sql
Normal file
@ -0,0 +1,78 @@
|
|||||||
|
-- Email Activity Log System
|
||||||
|
-- Tracks all actions and events for emails
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS email_activity_log (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
email_id INTEGER NOT NULL REFERENCES email_messages(id) ON DELETE CASCADE,
|
||||||
|
event_type VARCHAR(50) NOT NULL, -- fetched, saved, classified, workflow_executed, rule_matched, status_changed, read, attachment_downloaded, linked, etc.
|
||||||
|
event_category VARCHAR(30) NOT NULL DEFAULT 'system', -- system, user, workflow, rule, integration
|
||||||
|
description TEXT NOT NULL,
|
||||||
|
metadata JSONB, -- Flexible storage for event-specific data
|
||||||
|
user_id INTEGER REFERENCES users(user_id) ON DELETE SET NULL, -- NULL for system events
|
||||||
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
created_by VARCHAR(255) -- email or system identifier
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Indexes for performance
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_email_activity_log_email_id ON email_activity_log(email_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_email_activity_log_event_type ON email_activity_log(event_type);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_email_activity_log_created_at ON email_activity_log(created_at DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_email_activity_log_category ON email_activity_log(event_category);
|
||||||
|
|
||||||
|
-- View for easy email timeline
|
||||||
|
CREATE OR REPLACE VIEW email_timeline AS
|
||||||
|
SELECT
|
||||||
|
eal.id,
|
||||||
|
eal.email_id,
|
||||||
|
em.subject,
|
||||||
|
em.sender_email,
|
||||||
|
eal.event_type,
|
||||||
|
eal.event_category,
|
||||||
|
eal.description,
|
||||||
|
eal.metadata,
|
||||||
|
eal.user_id,
|
||||||
|
u.username as user_name,
|
||||||
|
eal.created_at,
|
||||||
|
eal.created_by
|
||||||
|
FROM email_activity_log eal
|
||||||
|
LEFT JOIN email_messages em ON eal.email_id = em.id
|
||||||
|
LEFT JOIN users u ON eal.user_id = u.user_id
|
||||||
|
ORDER BY eal.created_at DESC;
|
||||||
|
|
||||||
|
-- Helper function to log email events
|
||||||
|
CREATE OR REPLACE FUNCTION log_email_event(
|
||||||
|
p_email_id INTEGER,
|
||||||
|
p_event_type VARCHAR(50),
|
||||||
|
p_event_category VARCHAR(30),
|
||||||
|
p_description TEXT,
|
||||||
|
p_metadata JSONB DEFAULT NULL,
|
||||||
|
p_user_id INTEGER DEFAULT NULL,
|
||||||
|
p_created_by VARCHAR(255) DEFAULT 'system'
|
||||||
|
) RETURNS INTEGER AS $$
|
||||||
|
DECLARE
|
||||||
|
v_log_id INTEGER;
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO email_activity_log (
|
||||||
|
email_id,
|
||||||
|
event_type,
|
||||||
|
event_category,
|
||||||
|
description,
|
||||||
|
metadata,
|
||||||
|
user_id,
|
||||||
|
created_by
|
||||||
|
) VALUES (
|
||||||
|
p_email_id,
|
||||||
|
p_event_type,
|
||||||
|
p_event_category,
|
||||||
|
p_description,
|
||||||
|
p_metadata,
|
||||||
|
p_user_id,
|
||||||
|
p_created_by
|
||||||
|
) RETURNING id INTO v_log_id;
|
||||||
|
|
||||||
|
RETURN v_log_id;
|
||||||
|
END;
|
||||||
|
$$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
COMMENT ON TABLE email_activity_log IS 'Complete audit trail of all email events and actions';
|
||||||
|
COMMENT ON FUNCTION log_email_event IS 'Helper function to log email events with consistent format';
|
||||||
@ -13,6 +13,9 @@ aiohttp==3.10.10
|
|||||||
APScheduler==3.10.4
|
APScheduler==3.10.4
|
||||||
msal==1.31.1
|
msal==1.31.1
|
||||||
|
|
||||||
|
# Backup & SSH
|
||||||
|
paramiko==3.4.0
|
||||||
|
|
||||||
# AI & Document Processing
|
# AI & Document Processing
|
||||||
httpx==0.27.2
|
httpx==0.27.2
|
||||||
PyPDF2==3.0.1
|
PyPDF2==3.0.1
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user