feat: Implement supplier invoices management with e-conomic integration
- Added FastAPI views for supplier invoices in the billing frontend. - Created EconomicService for handling e-conomic API interactions, including safety modes for read-only and dry-run operations. - Developed database migration for supplier invoices, including tables for invoices, line items, and settings. - Documented kassekladde module features, architecture, API endpoints, and usage guide in KASSEKLADDE.md. - Implemented views for overdue invoices and pending e-conomic sync.
This commit is contained in:
parent
974876ac67
commit
dcb4d8a280
@ -8,6 +8,9 @@ RUN apt-get update && apt-get install -y \
|
||||
git \
|
||||
libpq-dev \
|
||||
gcc \
|
||||
tesseract-ocr \
|
||||
tesseract-ocr-dan \
|
||||
tesseract-ocr-eng \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Build arguments for GitHub release deployment
|
||||
|
||||
@ -4,9 +4,13 @@ API endpoints for billing operations
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
from . import supplier_invoices
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# Include supplier invoices router
|
||||
router.include_router(supplier_invoices.router, prefix="", tags=["Supplier Invoices"])
|
||||
|
||||
|
||||
@router.get("/billing/invoices")
|
||||
async def list_invoices():
|
||||
|
||||
1333
app/billing/backend/supplier_invoices.py
Normal file
1333
app/billing/backend/supplier_invoices.py
Normal file
File diff suppressed because it is too large
Load Diff
3
app/billing/frontend/__init__.py
Normal file
3
app/billing/frontend/__init__.py
Normal file
@ -0,0 +1,3 @@
|
||||
"""
|
||||
Billing Frontend Package
|
||||
"""
|
||||
1364
app/billing/frontend/supplier_invoices.html
Normal file
1364
app/billing/frontend/supplier_invoices.html
Normal file
File diff suppressed because it is too large
Load Diff
1261
app/billing/frontend/template_builder.html
Normal file
1261
app/billing/frontend/template_builder.html
Normal file
File diff suppressed because it is too large
Load Diff
363
app/billing/frontend/templates_list.html
Normal file
363
app/billing/frontend/templates_list.html
Normal file
@ -0,0 +1,363 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="da">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Templates - BMC Hub</title>
|
||||
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet">
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.1/font/bootstrap-icons.css">
|
||||
<style>
|
||||
body {
|
||||
background-color: #f8f9fa;
|
||||
padding-top: 80px;
|
||||
}
|
||||
.navbar {
|
||||
background: #ffffff;
|
||||
box-shadow: 0 2px 15px rgba(0,0,0,0.03);
|
||||
border-bottom: 1px solid #eee;
|
||||
}
|
||||
.template-card {
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
.template-card:hover {
|
||||
transform: translateY(-2px);
|
||||
box-shadow: 0 4px 20px rgba(0,0,0,0.1);
|
||||
}
|
||||
.test-modal .pdf-preview {
|
||||
max-height: 400px;
|
||||
overflow-y: auto;
|
||||
background: #f8f9fa;
|
||||
padding: 15px;
|
||||
border-radius: 8px;
|
||||
font-family: monospace;
|
||||
font-size: 12px;
|
||||
white-space: pre-wrap;
|
||||
border: 1px solid #dee2e6;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<!-- Navbar -->
|
||||
<nav class="navbar navbar-expand-lg fixed-top">
|
||||
<div class="container-fluid">
|
||||
<a class="navbar-brand" href="/">
|
||||
<i class="bi bi-speedometer2 me-2"></i>BMC Hub
|
||||
</a>
|
||||
<div class="navbar-nav ms-auto">
|
||||
<a class="nav-link" href="/billing/supplier-invoices">
|
||||
<i class="bi bi-arrow-left me-1"></i>Tilbage til Fakturaer
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<div class="container mt-4">
|
||||
<div class="d-flex justify-content-between align-items-center mb-4">
|
||||
<div>
|
||||
<h2><i class="bi bi-grid-3x3 me-2"></i>Faktura Templates</h2>
|
||||
<p class="text-muted">Administrer templates til automatisk faktura-udtrækning</p>
|
||||
</div>
|
||||
<a href="/billing/template-builder" class="btn btn-primary">
|
||||
<i class="bi bi-plus-circle me-2"></i>Ny Template
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div id="templatesList" class="row">
|
||||
<!-- Templates loaded here -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Test Modal -->
|
||||
<div class="modal fade test-modal" id="testModal" tabindex="-1">
|
||||
<div class="modal-dialog modal-xl">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title">
|
||||
<i class="bi bi-flask me-2"></i>Test Template: <span id="modalTemplateName"></span>
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<div class="row">
|
||||
<div class="col-md-12 mb-3">
|
||||
<label class="form-label">Vælg PDF fil til test</label>
|
||||
<select class="form-select" id="testFileSelect">
|
||||
<option value="">-- Vælg fil --</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="testResultsContainer" class="d-none">
|
||||
<div class="row">
|
||||
<div class="col-md-5">
|
||||
<h6>PDF Preview</h6>
|
||||
<div class="pdf-preview" id="testPdfPreview"></div>
|
||||
</div>
|
||||
<div class="col-md-7">
|
||||
<div id="testResults" class="alert" role="alert">
|
||||
<!-- Test results shown here -->
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Luk</button>
|
||||
<button type="button" class="btn btn-primary" onclick="runTest()">
|
||||
<i class="bi bi-play-fill me-2"></i>Kør Test
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/js/bootstrap.bundle.min.js"></script>
|
||||
<script>
|
||||
let currentTemplateId = null;
|
||||
|
||||
document.addEventListener('DOMContentLoaded', async () => {
|
||||
await loadTemplates();
|
||||
await loadPendingFiles();
|
||||
});
|
||||
|
||||
async function loadTemplates() {
|
||||
try {
|
||||
const response = await fetch('/api/v1/supplier-invoices/templates');
|
||||
const templates = await response.json();
|
||||
|
||||
const container = document.getElementById('templatesList');
|
||||
container.innerHTML = '';
|
||||
|
||||
if (templates.length === 0) {
|
||||
container.innerHTML = `
|
||||
<div class="col-12">
|
||||
<div class="alert alert-info">
|
||||
<i class="bi bi-info-circle me-2"></i>
|
||||
Ingen templates fundet. Klik "Ny Template" for at oprette den første.
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
return;
|
||||
}
|
||||
|
||||
templates.forEach(template => {
|
||||
const detectionPatterns = template.detection_patterns || [];
|
||||
const fieldMappings = template.field_mappings || {};
|
||||
const fieldCount = Object.keys(fieldMappings).filter(k => !['lines_start', 'lines_end', 'line_item'].includes(k)).length;
|
||||
|
||||
container.innerHTML += `
|
||||
<div class="col-md-4 mb-3">
|
||||
<div class="card template-card">
|
||||
<div class="card-body">
|
||||
<h5 class="card-title">
|
||||
<i class="bi bi-file-text me-2"></i>${template.template_name}
|
||||
</h5>
|
||||
<p class="card-text text-muted mb-2">
|
||||
<small>
|
||||
<i class="bi bi-building me-1"></i>${template.vendor_name || 'Ingen leverandør'}<br>
|
||||
<i class="bi bi-check-circle me-1"></i>${detectionPatterns.length} detektionsmønstre<br>
|
||||
<i class="bi bi-input-cursor me-1"></i>${fieldCount} felter<br>
|
||||
<i class="bi bi-graph-up me-1"></i>${template.usage_count || 0} gange brugt
|
||||
</small>
|
||||
</p>
|
||||
<div class="d-flex gap-2">
|
||||
<button class="btn btn-sm btn-info" onclick="openTestModal(${template.template_id}, '${template.template_name}')">
|
||||
<i class="bi bi-flask"></i> Test
|
||||
</button>
|
||||
<button class="btn btn-sm btn-danger" onclick="deleteTemplate(${template.template_id})">
|
||||
<i class="bi bi-trash"></i>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to load templates:', error);
|
||||
alert('Kunne ikke hente templates');
|
||||
}
|
||||
}
|
||||
|
||||
async function loadPendingFiles() {
|
||||
try {
|
||||
const response = await fetch('/api/v1/supplier-invoices/pending-files');
|
||||
const data = await response.json();
|
||||
|
||||
const select = document.getElementById('testFileSelect');
|
||||
select.innerHTML = '<option value="">-- Vælg fil --</option>';
|
||||
|
||||
data.files.forEach(file => {
|
||||
select.innerHTML += `<option value="${file.file_id}">${file.filename}</option>`;
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to load files:', error);
|
||||
}
|
||||
}
|
||||
|
||||
function openTestModal(templateId, templateName) {
|
||||
currentTemplateId = templateId;
|
||||
document.getElementById('modalTemplateName').textContent = templateName;
|
||||
document.getElementById('testResultsContainer').classList.add('d-none');
|
||||
document.getElementById('testFileSelect').value = '';
|
||||
|
||||
const modal = new bootstrap.Modal(document.getElementById('testModal'));
|
||||
modal.show();
|
||||
}
|
||||
|
||||
async function runTest() {
|
||||
const fileId = document.getElementById('testFileSelect').value;
|
||||
|
||||
if (!fileId) {
|
||||
alert('Vælg en PDF fil');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!currentTemplateId) {
|
||||
alert('Ingen template valgt');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Load PDF text
|
||||
const fileResponse = await fetch(`/api/v1/supplier-invoices/reprocess/${fileId}`, {
|
||||
method: 'POST'
|
||||
});
|
||||
const fileData = await fileResponse.json();
|
||||
const pdfText = fileData.pdf_text;
|
||||
|
||||
// Show PDF preview
|
||||
document.getElementById('testPdfPreview').textContent = pdfText;
|
||||
document.getElementById('testResultsContainer').classList.remove('d-none');
|
||||
|
||||
// Test template
|
||||
const testResponse = await fetch(`/api/v1/supplier-invoices/templates/${currentTemplateId}/test`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ pdf_text: pdfText })
|
||||
});
|
||||
|
||||
if (!testResponse.ok) {
|
||||
throw new Error('Test fejlede');
|
||||
}
|
||||
|
||||
const result = await testResponse.json();
|
||||
|
||||
// Display results
|
||||
const testResults = document.getElementById('testResults');
|
||||
testResults.className = 'alert';
|
||||
|
||||
let detectionHtml = '<h6>Detektionsmønstre:</h6><ul class="mb-2">';
|
||||
for (let dr of result.detection_results) {
|
||||
detectionHtml += `<li>${dr.found ? '✅' : '❌'} "${dr.pattern}" (weight: ${dr.weight})</li>`;
|
||||
}
|
||||
detectionHtml += '</ul>';
|
||||
|
||||
let extractedHtml = '<h6>Udtrækkede felter:</h6><ul class="mb-2">';
|
||||
const extracted = result.extracted_fields || {};
|
||||
if (Object.keys(extracted).length > 0) {
|
||||
for (let [field, value] of Object.entries(extracted)) {
|
||||
extractedHtml += `<li>✅ <strong>${field}:</strong> "${value}"</li>`;
|
||||
}
|
||||
} else {
|
||||
extractedHtml += '<li class="text-muted">Ingen felter udtrækket</li>';
|
||||
}
|
||||
extractedHtml += '</ul>';
|
||||
|
||||
// Display line items
|
||||
let linesHtml = '';
|
||||
const lineItems = result.line_items || [];
|
||||
if (lineItems.length > 0) {
|
||||
linesHtml = `
|
||||
<h6 class="mt-3">Varelinjer (${lineItems.length} stk):</h6>
|
||||
<div class="table-responsive">
|
||||
<table class="table table-sm table-bordered">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>#</th>
|
||||
${lineItems[0].item_number ? '<th>Varenr</th>' : ''}
|
||||
${lineItems[0].description ? '<th>Beskrivelse</th>' : ''}
|
||||
${lineItems[0].quantity ? '<th>Antal</th>' : ''}
|
||||
${lineItems[0].unit_price ? '<th>Pris</th>' : ''}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>`;
|
||||
|
||||
lineItems.forEach(line => {
|
||||
linesHtml += `<tr>
|
||||
<td>${line.line_number}</td>
|
||||
${line.item_number ? `<td>${line.item_number}</td>` : ''}
|
||||
${line.description ? `<td>${line.description}</td>` : ''}
|
||||
${line.quantity ? `<td>${line.quantity}</td>` : ''}
|
||||
${line.unit_price ? `<td>${line.unit_price}</td>` : ''}
|
||||
</tr>`;
|
||||
});
|
||||
|
||||
linesHtml += `</tbody></table></div>`;
|
||||
} else {
|
||||
linesHtml = `
|
||||
<h6 class="mt-3 text-warning">⚠️ Ingen varelinjer fundet</h6>
|
||||
<p class="text-muted small">
|
||||
Tjek at:<br>
|
||||
• "Linje Start" markør findes i PDF'en<br>
|
||||
• "Linje Slut" markør findes i PDF'en<br>
|
||||
• Linje pattern matcher dine varelinjer (én linje ad gangen)<br>
|
||||
<br>
|
||||
Tip: Varelinjer skal være på én linje hver. Hvis din PDF har multi-line varelinjer,
|
||||
skal du justere pattern eller simplificere udtrækningen.
|
||||
</p>
|
||||
`;
|
||||
}
|
||||
|
||||
testResults.innerHTML = `
|
||||
<h5>${result.matched ? '✅' : '❌'} Template ${result.matched ? 'MATCHER' : 'MATCHER IKKE'}</h5>
|
||||
<p><strong>Confidence:</strong> ${(result.confidence * 100).toFixed(0)}% (threshold: 70%)</p>
|
||||
${detectionHtml}
|
||||
${extractedHtml}
|
||||
${linesHtml}
|
||||
`;
|
||||
|
||||
if (result.matched && (Object.keys(extracted).length > 0 || lineItems.length > 0)) {
|
||||
testResults.classList.add('alert-success');
|
||||
} else if (result.matched) {
|
||||
testResults.classList.add('alert-warning');
|
||||
} else {
|
||||
testResults.classList.add('alert-danger');
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Test failed:', error);
|
||||
const testResults = document.getElementById('testResults');
|
||||
testResults.className = 'alert alert-danger';
|
||||
testResults.innerHTML = `<strong>Test fejlede:</strong> ${error.message}`;
|
||||
document.getElementById('testResultsContainer').classList.remove('d-none');
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteTemplate(templateId) {
|
||||
if (!confirm('Er du sikker på at du vil slette denne template?')) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`/api/v1/supplier-invoices/templates/${templateId}`, {
|
||||
method: 'DELETE'
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
alert('✅ Template slettet');
|
||||
await loadTemplates();
|
||||
} else {
|
||||
throw new Error('Sletning fejlede');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Delete failed:', error);
|
||||
alert('❌ Kunne ikke slette template');
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
27
app/billing/frontend/views.py
Normal file
27
app/billing/frontend/views.py
Normal file
@ -0,0 +1,27 @@
|
||||
"""
|
||||
Billing Frontend Views
|
||||
Serves HTML pages for billing features
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/billing/supplier-invoices")
|
||||
async def supplier_invoices_page():
|
||||
"""Supplier invoices (kassekladde) page"""
|
||||
return FileResponse("app/billing/frontend/supplier_invoices.html")
|
||||
|
||||
|
||||
@router.get("/billing/template-builder")
|
||||
async def template_builder_page():
|
||||
"""Template builder for supplier invoice extraction"""
|
||||
return FileResponse("app/billing/frontend/template_builder.html")
|
||||
|
||||
|
||||
@router.get("/billing/templates")
|
||||
async def templates_list_page():
|
||||
"""Templates list and management page"""
|
||||
return FileResponse("app/billing/frontend/templates_list.html")
|
||||
@ -33,9 +33,19 @@ class Settings(BaseSettings):
|
||||
ECONOMIC_READ_ONLY: bool = True
|
||||
ECONOMIC_DRY_RUN: bool = True
|
||||
|
||||
# Ollama AI Integration
|
||||
OLLAMA_ENDPOINT: str = "http://ai_direct.cs.blaahund.dk"
|
||||
OLLAMA_MODEL: str = "qwen2.5:3b" # Hurtigere model til JSON extraction
|
||||
|
||||
# File Upload
|
||||
UPLOAD_DIR: str = "uploads"
|
||||
MAX_FILE_SIZE_MB: int = 50
|
||||
ALLOWED_EXTENSIONS: List[str] = [".pdf", ".png", ".jpg", ".jpeg", ".txt", ".csv"]
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
case_sensitive = True
|
||||
extra = "ignore" # Ignore extra fields from .env
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
||||
608
app/services/economic_service.py
Normal file
608
app/services/economic_service.py
Normal file
@ -0,0 +1,608 @@
|
||||
"""
|
||||
e-conomic Integration Service
|
||||
Send invoices and supplier invoices (kassekladde) to e-conomic accounting system
|
||||
|
||||
🚨 SAFETY MODES:
|
||||
- ECONOMIC_READ_ONLY: Blocks ALL write operations when True
|
||||
- ECONOMIC_DRY_RUN: Logs operations but doesn't send to e-conomic when True
|
||||
"""
|
||||
import logging
|
||||
import aiohttp
|
||||
import json
|
||||
from typing import Dict, Optional, List
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EconomicService:
|
||||
"""Service for integrating with e-conomic REST API"""
|
||||
|
||||
def __init__(self):
|
||||
self.api_url = getattr(settings, 'ECONOMIC_API_URL', 'https://restapi.e-conomic.com')
|
||||
self.app_secret_token = getattr(settings, 'ECONOMIC_APP_SECRET_TOKEN', None)
|
||||
self.agreement_grant_token = getattr(settings, 'ECONOMIC_AGREEMENT_GRANT_TOKEN', None)
|
||||
self.read_only = getattr(settings, 'ECONOMIC_READ_ONLY', True)
|
||||
self.dry_run = getattr(settings, 'ECONOMIC_DRY_RUN', True)
|
||||
|
||||
if not self.app_secret_token or not self.agreement_grant_token:
|
||||
logger.warning("⚠️ e-conomic credentials not configured")
|
||||
|
||||
# Log safety status at initialization
|
||||
if self.read_only:
|
||||
logger.warning("🔒 e-conomic READ-ONLY MODE ENABLED - All write operations will be blocked")
|
||||
elif self.dry_run:
|
||||
logger.warning("🏃 e-conomic DRY-RUN MODE ENABLED - Operations will be logged but not executed")
|
||||
else:
|
||||
logger.warning("⚠️ e-conomic WRITE MODE ACTIVE - Changes will be sent to production!")
|
||||
|
||||
def _check_write_permission(self, operation: str) -> bool:
|
||||
"""
|
||||
Check if write operations are allowed
|
||||
|
||||
Args:
|
||||
operation: Name of the operation being attempted
|
||||
|
||||
Returns:
|
||||
True if operation should proceed, False if blocked
|
||||
"""
|
||||
if self.read_only:
|
||||
logger.error(f"🚫 BLOCKED: {operation} - READ_ONLY mode is enabled")
|
||||
logger.error("To enable writes, set ECONOMIC_READ_ONLY=false in .env")
|
||||
return False
|
||||
|
||||
if self.dry_run:
|
||||
logger.warning(f"🏃 DRY-RUN: {operation} - Would execute but DRY_RUN mode is enabled")
|
||||
logger.warning("To actually send to e-conomic, set ECONOMIC_DRY_RUN=false in .env")
|
||||
return False
|
||||
|
||||
# Triple-check for production writes
|
||||
logger.warning(f"⚠️ EXECUTING WRITE OPERATION: {operation}")
|
||||
logger.warning(f"⚠️ This will modify production e-conomic at {self.api_url}")
|
||||
return True
|
||||
|
||||
def _log_api_call(self, method: str, endpoint: str, payload: Optional[Dict] = None,
|
||||
response_data: Optional[Dict] = None, status_code: Optional[int] = None):
|
||||
"""
|
||||
Comprehensive logging of all API calls
|
||||
|
||||
Args:
|
||||
method: HTTP method (GET, POST, etc.)
|
||||
endpoint: API endpoint
|
||||
payload: Request payload
|
||||
response_data: Response data
|
||||
status_code: HTTP status code
|
||||
"""
|
||||
log_entry = {
|
||||
"method": method,
|
||||
"endpoint": endpoint,
|
||||
"api_url": self.api_url,
|
||||
"read_only": self.read_only,
|
||||
"dry_run": self.dry_run
|
||||
}
|
||||
|
||||
if payload:
|
||||
log_entry["request_payload"] = payload
|
||||
if response_data:
|
||||
log_entry["response_data"] = response_data
|
||||
if status_code:
|
||||
log_entry["status_code"] = status_code
|
||||
|
||||
logger.info(f"📊 e-conomic API Call: {json.dumps(log_entry, indent=2, default=str)}")
|
||||
|
||||
def _get_headers(self) -> Dict[str, str]:
|
||||
"""Get HTTP headers for e-conomic API"""
|
||||
if not self.app_secret_token or not self.agreement_grant_token:
|
||||
raise ValueError("e-conomic credentials not configured")
|
||||
|
||||
return {
|
||||
'X-AppSecretToken': self.app_secret_token,
|
||||
'X-AgreementGrantToken': self.agreement_grant_token,
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
|
||||
async def test_connection(self) -> bool:
|
||||
"""
|
||||
Test e-conomic API connection
|
||||
|
||||
Returns:
|
||||
True if connection successful
|
||||
"""
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(
|
||||
f"{self.api_url}/self",
|
||||
headers=self._get_headers()
|
||||
) as response:
|
||||
if response.status == 200:
|
||||
data = await response.json()
|
||||
logger.info(f"✅ Connected to e-conomic: {data.get('agreementNumber')}")
|
||||
return True
|
||||
else:
|
||||
error = await response.text()
|
||||
logger.error(f"❌ e-conomic connection failed: {response.status} - {error}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"❌ e-conomic connection error: {e}")
|
||||
return False
|
||||
|
||||
# ========== SUPPLIER/VENDOR MANAGEMENT ==========
|
||||
|
||||
async def search_supplier_by_name(self, supplier_name: str) -> Optional[Dict]:
|
||||
"""
|
||||
Search for supplier in e-conomic based on name
|
||||
|
||||
Args:
|
||||
supplier_name: Name of supplier to search for
|
||||
|
||||
Returns:
|
||||
Supplier data if found, None otherwise
|
||||
"""
|
||||
try:
|
||||
url = f"{self.api_url}/suppliers"
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(url, headers=self._get_headers()) as response:
|
||||
if response.status != 200:
|
||||
logger.error(f"❌ Failed to fetch suppliers: {response.status}")
|
||||
return None
|
||||
|
||||
data = await response.json()
|
||||
suppliers = data.get('collection', [])
|
||||
|
||||
# Search for supplier by name (case-insensitive)
|
||||
search_name = supplier_name.lower().strip()
|
||||
|
||||
for supplier in suppliers:
|
||||
supplier_display_name = supplier.get('name', '').lower().strip()
|
||||
|
||||
# Exact match or contains
|
||||
if search_name in supplier_display_name or supplier_display_name in search_name:
|
||||
logger.info(f"✅ Found supplier match: {supplier.get('name')} (ID: {supplier.get('supplierNumber')})")
|
||||
return {
|
||||
'supplierNumber': supplier.get('supplierNumber'),
|
||||
'name': supplier.get('name'),
|
||||
'currency': supplier.get('currency'),
|
||||
'vatZone': supplier.get('vatZone')
|
||||
}
|
||||
|
||||
logger.warning(f"⚠️ No supplier found matching '{supplier_name}'")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error searching supplier: {e}")
|
||||
return None
|
||||
|
||||
async def create_supplier(self, supplier_data: Dict) -> Optional[Dict]:
|
||||
"""
|
||||
Create new supplier in e-conomic
|
||||
|
||||
🚨 WRITE OPERATION - Respects READ_ONLY and DRY_RUN modes
|
||||
|
||||
Args:
|
||||
supplier_data: {
|
||||
'name': str,
|
||||
'address': str (optional),
|
||||
'city': str (optional),
|
||||
'zip': str (optional),
|
||||
'country': str (optional),
|
||||
'corporate_identification_number': str (optional - CVR),
|
||||
'currency': str (default 'DKK'),
|
||||
'payment_terms_number': int (default 1),
|
||||
'vat_zone_number': int (default 1)
|
||||
}
|
||||
|
||||
Returns:
|
||||
Created supplier data with supplierNumber or None if failed
|
||||
"""
|
||||
if not self._check_write_permission("create_supplier"):
|
||||
return None
|
||||
|
||||
try:
|
||||
# Build supplier payload
|
||||
payload = {
|
||||
"name": supplier_data['name'],
|
||||
"currency": supplier_data.get('currency', 'DKK'),
|
||||
"supplierGroup": {
|
||||
"supplierGroupNumber": supplier_data.get('supplier_group_number', 1)
|
||||
},
|
||||
"paymentTerms": {
|
||||
"paymentTermsNumber": supplier_data.get('payment_terms_number', 4) # Netto 14 dage
|
||||
},
|
||||
"vatZone": {
|
||||
"vatZoneNumber": supplier_data.get('vat_zone_number', 1)
|
||||
}
|
||||
}
|
||||
|
||||
# Optional fields
|
||||
if supplier_data.get('address'):
|
||||
payload['address'] = supplier_data['address']
|
||||
if supplier_data.get('city'):
|
||||
payload['city'] = supplier_data['city']
|
||||
if supplier_data.get('zip'):
|
||||
payload['zip'] = supplier_data['zip']
|
||||
if supplier_data.get('country'):
|
||||
payload['country'] = supplier_data['country']
|
||||
if supplier_data.get('corporate_identification_number'):
|
||||
payload['corporateIdentificationNumber'] = supplier_data['corporate_identification_number']
|
||||
|
||||
url = f"{self.api_url}/suppliers"
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(url, headers=self._get_headers(), json=payload) as response:
|
||||
if response.status in [200, 201]:
|
||||
result = await response.json()
|
||||
logger.info(f"✅ Created supplier: {result.get('name')} (ID: {result.get('supplierNumber')})")
|
||||
|
||||
# Save to local vendors table
|
||||
try:
|
||||
from app.core.database import execute_insert
|
||||
|
||||
vendor_id = execute_insert("""
|
||||
INSERT INTO vendors (
|
||||
name,
|
||||
cvr,
|
||||
economic_supplier_number,
|
||||
created_at
|
||||
) VALUES (%s, %s, %s, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (economic_supplier_number)
|
||||
DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
cvr = EXCLUDED.cvr
|
||||
""", (
|
||||
result.get('name'),
|
||||
supplier_data.get('corporate_identification_number'),
|
||||
result.get('supplierNumber')
|
||||
))
|
||||
|
||||
logger.info(f"✅ Saved supplier to local database (vendor_id: {vendor_id})")
|
||||
except Exception as db_error:
|
||||
logger.warning(f"⚠️ Could not save to local database: {db_error}")
|
||||
|
||||
return result
|
||||
else:
|
||||
error_text = await response.text()
|
||||
logger.error(f"❌ Failed to create supplier: {response.status} - {error_text}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error creating supplier: {e}")
|
||||
return None
|
||||
|
||||
# ========== KASSEKLADDE (JOURNALS/VOUCHERS) ==========
|
||||
|
||||
async def get_supplier_invoice_journals(self) -> list:
|
||||
"""
|
||||
Get all available journals for supplier invoices (kassekladde)
|
||||
|
||||
Returns:
|
||||
List of journal dictionaries with journalNumber, name, and journalType
|
||||
"""
|
||||
try:
|
||||
url = f"{self.api_url}/journals"
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(url, headers=self._get_headers()) as response:
|
||||
if response.status != 200:
|
||||
error_text = await response.text()
|
||||
raise Exception(f"e-conomic API error: {response.status} - {error_text}")
|
||||
|
||||
data = await response.json()
|
||||
|
||||
# Filter for supplier invoice journals
|
||||
journals = []
|
||||
for journal in data.get('collection', []):
|
||||
journals.append({
|
||||
'journalNumber': journal.get('journalNumber'),
|
||||
'name': journal.get('name'),
|
||||
'journalType': journal.get('journalType')
|
||||
})
|
||||
|
||||
return journals
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error fetching journals: {e}")
|
||||
raise
|
||||
|
||||
async def create_journal_supplier_invoice(self,
|
||||
journal_number: int,
|
||||
supplier_number: int,
|
||||
invoice_number: str,
|
||||
invoice_date: str,
|
||||
total_amount: float,
|
||||
vat_breakdown: Dict[str, float],
|
||||
line_items: List[Dict] = None,
|
||||
due_date: Optional[str] = None,
|
||||
text: Optional[str] = None) -> Dict:
|
||||
"""
|
||||
Post supplier invoice to e-conomic kassekladde (journals API)
|
||||
|
||||
🚨 WRITE OPERATION - Respects READ_ONLY and DRY_RUN modes
|
||||
|
||||
Args:
|
||||
journal_number: Journal/kassekladde number (from system_settings)
|
||||
supplier_number: e-conomic supplier number
|
||||
invoice_number: Supplier's invoice number
|
||||
invoice_date: Invoice date (YYYY-MM-DD)
|
||||
total_amount: Total invoice amount including VAT
|
||||
vat_breakdown: Dict of {vat_code: {"net": X, "vat": Y, "gross": Z}} for each VAT group
|
||||
line_items: List of line items with contra_account and vat_code
|
||||
due_date: Payment due date (YYYY-MM-DD)
|
||||
text: Invoice description
|
||||
|
||||
Returns:
|
||||
Dict with voucher details or error info
|
||||
"""
|
||||
# 🚨 SAFETY CHECK
|
||||
if not self._check_write_permission("create_journal_supplier_invoice"):
|
||||
return {"error": True, "message": "Write operations blocked by READ_ONLY or DRY_RUN mode"}
|
||||
|
||||
try:
|
||||
# Extract year from invoice date for accounting year
|
||||
accounting_year = invoice_date[:4]
|
||||
|
||||
# Build supplier invoice entries - one per line item or per VAT group
|
||||
supplier_invoices = []
|
||||
|
||||
# If we have line items with contra accounts, use those
|
||||
if line_items and isinstance(line_items, list):
|
||||
# Group lines by VAT code and contra account combination
|
||||
line_groups = {}
|
||||
for line in line_items:
|
||||
vat_code = line.get('vat_code', 'I25')
|
||||
contra_account = line.get('contra_account', '5810')
|
||||
key = f"{vat_code}_{contra_account}"
|
||||
|
||||
if key not in line_groups:
|
||||
line_groups[key] = {
|
||||
'vat_code': vat_code,
|
||||
'contra_account': contra_account,
|
||||
'gross': 0,
|
||||
'vat': 0,
|
||||
'items': []
|
||||
}
|
||||
|
||||
line_total = line.get('line_total', 0)
|
||||
vat_amount = line.get('vat_amount', 0)
|
||||
|
||||
line_groups[key]['gross'] += line_total
|
||||
line_groups[key]['vat'] += vat_amount
|
||||
line_groups[key]['items'].append(line)
|
||||
|
||||
# Create entry for each group
|
||||
for key, group in line_groups.items():
|
||||
entry = {
|
||||
"supplier": {
|
||||
"supplierNumber": supplier_number
|
||||
},
|
||||
"amount": round(group['gross'], 2),
|
||||
"contraAccount": {
|
||||
"accountNumber": int(group['contra_account'])
|
||||
},
|
||||
"currency": {
|
||||
"code": "DKK"
|
||||
},
|
||||
"date": invoice_date,
|
||||
"supplierInvoiceNumber": invoice_number[:30] if invoice_number else ""
|
||||
}
|
||||
|
||||
# Add text with product descriptions
|
||||
descriptions = [item.get('description', '') for item in group['items'][:2]]
|
||||
entry_text = text if text else f"Faktura {invoice_number}"
|
||||
if descriptions:
|
||||
entry_text = f"{entry_text} - {', '.join(filter(None, descriptions))}"
|
||||
entry["text"] = entry_text[:250]
|
||||
|
||||
if due_date:
|
||||
entry["dueDate"] = due_date
|
||||
|
||||
# Add VAT details
|
||||
if group['vat'] > 0:
|
||||
entry["contraVatAccount"] = {
|
||||
"vatCode": group['vat_code']
|
||||
}
|
||||
entry["contraVatAmount"] = round(group['vat'], 2)
|
||||
|
||||
supplier_invoices.append(entry)
|
||||
|
||||
elif vat_breakdown and isinstance(vat_breakdown, dict):
|
||||
# Fallback: vat_breakdown format: {"I25": {"net": 1110.672, "vat": 277.668, "rate": 25, "gross": 1388.34}, ...}
|
||||
for vat_code, vat_data in vat_breakdown.items():
|
||||
if not isinstance(vat_data, dict):
|
||||
continue
|
||||
|
||||
net_amount = vat_data.get('net', 0)
|
||||
vat_amount = vat_data.get('vat', 0)
|
||||
gross_amount = vat_data.get('gross', net_amount + vat_amount)
|
||||
|
||||
if gross_amount <= 0:
|
||||
continue
|
||||
|
||||
entry = {
|
||||
"supplier": {
|
||||
"supplierNumber": supplier_number
|
||||
},
|
||||
"amount": round(gross_amount, 2),
|
||||
"contraAccount": {
|
||||
"accountNumber": 5810 # Default fallback account
|
||||
},
|
||||
"currency": {
|
||||
"code": "DKK"
|
||||
},
|
||||
"date": invoice_date,
|
||||
"supplierInvoiceNumber": invoice_number[:30] if invoice_number else ""
|
||||
}
|
||||
|
||||
# Add text with VAT code for clarity
|
||||
entry_text = text if text else f"Faktura {invoice_number}"
|
||||
if len(vat_breakdown) > 1:
|
||||
entry_text = f"{entry_text} ({vat_code})"
|
||||
entry["text"] = entry_text[:250]
|
||||
|
||||
if due_date:
|
||||
entry["dueDate"] = due_date
|
||||
|
||||
# Add VAT details
|
||||
if vat_amount > 0:
|
||||
entry["contraVatAccount"] = {
|
||||
"vatCode": vat_code
|
||||
}
|
||||
entry["contraVatAmount"] = round(vat_amount, 2)
|
||||
|
||||
supplier_invoices.append(entry)
|
||||
else:
|
||||
# No VAT breakdown - create single entry
|
||||
supplier_invoice = {
|
||||
"supplier": {
|
||||
"supplierNumber": supplier_number
|
||||
},
|
||||
"amount": total_amount,
|
||||
"contraAccount": {
|
||||
"accountNumber": 5810 # Default fallback account
|
||||
},
|
||||
"currency": {
|
||||
"code": "DKK"
|
||||
},
|
||||
"date": invoice_date,
|
||||
"supplierInvoiceNumber": invoice_number[:30] if invoice_number else ""
|
||||
}
|
||||
|
||||
if text:
|
||||
supplier_invoice["text"] = text[:250]
|
||||
if due_date:
|
||||
supplier_invoice["dueDate"] = due_date
|
||||
|
||||
supplier_invoices.append(supplier_invoice)
|
||||
|
||||
# Build voucher payload
|
||||
payload = {
|
||||
"accountingYear": {
|
||||
"year": accounting_year
|
||||
},
|
||||
"journal": {
|
||||
"journalNumber": journal_number
|
||||
},
|
||||
"entries": {
|
||||
"supplierInvoices": supplier_invoices
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(f"📤 Posting supplier invoice to journal {journal_number}")
|
||||
logger.debug(f"Payload: {json.dumps(payload, indent=2)}")
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(
|
||||
f"{self.api_url}/journals/{journal_number}/vouchers",
|
||||
headers=self._get_headers(),
|
||||
json=payload
|
||||
) as response:
|
||||
response_text = await response.text()
|
||||
|
||||
self._log_api_call(
|
||||
"POST",
|
||||
f"/journals/{journal_number}/vouchers",
|
||||
payload,
|
||||
await response.json() if response.status in [200, 201] and response_text else None,
|
||||
response.status
|
||||
)
|
||||
|
||||
if response.status in [200, 201]:
|
||||
data = await response.json() if response_text else {}
|
||||
|
||||
# e-conomic returns array of created vouchers
|
||||
if isinstance(data, list) and len(data) > 0:
|
||||
voucher_data = data[0]
|
||||
else:
|
||||
voucher_data = data
|
||||
|
||||
voucher_number = voucher_data.get('voucherNumber')
|
||||
logger.info(f"✅ Supplier invoice posted to kassekladde: voucher #{voucher_number}")
|
||||
return {
|
||||
"success": True,
|
||||
"voucher_number": voucher_number,
|
||||
"journal_number": journal_number,
|
||||
"accounting_year": accounting_year,
|
||||
"data": voucher_data
|
||||
}
|
||||
else:
|
||||
logger.error(f"❌ Post to kassekladde failed: {response.status}")
|
||||
logger.error(f"Response: {response_text}")
|
||||
return {
|
||||
"error": True,
|
||||
"status": response.status,
|
||||
"message": response_text
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ create_journal_supplier_invoice error: {e}")
|
||||
logger.exception("Full traceback:")
|
||||
return {"error": True, "status": 500, "message": str(e)}
|
||||
|
||||
async def upload_voucher_attachment(self,
|
||||
journal_number: int,
|
||||
accounting_year: str,
|
||||
voucher_number: int,
|
||||
pdf_path: str,
|
||||
filename: str) -> Dict:
|
||||
"""
|
||||
Upload PDF attachment to e-conomic voucher
|
||||
|
||||
🚨 WRITE OPERATION - Respects READ_ONLY and DRY_RUN modes
|
||||
|
||||
Args:
|
||||
journal_number: Journal number
|
||||
accounting_year: Accounting year (e.g., "2025")
|
||||
voucher_number: Voucher number
|
||||
pdf_path: Local path to PDF file
|
||||
filename: Filename for attachment
|
||||
|
||||
Returns:
|
||||
Dict with success status
|
||||
"""
|
||||
# 🚨 SAFETY CHECK
|
||||
if not self._check_write_permission("upload_voucher_attachment"):
|
||||
return {"error": True, "message": "Write operations blocked by READ_ONLY or DRY_RUN mode"}
|
||||
|
||||
try:
|
||||
# Read PDF file
|
||||
with open(pdf_path, 'rb') as f:
|
||||
pdf_data = f.read()
|
||||
|
||||
# e-conomic attachment/file endpoint (POST is allowed here, not on /attachment)
|
||||
url = f"{self.api_url}/journals/{journal_number}/vouchers/{accounting_year}-{voucher_number}/attachment/file"
|
||||
|
||||
headers = {
|
||||
'X-AppSecretToken': self.app_secret_token,
|
||||
'X-AgreementGrantToken': self.agreement_grant_token
|
||||
}
|
||||
|
||||
# Use multipart/form-data as required by e-conomic API
|
||||
form_data = aiohttp.FormData()
|
||||
form_data.add_field('file',
|
||||
pdf_data,
|
||||
filename=filename,
|
||||
content_type='application/pdf')
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.post(url, headers=headers, data=form_data) as response:
|
||||
if response.status in [200, 201, 204]:
|
||||
logger.info(f"📎 PDF attachment uploaded to voucher {accounting_year}-{voucher_number}")
|
||||
return {"success": True}
|
||||
else:
|
||||
error_text = await response.text()
|
||||
logger.error(f"❌ Failed to upload attachment: {response.status} - {error_text}")
|
||||
return {"error": True, "status": response.status, "message": error_text}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ upload_voucher_attachment error: {e}")
|
||||
return {"error": True, "message": str(e)}
|
||||
|
||||
|
||||
# Singleton instance
|
||||
_economic_service_instance = None
|
||||
|
||||
def get_economic_service() -> EconomicService:
|
||||
"""Get singleton instance of EconomicService"""
|
||||
global _economic_service_instance
|
||||
if _economic_service_instance is None:
|
||||
_economic_service_instance = EconomicService()
|
||||
return _economic_service_instance
|
||||
331
app/services/ollama_service.py
Normal file
331
app/services/ollama_service.py
Normal file
@ -0,0 +1,331 @@
|
||||
"""
|
||||
Ollama Integration Service for BMC Hub
|
||||
Handles supplier invoice extraction using Ollama LLM with CVR matching
|
||||
"""
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, List, Tuple
|
||||
from datetime import datetime
|
||||
import re
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.database import execute_insert, execute_query, execute_update
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class OllamaService:
|
||||
"""Service for extracting supplier invoice data using Ollama LLM"""
|
||||
|
||||
def __init__(self):
|
||||
self.endpoint = settings.OLLAMA_ENDPOINT
|
||||
self.model = settings.OLLAMA_MODEL
|
||||
self.system_prompt = self._build_system_prompt()
|
||||
logger.info(f"🤖 Initialized OllamaService: {self.endpoint}, model={self.model}")
|
||||
|
||||
def _build_system_prompt(self) -> str:
|
||||
"""Build Danish system prompt for invoice extraction with CVR"""
|
||||
return """Du er en ekspert i at læse og udtrække strukturerede data fra danske fakturaer og leverandørdokumenter.
|
||||
|
||||
VIGTIGE REGLER:
|
||||
1. Returner KUN gyldig JSON - ingen forklaring eller ekstra tekst
|
||||
2. Hvis et felt ikke findes, sæt det til null
|
||||
3. Beregn confidence baseret på hvor sikker du er på hvert felt (0.0-1.0)
|
||||
4. Datoer skal være i format YYYY-MM-DD
|
||||
5. Tal skal være decimaler (brug . som decimalseparator)
|
||||
6. CVR-nummer skal være 8 cifre uden mellemrum
|
||||
7. Moms/VAT skal udtrækkes fra hver linje hvis muligt
|
||||
|
||||
JSON format skal være:
|
||||
{
|
||||
"document_type": "invoice",
|
||||
"invoice_number": "fakturanummer",
|
||||
"vendor_name": "leverandør firmanavn",
|
||||
"vendor_cvr": "12345678",
|
||||
"invoice_date": "YYYY-MM-DD",
|
||||
"due_date": "YYYY-MM-DD",
|
||||
"currency": "DKK",
|
||||
"total_amount": 1234.56,
|
||||
"vat_amount": 123.45,
|
||||
"lines": [
|
||||
{
|
||||
"line_number": 1,
|
||||
"description": "beskrivelse af varen/ydelsen",
|
||||
"quantity": antal_som_tal,
|
||||
"unit_price": pris_per_stk,
|
||||
"line_total": total_for_linjen,
|
||||
"vat_rate": 25.00,
|
||||
"vat_amount": moms_beløb,
|
||||
"confidence": 0.0_til_1.0
|
||||
}
|
||||
],
|
||||
"confidence": gennemsnits_confidence,
|
||||
"raw_text_snippet": "første 200 tegn fra dokumentet"
|
||||
}
|
||||
|
||||
EKSEMPEL:
|
||||
Input: "FAKTURA 2025-001\\nGlobalConnect A/S\\nCVR: 12345678\\n1 stk Fiber 100/100 Mbit @ 299,00 DKK\\nMoms (25%): 74,75 DKK\\nTotal: 373,75 DKK"
|
||||
|
||||
Output: {
|
||||
"document_type": "invoice",
|
||||
"invoice_number": "2025-001",
|
||||
"vendor_name": "GlobalConnect A/S",
|
||||
"vendor_cvr": "12345678",
|
||||
"total_amount": 373.75,
|
||||
"vat_amount": 74.75,
|
||||
"lines": [{
|
||||
"line_number": 1,
|
||||
"description": "Fiber 100/100 Mbit",
|
||||
"quantity": 1,
|
||||
"unit_price": 299.00,
|
||||
"line_total": 299.00,
|
||||
"vat_rate": 25.00,
|
||||
"vat_amount": 74.75,
|
||||
"confidence": 0.95
|
||||
}],
|
||||
"confidence": 0.95
|
||||
}"""
|
||||
|
||||
async def extract_from_text(self, text: str) -> Dict:
|
||||
"""
|
||||
Extract structured invoice data from text using Ollama
|
||||
|
||||
Args:
|
||||
text: Document text content
|
||||
|
||||
Returns:
|
||||
Extracted data as dict with CVR, invoice number, amounts, etc.
|
||||
"""
|
||||
|
||||
# Truncate text if too long (keep first 4000 chars)
|
||||
if len(text) > 4000:
|
||||
text = text[:4000] + "\\n[... tekst afkortet ...]"
|
||||
|
||||
prompt = f"{self.system_prompt}\\n\\nNU SKAL DU UDTRÆKKE DATA FRA DENNE FAKTURA:\\n{text}\\n\\nReturner kun gyldig JSON:"
|
||||
|
||||
logger.info(f"🤖 Extracting invoice data from text (length: {len(text)})")
|
||||
|
||||
try:
|
||||
import httpx
|
||||
|
||||
async with httpx.AsyncClient(timeout=1000.0) as client:
|
||||
response = await client.post(
|
||||
f"{self.endpoint}/api/generate",
|
||||
json={
|
||||
"model": self.model,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {
|
||||
"temperature": 0.1,
|
||||
"top_p": 0.9,
|
||||
"num_predict": 2000
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Ollama returned status {response.status_code}: {response.text}")
|
||||
|
||||
result = response.json()
|
||||
raw_response = result.get("response", "")
|
||||
|
||||
logger.info(f"✅ Ollama extraction completed (response length: {len(raw_response)})")
|
||||
|
||||
# Parse JSON from response
|
||||
extraction = self._parse_json_response(raw_response)
|
||||
|
||||
# Add raw response for debugging
|
||||
extraction['_raw_llm_response'] = raw_response
|
||||
|
||||
return extraction
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Ollama extraction failed: {str(e)}"
|
||||
logger.error(f"❌ {error_msg}")
|
||||
|
||||
error_str = str(e).lower()
|
||||
if "timeout" in error_str:
|
||||
return {
|
||||
"error": f"Ollama timeout efter 1000 sekunder",
|
||||
"confidence": 0.0
|
||||
}
|
||||
elif "connection" in error_str or "connect" in error_str:
|
||||
return {
|
||||
"error": f"Kan ikke forbinde til Ollama på {self.endpoint}",
|
||||
"confidence": 0.0
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"error": error_msg,
|
||||
"confidence": 0.0
|
||||
}
|
||||
|
||||
def _parse_json_response(self, response: str) -> Dict:
|
||||
"""Parse JSON from LLM response with improved error handling"""
|
||||
try:
|
||||
# Find JSON in response (between first { and last })
|
||||
start = response.find('{')
|
||||
end = response.rfind('}') + 1
|
||||
|
||||
if start >= 0 and end > start:
|
||||
json_str = response[start:end]
|
||||
|
||||
# Try to fix common JSON issues
|
||||
# Remove trailing commas before } or ]
|
||||
json_str = re.sub(r',(\s*[}\]])', r'\1', json_str)
|
||||
# Fix single quotes to double quotes (but not in values)
|
||||
# This is risky, so we only do it if initial parse fails
|
||||
|
||||
try:
|
||||
data = json.loads(json_str)
|
||||
return data
|
||||
except json.JSONDecodeError:
|
||||
# Try to fix common issues
|
||||
# Replace single quotes with double quotes (simple approach)
|
||||
fixed_json = json_str.replace("'", '"')
|
||||
try:
|
||||
data = json.loads(fixed_json)
|
||||
logger.warning("⚠️ Fixed JSON with quote replacement")
|
||||
return data
|
||||
except:
|
||||
pass
|
||||
|
||||
# Last resort: log the problematic JSON
|
||||
logger.error(f"❌ Problematic JSON: {json_str[:300]}")
|
||||
raise
|
||||
else:
|
||||
raise ValueError("No JSON found in response")
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"❌ JSON parsing failed: {e}")
|
||||
logger.error(f"Raw response preview: {response[:500]}")
|
||||
return {
|
||||
"error": f"JSON parsing failed: {str(e)}",
|
||||
"confidence": 0.0,
|
||||
"raw_response": response[:500]
|
||||
}
|
||||
|
||||
def calculate_file_checksum(self, file_path: Path) -> str:
|
||||
"""Calculate SHA256 checksum of file for duplicate detection"""
|
||||
sha256 = hashlib.sha256()
|
||||
with open(file_path, 'rb') as f:
|
||||
while chunk := f.read(8192):
|
||||
sha256.update(chunk)
|
||||
checksum = sha256.hexdigest()
|
||||
logger.info(f"📋 Calculated checksum: {checksum[:16]}... for {file_path.name}")
|
||||
return checksum
|
||||
|
||||
async def _extract_text_from_file(self, file_path: Path) -> str:
|
||||
"""Extract text from PDF, image, or text file"""
|
||||
suffix = file_path.suffix.lower()
|
||||
|
||||
try:
|
||||
if suffix == '.pdf':
|
||||
return await self._extract_text_from_pdf(file_path)
|
||||
elif suffix in ['.png', '.jpg', '.jpeg']:
|
||||
return await self._extract_text_from_image(file_path)
|
||||
elif suffix in ['.txt', '.csv']:
|
||||
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
|
||||
return f.read()
|
||||
else:
|
||||
raise ValueError(f"Unsupported file type: {suffix}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Text extraction failed for {file_path.name}: {e}")
|
||||
raise
|
||||
|
||||
async def _extract_text_from_pdf(self, file_path: Path) -> str:
|
||||
"""Extract text from PDF using PyPDF2"""
|
||||
try:
|
||||
from PyPDF2 import PdfReader
|
||||
|
||||
reader = PdfReader(file_path)
|
||||
text = ""
|
||||
|
||||
for page_num, page in enumerate(reader.pages):
|
||||
page_text = page.extract_text()
|
||||
text += f"\\n--- Side {page_num + 1} ---\\n{page_text}"
|
||||
|
||||
logger.info(f"📄 Extracted {len(text)} chars from PDF with {len(reader.pages)} pages")
|
||||
return text
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ PDF extraction failed: {e}")
|
||||
raise
|
||||
|
||||
async def _extract_text_from_image(self, file_path: Path) -> str:
|
||||
"""Extract text from image using Tesseract OCR"""
|
||||
try:
|
||||
import pytesseract
|
||||
from PIL import Image
|
||||
|
||||
image = Image.open(file_path)
|
||||
|
||||
# Use Danish + English for OCR
|
||||
text = pytesseract.image_to_string(image, lang='dan+eng')
|
||||
|
||||
logger.info(f"🖼️ Extracted {len(text)} chars from image via OCR")
|
||||
return text
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ OCR extraction failed: {e}")
|
||||
# Fallback to English only
|
||||
try:
|
||||
text = pytesseract.image_to_string(Image.open(file_path), lang='eng')
|
||||
logger.warning(f"⚠️ Fallback to English OCR: {len(text)} chars")
|
||||
return text
|
||||
except:
|
||||
raise
|
||||
|
||||
def _get_mime_type(self, file_path: Path) -> str:
|
||||
"""Get MIME type from file extension"""
|
||||
suffix = file_path.suffix.lower()
|
||||
mime_types = {
|
||||
'.pdf': 'application/pdf',
|
||||
'.png': 'image/png',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.txt': 'text/plain',
|
||||
'.csv': 'text/csv'
|
||||
}
|
||||
return mime_types.get(suffix, 'application/octet-stream')
|
||||
|
||||
def match_vendor_by_cvr(self, vendor_cvr: Optional[str]) -> Optional[Dict]:
|
||||
"""
|
||||
Match vendor from database using CVR number
|
||||
|
||||
Args:
|
||||
vendor_cvr: CVR number from extraction
|
||||
|
||||
Returns:
|
||||
Vendor dict if found, None otherwise
|
||||
"""
|
||||
if not vendor_cvr:
|
||||
return None
|
||||
|
||||
# Clean CVR (remove spaces, dashes)
|
||||
cvr_clean = re.sub(r'[^0-9]', '', vendor_cvr)
|
||||
|
||||
if len(cvr_clean) != 8:
|
||||
logger.warning(f"⚠️ Invalid CVR format: {vendor_cvr} (cleaned: {cvr_clean})")
|
||||
return None
|
||||
|
||||
# Search vendors table
|
||||
vendor = execute_query(
|
||||
"SELECT * FROM vendors WHERE cvr = %s",
|
||||
(cvr_clean,),
|
||||
fetchone=True
|
||||
)
|
||||
|
||||
if vendor:
|
||||
logger.info(f"✅ Matched vendor: {vendor['name']} (CVR: {cvr_clean})")
|
||||
return vendor
|
||||
else:
|
||||
logger.info(f"⚠️ No vendor found with CVR: {cvr_clean}")
|
||||
return None
|
||||
|
||||
|
||||
# Global instance
|
||||
ollama_service = OllamaService()
|
||||
305
app/services/template_service.py
Normal file
305
app/services/template_service.py
Normal file
@ -0,0 +1,305 @@
|
||||
"""
|
||||
Supplier Invoice Template Service
|
||||
Simple template-based invoice field extraction (no AI)
|
||||
Inspired by OmniSync's invoice template system
|
||||
"""
|
||||
|
||||
import re
|
||||
import logging
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from app.core.database import execute_query, execute_insert, execute_update
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TemplateService:
|
||||
"""Service for template-based invoice extraction"""
|
||||
|
||||
def __init__(self):
|
||||
self.templates_cache = {}
|
||||
self._load_templates()
|
||||
|
||||
def _load_templates(self):
|
||||
"""Load all active templates into cache"""
|
||||
try:
|
||||
templates = execute_query(
|
||||
"""SELECT t.*, v.name as vendor_name, v.cvr as vendor_cvr
|
||||
FROM supplier_invoice_templates t
|
||||
LEFT JOIN vendors v ON t.vendor_id = v.id
|
||||
WHERE t.is_active = TRUE"""
|
||||
)
|
||||
|
||||
if templates:
|
||||
for template in templates:
|
||||
self.templates_cache[template['template_id']] = template
|
||||
logger.info(f"📚 Loaded {len(self.templates_cache)} active templates")
|
||||
else:
|
||||
logger.warning("⚠️ No templates found")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Failed to load templates: {e}")
|
||||
|
||||
def match_template(self, pdf_text: str) -> Tuple[Optional[int], float]:
|
||||
"""
|
||||
Find best matching template for PDF text
|
||||
Returns: (template_id, confidence_score)
|
||||
"""
|
||||
best_match = None
|
||||
best_score = 0.0
|
||||
pdf_text_lower = pdf_text.lower()
|
||||
|
||||
for template_id, template in self.templates_cache.items():
|
||||
score = self._calculate_match_score(pdf_text_lower, template)
|
||||
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_match = template_id
|
||||
|
||||
if best_match:
|
||||
logger.info(f"✅ Matched template {best_match} ({self.templates_cache[best_match]['template_name']}) with {best_score:.0%} confidence")
|
||||
|
||||
return best_match, best_score
|
||||
|
||||
def _calculate_match_score(self, pdf_text: str, template: Dict) -> float:
|
||||
"""Calculate match score based on detection patterns"""
|
||||
score = 0.0
|
||||
patterns = template.get('detection_patterns', [])
|
||||
|
||||
if not patterns:
|
||||
return 0.0
|
||||
|
||||
for pattern_obj in patterns:
|
||||
pattern_type = pattern_obj.get('type')
|
||||
weight = pattern_obj.get('weight', 0.5)
|
||||
|
||||
if pattern_type == 'text':
|
||||
# Simple text search
|
||||
pattern = pattern_obj.get('pattern', '').lower()
|
||||
if pattern in pdf_text:
|
||||
score += weight
|
||||
|
||||
elif pattern_type == 'cvr':
|
||||
# CVR number match (exact)
|
||||
cvr = str(pattern_obj.get('value', ''))
|
||||
if cvr in pdf_text:
|
||||
score += weight # CVR match is strong signal
|
||||
|
||||
elif pattern_type == 'regex':
|
||||
# Regex pattern match
|
||||
pattern = pattern_obj.get('pattern', '')
|
||||
if re.search(pattern, pdf_text, re.IGNORECASE):
|
||||
score += weight
|
||||
|
||||
return min(score, 1.0) # Cap at 100%
|
||||
|
||||
def extract_fields(self, pdf_text: str, template_id: int) -> Dict:
|
||||
"""Extract invoice fields using template's regex patterns"""
|
||||
template = self.templates_cache.get(template_id)
|
||||
if not template:
|
||||
logger.warning(f"⚠️ Template {template_id} not found in cache")
|
||||
return {}
|
||||
|
||||
field_mappings = template.get('field_mappings', {})
|
||||
extracted = {}
|
||||
|
||||
for field_name, field_config in field_mappings.items():
|
||||
pattern = field_config.get('pattern')
|
||||
group = field_config.get('group', 1)
|
||||
|
||||
if not pattern:
|
||||
continue
|
||||
|
||||
try:
|
||||
match = re.search(pattern, pdf_text, re.IGNORECASE | re.MULTILINE)
|
||||
if match and len(match.groups()) >= group:
|
||||
value = match.group(group).strip()
|
||||
extracted[field_name] = value
|
||||
logger.debug(f" ✓ {field_name}: {value}")
|
||||
except Exception as e:
|
||||
logger.warning(f" ✗ Failed to extract {field_name}: {e}")
|
||||
|
||||
return extracted
|
||||
|
||||
def extract_line_items(self, pdf_text: str, template_id: int) -> List[Dict]:
|
||||
"""Extract invoice line items using template's line patterns"""
|
||||
template = self.templates_cache.get(template_id)
|
||||
if not template:
|
||||
logger.warning(f"⚠️ Template {template_id} not found in cache")
|
||||
return []
|
||||
|
||||
field_mappings = template.get('field_mappings', {})
|
||||
|
||||
# Get line extraction config
|
||||
lines_start = field_mappings.get('lines_start', {}).get('pattern')
|
||||
lines_end = field_mappings.get('lines_end', {}).get('pattern')
|
||||
line_pattern = field_mappings.get('line_item', {}).get('pattern')
|
||||
line_fields = field_mappings.get('line_item', {}).get('fields', [])
|
||||
|
||||
if not line_pattern:
|
||||
logger.debug("No line_item pattern configured")
|
||||
return []
|
||||
|
||||
# Extract section between start and end markers
|
||||
text_section = pdf_text
|
||||
if lines_start:
|
||||
try:
|
||||
start_match = re.search(lines_start, pdf_text, re.IGNORECASE)
|
||||
if start_match:
|
||||
text_section = pdf_text[start_match.end():]
|
||||
logger.debug(f"Found lines_start, section starts at position {start_match.end()}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to find lines_start: {e}")
|
||||
|
||||
if lines_end:
|
||||
try:
|
||||
end_match = re.search(lines_end, text_section, re.IGNORECASE)
|
||||
if end_match:
|
||||
text_section = text_section[:end_match.start()]
|
||||
logger.debug(f"Found lines_end, section ends at position {end_match.start()}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to find lines_end: {e}")
|
||||
|
||||
# Try multiple extraction strategies
|
||||
lines = self._extract_with_pattern(text_section, line_pattern, line_fields)
|
||||
|
||||
if not lines:
|
||||
# Fallback: Try smart extraction for common formats
|
||||
lines = self._smart_line_extraction(text_section, line_fields)
|
||||
|
||||
logger.info(f"📦 Extracted {len(lines)} line items")
|
||||
return lines
|
||||
|
||||
def _extract_with_pattern(self, text: str, pattern: str, field_names: List[str]) -> List[Dict]:
|
||||
"""Extract lines using regex pattern"""
|
||||
lines = []
|
||||
try:
|
||||
for match in re.finditer(pattern, text, re.MULTILINE):
|
||||
line_data = {
|
||||
'line_number': len(lines) + 1,
|
||||
'raw_text': match.group(0)
|
||||
}
|
||||
|
||||
# Map captured groups to field names
|
||||
for idx, field_name in enumerate(field_names, start=1):
|
||||
if idx <= len(match.groups()):
|
||||
line_data[field_name] = match.group(idx).strip()
|
||||
|
||||
lines.append(line_data)
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Pattern extraction failed: {e}")
|
||||
|
||||
return lines
|
||||
|
||||
def _smart_line_extraction(self, text: str, field_names: List[str]) -> List[Dict]:
|
||||
"""
|
||||
Multi-line extraction for ALSO-style invoices.
|
||||
|
||||
Format:
|
||||
100 48023976 REFURB LENOVO ThinkPad P15 G1 Grde A
|
||||
...metadata lines...
|
||||
1ST 3.708,27 3.708,27
|
||||
|
||||
Combines data from description line + price line.
|
||||
"""
|
||||
lines_arr = text.split('\n')
|
||||
items = []
|
||||
i = 0
|
||||
|
||||
while i < len(lines_arr):
|
||||
line = lines_arr[i].strip()
|
||||
|
||||
# Find position + varenr + beskrivelse linje
|
||||
# Match: "100 48023976 REFURB LENOVO ThinkPad P15 G1 Grde A"
|
||||
item_match = re.match(r'^(\d{1,3})\s+(\d{6,})\s+(.+)', line)
|
||||
if item_match:
|
||||
position = item_match.group(1)
|
||||
item_number = item_match.group(2)
|
||||
description = item_match.group(3).strip()
|
||||
|
||||
# Skip hvis det er en header
|
||||
if re.search(r'(Position|Varenr|Beskrivelse|Antal|Pris|Total)', line, re.IGNORECASE):
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# Find næste linje med antal+priser (inden for 10 linjer)
|
||||
quantity = None
|
||||
unit_price = None
|
||||
total_price = None
|
||||
|
||||
for j in range(i+1, min(i+10, len(lines_arr))):
|
||||
price_line = lines_arr[j].strip()
|
||||
# Match: "1ST 3.708,27 3.708,27"
|
||||
price_match = re.match(r'^(\d+)\s*(?:ST|stk|pc|pcs)\s+([\d.,]+)\s+([\d.,]+)', price_line, re.IGNORECASE)
|
||||
if price_match:
|
||||
quantity = price_match.group(1)
|
||||
unit_price = price_match.group(2).replace(',', '.')
|
||||
total_price = price_match.group(3).replace(',', '.')
|
||||
break
|
||||
|
||||
# Kun tilføj hvis vi fandt priser
|
||||
if quantity and unit_price:
|
||||
items.append({
|
||||
'line_number': len(items) + 1,
|
||||
'position': position,
|
||||
'item_number': item_number,
|
||||
'description': description,
|
||||
'quantity': quantity,
|
||||
'unit_price': unit_price,
|
||||
'total_price': total_price,
|
||||
'raw_text': f"{line} ... {quantity}ST {unit_price} {total_price}"
|
||||
})
|
||||
logger.info(f"✅ Multi-line item: {item_number} - {description[:30]}... ({quantity}ST @ {unit_price})")
|
||||
|
||||
i += 1
|
||||
|
||||
if items:
|
||||
logger.info(f"📦 Multi-line extraction found {len(items)} items")
|
||||
else:
|
||||
logger.warning("⚠️ Multi-line extraction found no items")
|
||||
|
||||
return items
|
||||
|
||||
def log_usage(self, template_id: int, file_id: int, matched: bool,
|
||||
confidence: float, fields: Dict):
|
||||
"""Log template usage for statistics"""
|
||||
try:
|
||||
execute_insert(
|
||||
"""INSERT INTO template_usage_log
|
||||
(template_id, file_id, matched, confidence, fields_extracted)
|
||||
VALUES (%s, %s, %s, %s, %s)""",
|
||||
(template_id, file_id, matched, confidence, fields)
|
||||
)
|
||||
|
||||
if matched:
|
||||
# Update template stats
|
||||
execute_update(
|
||||
"""UPDATE supplier_invoice_templates
|
||||
SET usage_count = usage_count + 1,
|
||||
success_count = success_count + 1,
|
||||
last_used_at = CURRENT_TIMESTAMP
|
||||
WHERE template_id = %s""",
|
||||
(template_id,)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Failed to log template usage: {e}")
|
||||
|
||||
def get_vendor_templates(self, vendor_id: int) -> List[Dict]:
|
||||
"""Get all templates for a vendor"""
|
||||
return execute_query(
|
||||
"""SELECT * FROM supplier_invoice_templates
|
||||
WHERE vendor_id = %s AND is_active = TRUE
|
||||
ORDER BY usage_count DESC""",
|
||||
(vendor_id,),
|
||||
fetchall=True
|
||||
)
|
||||
|
||||
def reload_templates(self):
|
||||
"""Reload templates from database"""
|
||||
self.templates_cache = {}
|
||||
self._load_templates()
|
||||
|
||||
|
||||
# Global instance
|
||||
template_service = TemplateService()
|
||||
@ -205,6 +205,7 @@
|
||||
</a>
|
||||
<ul class="dropdown-menu mt-2">
|
||||
<li><a class="dropdown-item py-2" href="#">Fakturaer</a></li>
|
||||
<li><a class="dropdown-item py-2" href="/billing/supplier-invoices"><i class="bi bi-receipt me-2"></i>Kassekladde</a></li>
|
||||
<li><a class="dropdown-item py-2" href="#">Abonnementer</a></li>
|
||||
<li><a class="dropdown-item py-2" href="#">Betalinger</a></li>
|
||||
<li><hr class="dropdown-divider"></li>
|
||||
|
||||
363
docs/KASSEKLADDE.md
Normal file
363
docs/KASSEKLADDE.md
Normal file
@ -0,0 +1,363 @@
|
||||
# Kassekladde (Supplier Invoices) - BMC Hub
|
||||
|
||||
## Overview
|
||||
|
||||
BMC Hub's kassekladde module enables management of supplier invoices (incoming invoices that the company must pay) with full integration to e-conomic accounting system via the journals/vouchers API.
|
||||
|
||||
## Features
|
||||
|
||||
✅ **Complete CRUD Operations**
|
||||
- Create, view, update, and delete supplier invoices
|
||||
- Multi-line invoice support with VAT breakdown
|
||||
- Vendor linking and automatic creation in e-conomic
|
||||
|
||||
✅ **Approval Workflow**
|
||||
- Pending → Approved → Sent to e-conomic → Paid
|
||||
- Approval tracking with user and timestamp
|
||||
|
||||
✅ **e-conomic Integration**
|
||||
- Automatic supplier matching and creation
|
||||
- Journal/voucher posting to kassekladde
|
||||
- PDF attachment upload to vouchers
|
||||
- Configurable journal number and default accounts
|
||||
|
||||
✅ **VAT Handling**
|
||||
- Support for multiple VAT codes (I25, I0, IY25, etc.)
|
||||
- Automatic VAT calculation per line
|
||||
- VAT breakdown in e-conomic entries
|
||||
|
||||
✅ **Nordic Top UI**
|
||||
- Modern, clean design
|
||||
- Real-time statistics dashboard
|
||||
- Filter and search functionality
|
||||
- Responsive mobile support
|
||||
|
||||
## Architecture
|
||||
|
||||
### Database Schema
|
||||
|
||||
**Main Tables:**
|
||||
- `supplier_invoices` - Invoice headers with e-conomic tracking
|
||||
- `supplier_invoice_lines` - Line items with VAT and account details
|
||||
- `supplier_invoice_settings` - System configuration
|
||||
- `vendors` - Supplier information with e-conomic IDs
|
||||
|
||||
**Views:**
|
||||
- `overdue_supplier_invoices` - All overdue unpaid invoices
|
||||
- `pending_economic_sync` - Approved invoices ready for e-conomic
|
||||
|
||||
### Backend Structure
|
||||
|
||||
```
|
||||
app/
|
||||
billing/
|
||||
backend/
|
||||
supplier_invoices.py # FastAPI router with endpoints
|
||||
frontend/
|
||||
supplier_invoices.html # Nordic Top UI
|
||||
views.py # Frontend routes
|
||||
services/
|
||||
economic_service.py # e-conomic API integration
|
||||
```
|
||||
|
||||
### API Endpoints
|
||||
|
||||
**CRUD Operations:**
|
||||
- `GET /api/v1/supplier-invoices` - List invoices with filters
|
||||
- `GET /api/v1/supplier-invoices/{id}` - Get invoice details
|
||||
- `POST /api/v1/supplier-invoices` - Create new invoice
|
||||
- `PUT /api/v1/supplier-invoices/{id}` - Update invoice
|
||||
- `DELETE /api/v1/supplier-invoices/{id}` - Delete invoice
|
||||
|
||||
**Workflow Actions:**
|
||||
- `POST /api/v1/supplier-invoices/{id}/approve` - Approve invoice
|
||||
- `POST /api/v1/supplier-invoices/{id}/send-to-economic` - Send to e-conomic
|
||||
|
||||
**Statistics:**
|
||||
- `GET /api/v1/supplier-invoices/stats/overview` - Payment overview
|
||||
- `GET /api/v1/supplier-invoices/stats/by-vendor` - Stats by vendor
|
||||
|
||||
**e-conomic Integration:**
|
||||
- `GET /api/v1/supplier-invoices/economic/journals` - Available kassekladder
|
||||
|
||||
## Installation & Setup
|
||||
|
||||
### 1. Database Migration
|
||||
|
||||
Run the migration to create tables:
|
||||
|
||||
```bash
|
||||
# Execute migration SQL
|
||||
psql -U bmc_hub -d bmc_hub < migrations/008_supplier_invoices.sql
|
||||
```
|
||||
|
||||
Or via Docker:
|
||||
|
||||
```bash
|
||||
docker-compose exec postgres psql -U bmc_hub -d bmc_hub < /app/migrations/008_supplier_invoices.sql
|
||||
```
|
||||
|
||||
### 2. Configure e-conomic Credentials
|
||||
|
||||
Add to `.env` file:
|
||||
|
||||
```env
|
||||
# e-conomic Integration
|
||||
ECONOMIC_API_URL=https://restapi.e-conomic.com
|
||||
ECONOMIC_APP_SECRET_TOKEN=your_app_secret_token
|
||||
ECONOMIC_AGREEMENT_GRANT_TOKEN=your_agreement_grant_token
|
||||
|
||||
# Safety switches (ALWAYS start with these enabled)
|
||||
ECONOMIC_READ_ONLY=true
|
||||
ECONOMIC_DRY_RUN=true
|
||||
```
|
||||
|
||||
### 3. Configure Default Settings
|
||||
|
||||
The default journal number and accounts can be configured via database:
|
||||
|
||||
```sql
|
||||
-- Update default kassekladde number
|
||||
UPDATE supplier_invoice_settings
|
||||
SET setting_value = '1'
|
||||
WHERE setting_key = 'economic_default_journal';
|
||||
|
||||
-- Update default expense account
|
||||
UPDATE supplier_invoice_settings
|
||||
SET setting_value = '5810'
|
||||
WHERE setting_key = 'economic_default_contra_account';
|
||||
```
|
||||
|
||||
### 4. Restart Application
|
||||
|
||||
```bash
|
||||
docker-compose restart api
|
||||
```
|
||||
|
||||
## Usage Guide
|
||||
|
||||
### Creating a Supplier Invoice
|
||||
|
||||
1. Navigate to `/billing/supplier-invoices`
|
||||
2. Click "Ny Faktura" button
|
||||
3. Fill in required fields:
|
||||
- Invoice number (from supplier)
|
||||
- Vendor (select from dropdown)
|
||||
- Invoice date
|
||||
- Total amount (incl. VAT)
|
||||
4. Add line items with:
|
||||
- Description
|
||||
- Quantity & price
|
||||
- VAT code (25%, 0%, reverse charge, etc.)
|
||||
5. Click "Gem" to save
|
||||
|
||||
### Approval Workflow
|
||||
|
||||
**Status Flow:**
|
||||
```
|
||||
pending → approved → sent_to_economic → paid
|
||||
```
|
||||
|
||||
**Steps:**
|
||||
1. Invoice created → Status: `pending`
|
||||
2. Review and approve → Status: `approved`
|
||||
3. Send to e-conomic → Status: `sent_to_economic` (voucher created)
|
||||
4. Mark as paid → Status: `paid`
|
||||
|
||||
### Sending to e-conomic
|
||||
|
||||
**Prerequisites:**
|
||||
- Invoice must be `approved`
|
||||
- Vendor must exist (auto-created if needed)
|
||||
- At least one line item
|
||||
|
||||
**Process:**
|
||||
1. Click "Send til e-conomic" button
|
||||
2. System will:
|
||||
- Check/create vendor in e-conomic
|
||||
- Build VAT breakdown from lines
|
||||
- Create journal voucher entry
|
||||
- Upload PDF attachment (if available)
|
||||
- Update invoice with voucher number
|
||||
|
||||
**Result:**
|
||||
- Voucher created in e-conomic kassekladde
|
||||
- Invoice status → `sent_to_economic`
|
||||
- Voucher number stored for reference
|
||||
|
||||
## e-conomic Integration Details
|
||||
|
||||
### Safety Modes
|
||||
|
||||
**READ_ONLY Mode** (default: `true`)
|
||||
- Blocks ALL write operations to e-conomic
|
||||
- Only GET requests allowed
|
||||
- Use for testing API connection
|
||||
|
||||
**DRY_RUN Mode** (default: `true`)
|
||||
- Logs all operations but doesn't send to e-conomic
|
||||
- Full payload preview in logs
|
||||
- Safe for development/testing
|
||||
|
||||
**Production Mode** (both `false`)
|
||||
- Actually sends data to e-conomic
|
||||
- ⚠️ **Use with caution!**
|
||||
- Always test with dry-run first
|
||||
|
||||
### Journal Voucher Structure
|
||||
|
||||
e-conomic vouchers use this format:
|
||||
|
||||
```json
|
||||
{
|
||||
"accountingYear": {"year": "2025"},
|
||||
"journal": {"journalNumber": 1},
|
||||
"entries": {
|
||||
"supplierInvoices": [
|
||||
{
|
||||
"supplier": {"supplierNumber": 123},
|
||||
"amount": 1250.00,
|
||||
"contraAccount": {"accountNumber": 5810},
|
||||
"currency": {"code": "DKK"},
|
||||
"date": "2025-12-06",
|
||||
"dueDate": "2026-01-05",
|
||||
"supplierInvoiceNumber": "INV-12345",
|
||||
"text": "Invoice description",
|
||||
"contraVatAccount": {"vatCode": "I25"},
|
||||
"contraVatAmount": 250.00
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### VAT Code Mapping
|
||||
|
||||
| VAT Code | Description | Rate | Use Case |
|
||||
|----------|-------------|------|----------|
|
||||
| `I25` | Indenlandsk købsmoms 25% | 25% | Standard Danish purchases |
|
||||
| `I0` | Momsfri køb | 0% | VAT exempt |
|
||||
| `IY25` | Omvendt betalingspligt 25% | 25% | Reverse charge |
|
||||
| `IYEU` | Omvendt EU | 0% | EU reverse charge |
|
||||
| `IVEU` | Erhvervelse EU | 25% | EU acquisition |
|
||||
|
||||
### Account Number Mapping
|
||||
|
||||
Default expense accounts (can be customized per line):
|
||||
|
||||
- `5810` - Drift og materialer (default)
|
||||
- `5820` - IT og software
|
||||
- `5830` - Telefoni og internet
|
||||
- `5840` - Kontorartikler
|
||||
- `6000` - Løn og honorarer
|
||||
|
||||
## Development Guide
|
||||
|
||||
### Adding New Features
|
||||
|
||||
**1. Backend Endpoint:**
|
||||
|
||||
```python
|
||||
# app/billing/backend/supplier_invoices.py
|
||||
@router.post("/supplier-invoices/{invoice_id}/custom-action")
|
||||
async def custom_action(invoice_id: int):
|
||||
# Your logic here
|
||||
return {"success": True}
|
||||
```
|
||||
|
||||
**2. Frontend Integration:**
|
||||
|
||||
```javascript
|
||||
// supplier_invoices.html
|
||||
async function customAction(invoiceId) {
|
||||
const response = await fetch(`/api/v1/supplier-invoices/${invoiceId}/custom-action`, {
|
||||
method: 'POST'
|
||||
});
|
||||
// Handle response
|
||||
}
|
||||
```
|
||||
|
||||
### Testing e-conomic Integration
|
||||
|
||||
**1. Test Connection:**
|
||||
|
||||
```python
|
||||
from app.services.economic_service import get_economic_service
|
||||
|
||||
economic = get_economic_service()
|
||||
result = await economic.test_connection()
|
||||
# Should return True if credentials are valid
|
||||
```
|
||||
|
||||
**2. Test Dry-Run Mode:**
|
||||
|
||||
```bash
|
||||
# In .env
|
||||
ECONOMIC_READ_ONLY=false
|
||||
ECONOMIC_DRY_RUN=true
|
||||
```
|
||||
|
||||
Then create and approve an invoice, send to e-conomic. Check logs for full payload without actually posting.
|
||||
|
||||
**3. Production Test:**
|
||||
|
||||
```bash
|
||||
# WARNING: This will create real data in e-conomic!
|
||||
ECONOMIC_READ_ONLY=false
|
||||
ECONOMIC_DRY_RUN=false
|
||||
```
|
||||
|
||||
Start with a small test invoice to verify everything works.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Issue: "No journals found"
|
||||
|
||||
**Solution:** Check e-conomic credentials and ensure user has access to journals/kassekladder.
|
||||
|
||||
### Issue: "Supplier not found in e-conomic"
|
||||
|
||||
**Solution:** System will auto-create supplier if `ECONOMIC_DRY_RUN=false`. Verify vendor name is correct.
|
||||
|
||||
### Issue: "VAT validation failed"
|
||||
|
||||
**Solution:** Ensure VAT codes match e-conomic settings. Check `vat_code` in line items (I25, I0, etc.).
|
||||
|
||||
### Issue: "Voucher creation failed"
|
||||
|
||||
**Solution:**
|
||||
1. Check e-conomic API logs in application logs
|
||||
2. Verify journal number exists in e-conomic
|
||||
3. Ensure all required fields are present (supplier, amount, date)
|
||||
4. Check contra account number is valid
|
||||
|
||||
## Integration with OmniSync
|
||||
|
||||
This module is based on OmniSync's proven kassekladde implementation with the following enhancements:
|
||||
|
||||
- ✅ PostgreSQL instead of SQLite
|
||||
- ✅ Nordic Top design instead of custom CSS
|
||||
- ✅ Integrated with BMC Hub's vendor system
|
||||
- ✅ Simplified approval workflow
|
||||
- ✅ Better error handling and logging
|
||||
|
||||
## References
|
||||
|
||||
- **e-conomic API Docs:** https://restdocs.e-conomic.com/
|
||||
- **Journals API:** https://restdocs.e-conomic.com/#journals
|
||||
- **Vouchers API:** https://restdocs.e-conomic.com/#vouchers
|
||||
- **Suppliers API:** https://restdocs.e-conomic.com/#suppliers
|
||||
|
||||
## Support
|
||||
|
||||
For issues or questions:
|
||||
1. Check application logs: `docker-compose logs -f api`
|
||||
2. Review e-conomic API response in logs
|
||||
3. Test with DRY_RUN mode first
|
||||
4. Contact system administrator
|
||||
|
||||
---
|
||||
|
||||
**Last Updated:** December 6, 2025
|
||||
**Version:** 1.0.0
|
||||
**Maintained by:** BMC Networks Development Team
|
||||
315
docs/TEMPLATE_CREATION_FLOW.md
Normal file
315
docs/TEMPLATE_CREATION_FLOW.md
Normal file
@ -0,0 +1,315 @@
|
||||
# Perfect Template Creation Flow - BMC Hub
|
||||
|
||||
## 🎯 Workflow Overview
|
||||
|
||||
```
|
||||
Upload Invoice
|
||||
↓
|
||||
No Template Match? → AI Extracts (10s)
|
||||
↓
|
||||
"Opret Template" knap vises
|
||||
↓
|
||||
Click → Auto-creates template
|
||||
↓
|
||||
Next invoice: 0.1s (100x faster!)
|
||||
```
|
||||
|
||||
## 📋 Step-by-Step Guide
|
||||
|
||||
### Method 1: From Upload (Recommended for First-Time)
|
||||
|
||||
1. **Upload faktura** via `/billing/supplier-invoices`
|
||||
2. If no template matches:
|
||||
- AI extracts data (10s)
|
||||
- **"🪄 Opret Template" knap** vises
|
||||
3. Click "Opret Template"
|
||||
- System AI-analyzes PDF
|
||||
- Converts to template format
|
||||
- Creates template automatically
|
||||
4. **Redirect to Template Builder** for fine-tuning (optional)
|
||||
|
||||
**Benefits:**
|
||||
- ✅ One-click from upload success
|
||||
- ✅ Uses real invoice data
|
||||
- ✅ Automatic field mapping
|
||||
- ✅ Ready to use immediately
|
||||
|
||||
### Method 2: Template Builder Manual
|
||||
|
||||
1. Go to `/billing/template-builder`
|
||||
2. **Step 1:** Select PDF file
|
||||
3. **Step 2:** Choose vendor + name template
|
||||
4. **Step 3:** Click **"🤖 AI Auto-generer Template"**
|
||||
- AI analyzes PDF
|
||||
- Auto-fills all patterns
|
||||
- Shows detection confidence
|
||||
5. **Step 4:** Test & Save
|
||||
|
||||
**Benefits:**
|
||||
- ✅ More control
|
||||
- ✅ Preview before saving
|
||||
- ✅ Can edit patterns manually
|
||||
- ✅ Test against PDF first
|
||||
|
||||
### Method 3: Templates List (Batch)
|
||||
|
||||
1. Go to `/billing/templates`
|
||||
2. View existing templates
|
||||
3. Click "Test" to validate
|
||||
4. Create new from scratch
|
||||
|
||||
## 🔄 Complete User Journey
|
||||
|
||||
### First Invoice from New Vendor
|
||||
|
||||
```
|
||||
User uploads ALSO invoice #1
|
||||
↓
|
||||
System: "Ingen template match"
|
||||
↓
|
||||
AI extracts in 10s
|
||||
↓
|
||||
Shows: "✅ Faktura uploadet!"
|
||||
"⚠️ Ingen template - næste gang vil være langsom"
|
||||
[🪄 Opret Template knap]
|
||||
↓
|
||||
User clicks "Opret Template"
|
||||
↓
|
||||
System creates template automatically
|
||||
↓
|
||||
"✅ Template oprettet! Næste faktura vil være 100x hurtigere"
|
||||
```
|
||||
|
||||
### Second Invoice from Same Vendor
|
||||
|
||||
```
|
||||
User uploads ALSO invoice #2
|
||||
↓
|
||||
Template matches (0.1s) ⚡
|
||||
↓
|
||||
"✅ Faktura uploadet - auto-matched template!"
|
||||
NO template creation button (already exists)
|
||||
```
|
||||
|
||||
## 🎨 UI/UX Details
|
||||
|
||||
### Upload Success Message (No Template Match)
|
||||
|
||||
```html
|
||||
✅ Faktura Uploadet & Analyseret!
|
||||
|
||||
AI Udtrækning: [Vendor Badge] [Confidence Badge]
|
||||
Leverandør: ALSO A/S
|
||||
CVR: 17630903
|
||||
Fakturanr: 974733485
|
||||
Beløb: 5.165,61 DKK
|
||||
|
||||
⚠️ Ingen template match - fremtidige uploads vil være langsomme
|
||||
|
||||
[👁️ Vis Faktura] [🪄 Opret Template] [✅ Luk]
|
||||
```
|
||||
|
||||
### After Template Creation
|
||||
|
||||
```html
|
||||
✅ Template Oprettet!
|
||||
|
||||
Næste gang en faktura fra denne leverandør uploades, vil den blive
|
||||
behandlet automatisk på 0.1 sekunder i stedet for 10 sekunder!
|
||||
|
||||
Template ID: 42
|
||||
Fields: vendor_cvr, invoice_number, invoice_date, total_amount
|
||||
Detection patterns: 3
|
||||
|
||||
[✏️ Rediger Template] [📋 Se Alle Templates] [✅ Luk]
|
||||
```
|
||||
|
||||
## 🚀 Performance Metrics
|
||||
|
||||
| Scenario | Processing Time | User Action |
|
||||
|----------|----------------|-------------|
|
||||
| **No Template** | 10s (AI) | Manual → "Opret Template" |
|
||||
| **Template Match** | 0.1s (Regex) | None - automatic |
|
||||
| **Template Creation** | 15s total | One-click |
|
||||
|
||||
**ROI Example:**
|
||||
- 100 invoices from same vendor
|
||||
- Without template: 100 × 10s = **16.7 minutes**
|
||||
- With template: 1 × 15s + 99 × 0.1s = **~25 seconds**
|
||||
- **Time saved: 16 minutes!**
|
||||
|
||||
## 🧠 AI Auto-Generate Details
|
||||
|
||||
### Input
|
||||
```javascript
|
||||
{
|
||||
pdf_text: "ALSO A/S\nNummer 974733485\n...",
|
||||
vendor_id: 1
|
||||
}
|
||||
```
|
||||
|
||||
### AI Output (qwen2.5:3b - 10s)
|
||||
```json
|
||||
{
|
||||
"vendor_cvr": "17630903",
|
||||
"invoice_number": "974733485",
|
||||
"invoice_date": "30.06.2025",
|
||||
"total_amount": "5165.61",
|
||||
"detection_patterns": ["ALSO A/S", "Mårkærvej 2", "Faktura"],
|
||||
"lines_start": "Position Varenr. Beskrivelse",
|
||||
"lines_end": "Subtotal"
|
||||
}
|
||||
```
|
||||
|
||||
### Converted to Template Format
|
||||
```json
|
||||
{
|
||||
"vendor_id": 1,
|
||||
"template_name": "Auto-generated 2025-12-07",
|
||||
"detection_patterns": [
|
||||
{"type": "text", "pattern": "ALSO A/S", "weight": 0.5},
|
||||
{"type": "text", "pattern": "Mårkærvej 2", "weight": 0.3}
|
||||
],
|
||||
"field_mappings": {
|
||||
"vendor_cvr": {"pattern": "DK\\s*(\\d{8})", "group": 1},
|
||||
"invoice_number": {"pattern": "Nummer\\s*(\\d+)", "group": 1},
|
||||
"invoice_date": {"pattern": "Dato\\s*(\\d{1,2}[\\/.-]\\d{1,2}[\\/.-]\\d{4})", "group": 1},
|
||||
"total_amount": {"pattern": "Total\\s*([\\d.,]+)", "group": 1},
|
||||
"lines_start": {"pattern": "Position Varenr\\. Beskrivelse"},
|
||||
"lines_end": {"pattern": "Subtotal"}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## ✨ Smart Features
|
||||
|
||||
### 1. Auto-Detection of Template Need
|
||||
```javascript
|
||||
if (!result.template_matched && result.vendor_id) {
|
||||
showButton("Opret Template"); // Only if vendor exists
|
||||
}
|
||||
```
|
||||
|
||||
### 2. One-Click Creation
|
||||
- Fetches PDF text
|
||||
- AI analyzes
|
||||
- Converts format
|
||||
- Saves template
|
||||
- Shows success with edit link
|
||||
|
||||
### 3. Field Mapping Intelligence
|
||||
```javascript
|
||||
// Handles both nested and flat AI responses
|
||||
const cvrValue = aiData.vendor_cvr?.value || aiData.vendor_cvr || aiData.cvr;
|
||||
```
|
||||
|
||||
### 4. Multi-line Item Support
|
||||
- NO line_pattern in template
|
||||
- Uses smart multi-line extraction
|
||||
- Combines description + price lines automatically
|
||||
|
||||
## 🔧 Technical Implementation
|
||||
|
||||
### Frontend Flow
|
||||
```javascript
|
||||
async function createTemplateFromInvoice(invoiceId, vendorId) {
|
||||
// 1. Get PDF text
|
||||
const pdfData = await reprocess(invoiceId);
|
||||
|
||||
// 2. AI analyze
|
||||
const aiData = await aiAnalyze(pdfData.pdf_text, vendorId);
|
||||
|
||||
// 3. Convert to template format
|
||||
const template = convertAiToTemplate(aiData, vendorId);
|
||||
|
||||
// 4. Save
|
||||
await createTemplate(template);
|
||||
|
||||
// 5. Show success + edit link
|
||||
showSuccess(template.template_id);
|
||||
}
|
||||
```
|
||||
|
||||
### Backend Endpoints
|
||||
```
|
||||
POST /api/v1/supplier-invoices/reprocess/{id}
|
||||
→ Returns: {pdf_text, ...}
|
||||
|
||||
POST /api/v1/supplier-invoices/ai-analyze
|
||||
→ Input: {pdf_text, vendor_id}
|
||||
→ Returns: AI extracted fields
|
||||
|
||||
POST /api/v1/supplier-invoices/templates
|
||||
→ Input: {vendor_id, template_name, detection_patterns, field_mappings}
|
||||
→ Returns: {template_id, ...}
|
||||
```
|
||||
|
||||
## 📊 Success Metrics
|
||||
|
||||
Track these in template_usage_log:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
template_id,
|
||||
COUNT(*) as uses,
|
||||
AVG(CASE WHEN matched THEN 1 ELSE 0 END) as success_rate,
|
||||
AVG(confidence) as avg_confidence
|
||||
FROM template_usage_log
|
||||
GROUP BY template_id
|
||||
```
|
||||
|
||||
## 🎯 Best Practices
|
||||
|
||||
### For Users
|
||||
1. **Always create template** after first invoice from new vendor
|
||||
2. **Test template** with 2-3 invoices before trusting
|
||||
3. **Edit patterns** if confidence < 80%
|
||||
4. **Use descriptive names**: "ALSO Standard", "ALSO Email Format"
|
||||
|
||||
### For Admins
|
||||
1. Review auto-generated templates weekly
|
||||
2. Merge duplicate templates (same vendor, similar patterns)
|
||||
3. Disable low-performing templates (success_rate < 0.7)
|
||||
4. Keep AI model updated (qwen2.5:3b or better)
|
||||
|
||||
## 🚨 Edge Cases Handled
|
||||
|
||||
### Vendor Not Found
|
||||
- AI extracts CVR but vendor doesn't exist in DB
|
||||
- Shows warning: "Du skal oprette leverandør først"
|
||||
- No template creation button (needs vendor_id)
|
||||
|
||||
### AI Returns Incomplete Data
|
||||
- Template created with available fields only
|
||||
- Missing fields can be added manually later
|
||||
- Template still speeds up future processing
|
||||
|
||||
### Duplicate Templates
|
||||
- System allows multiple templates per vendor
|
||||
- Each can target different invoice formats
|
||||
- Detection patterns differentiate them
|
||||
|
||||
## 🎓 Training Users
|
||||
|
||||
### Quick Start Tutorial
|
||||
```
|
||||
1. Upload en faktura
|
||||
2. Klik "Opret Template" når den vises
|
||||
3. Næste gang = automatisk!
|
||||
```
|
||||
|
||||
### Power User Tips
|
||||
```
|
||||
- Brug Template Builder for bedre kontrol
|
||||
- Test templates før production
|
||||
- Kombiner AI + manual editing
|
||||
- Gennemgå templates månedligt
|
||||
```
|
||||
|
||||
## 📈 Future Enhancements
|
||||
|
||||
1. **Batch Template Creation**: Upload 10 PDFs → Create 10 templates
|
||||
2. **Template Suggestions**: "Found similar template - use this instead?"
|
||||
3. **Auto-Merge**: Detect duplicate templates and suggest merge
|
||||
4. **Confidence Tracking**: Dashboard showing template performance
|
||||
5. **A/B Testing**: Test pattern variations automatically
|
||||
213
docs/ai_template_prompt_example.md
Normal file
213
docs/ai_template_prompt_example.md
Normal file
@ -0,0 +1,213 @@
|
||||
# AI Template Generation - Perfect Prompt Example
|
||||
|
||||
## Prompt til Ollama/LLM
|
||||
|
||||
```
|
||||
OPGAVE: Analyser denne danske faktura og udtræk information til template-generering.
|
||||
|
||||
RETURNER KUN VALID JSON - ingen forklaring, ingen markdown, kun ren JSON!
|
||||
|
||||
REQUIRED JSON STRUKTUR:
|
||||
{
|
||||
"vendor_cvr": {
|
||||
"value": "17630903",
|
||||
"pattern": "DK\\s*(\\d{8})",
|
||||
"group": 1
|
||||
},
|
||||
"invoice_number": {
|
||||
"value": "974733485",
|
||||
"pattern": "Nummer\\s*(\\d+)",
|
||||
"group": 1
|
||||
},
|
||||
"invoice_date": {
|
||||
"value": "30.06.2025",
|
||||
"pattern": "Dato\\s*(\\d{1,2}[\\/.\\-]\\d{1,2}[\\/.\\-]\\d{4})",
|
||||
"group": 1,
|
||||
"format": "DD.MM.YYYY"
|
||||
},
|
||||
"total_amount": {
|
||||
"value": "5.165,61",
|
||||
"pattern": "Total\\s*([\\d.,]+)",
|
||||
"group": 1
|
||||
},
|
||||
"detection_patterns": [
|
||||
{"type": "text", "pattern": "ALSO A/S", "weight": 0.5},
|
||||
{"type": "text", "pattern": "Mårkærvej 2", "weight": 0.3},
|
||||
{"type": "text", "pattern": "Faktura", "weight": 0.2}
|
||||
],
|
||||
"lines_start": {
|
||||
"pattern": "Position Varenr\\. Beskrivelse Antal/Enhed"
|
||||
},
|
||||
"lines_end": {
|
||||
"pattern": "Subtotal|I alt ekskl\\. moms"
|
||||
}
|
||||
}
|
||||
|
||||
REGLER:
|
||||
1. Pattern skal være regex med escaped backslashes (\\s, \\d)
|
||||
2. Group angiver hvilken gruppe i regex der skal udtrækkes (1-baseret)
|
||||
3. Value skal være den faktiske værdi fundet i dokumentet
|
||||
4. Detection_patterns skal være 3-5 unikke tekststrenge der identificerer leverandøren
|
||||
5. lines_start er teksten LIGE FØR varelinjer starter
|
||||
6. lines_end er teksten EFTER varelinjer slutter
|
||||
7. LAV IKKE line_pattern - systemet bruger automatisk multi-line extraction
|
||||
|
||||
PDF TEKST:
|
||||
[PDF_CONTENT_HER]
|
||||
|
||||
RETURNER KUN JSON - intet andet!
|
||||
```
|
||||
|
||||
## Eksempel Response (det du skal få tilbage)
|
||||
|
||||
```json
|
||||
{
|
||||
"vendor_cvr": {
|
||||
"value": "17630903",
|
||||
"pattern": "DK\\s*(\\d{8})",
|
||||
"group": 1
|
||||
},
|
||||
"invoice_number": {
|
||||
"value": "974733485",
|
||||
"pattern": "Nummer\\s*(\\d+)",
|
||||
"group": 1
|
||||
},
|
||||
"invoice_date": {
|
||||
"value": "30.06.2025",
|
||||
"pattern": "Dato\\s*(\\d{1,2}[\\/.\\-]\\d{1,2}[\\/.\\-]\\d{4})",
|
||||
"group": 1,
|
||||
"format": "DD.MM.YYYY"
|
||||
},
|
||||
"total_amount": {
|
||||
"value": "5.165,61",
|
||||
"pattern": "beløb\\s*([\\d.,]+)",
|
||||
"group": 1
|
||||
},
|
||||
"detection_patterns": [
|
||||
{"type": "text", "pattern": "ALSO A/S", "weight": 0.5},
|
||||
{"type": "text", "pattern": "Mårkærvej 2", "weight": 0.3},
|
||||
{"type": "text", "pattern": "DK-2630 Taastrup", "weight": 0.2}
|
||||
],
|
||||
"lines_start": {
|
||||
"pattern": "Position Varenr\\. Beskrivelse Antal/Enhed Pris pr\\. enhed Total pris"
|
||||
},
|
||||
"lines_end": {
|
||||
"pattern": "Subtotal"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Hvordan bruges det i kode
|
||||
|
||||
```python
|
||||
import json
|
||||
import requests
|
||||
|
||||
pdf_text = "ALSO A/S\nMårkærvej 2\n2630 Taastrup\nNummer 974733485..."
|
||||
|
||||
prompt = f"""OPGAVE: Analyser denne danske faktura og udtræk information til template-generering.
|
||||
|
||||
RETURNER KUN VALID JSON - ingen forklaring, kun JSON!
|
||||
|
||||
REQUIRED JSON STRUKTUR:
|
||||
{{
|
||||
"vendor_cvr": {{"value": "17630903", "pattern": "DK\\\\s*(\\\\d{{8}})", "group": 1}},
|
||||
"invoice_number": {{"value": "974733485", "pattern": "Nummer\\\\s*(\\\\d+)", "group": 1}},
|
||||
"invoice_date": {{"value": "30.06.2025", "pattern": "Dato\\\\s*(\\\\d{{1,2}}[\\\\/.\\\\-]\\\\d{{1,2}}[\\\\/.\\\\-]\\\\d{{4}})", "group": 1}},
|
||||
"total_amount": {{"value": "5.165,61", "pattern": "Total\\\\s*([\\\\d.,]+)", "group": 1}},
|
||||
"detection_patterns": [{{"type": "text", "pattern": "ALSO A/S", "weight": 0.5}}],
|
||||
"lines_start": {{"pattern": "Position Varenr"}},
|
||||
"lines_end": {{"pattern": "Subtotal"}}
|
||||
}}
|
||||
|
||||
PDF TEKST:
|
||||
{pdf_text[:2000]}
|
||||
|
||||
RETURNER KUN JSON!"""
|
||||
|
||||
# Send til Ollama
|
||||
response = requests.post('http://localhost:11434/api/generate', json={
|
||||
'model': 'llama3.2',
|
||||
'prompt': prompt,
|
||||
'stream': False,
|
||||
'options': {'temperature': 0.1}
|
||||
})
|
||||
|
||||
result = json.loads(response.json()['response'])
|
||||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||||
```
|
||||
|
||||
## Test via curl
|
||||
|
||||
```bash
|
||||
# Hent PDF tekst
|
||||
PDF_TEXT=$(curl -s -X POST http://localhost:8001/api/v1/supplier-invoices/reprocess/4 | jq -r '.pdf_text')
|
||||
|
||||
# Send til AI endpoint
|
||||
curl -X POST http://localhost:8001/api/v1/supplier-invoices/ai-analyze \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"pdf_text\": \"$PDF_TEXT\", \"vendor_id\": 1}" | jq .
|
||||
```
|
||||
|
||||
## Tips for bedste resultater
|
||||
|
||||
1. **Brug temperature 0.1** - For konsistente JSON responses
|
||||
2. **Escaping**: Brug `\\\\s` i Python strings (bliver til `\\s` i JSON, `\s` i regex)
|
||||
3. **Specificer format**: Vis eksempel-output i prompten
|
||||
4. **Vis struktur**: Giv klar JSON struktur med alle required felter
|
||||
5. **Begræns tekst**: Kun første 2000 tegn (indeholder det vigtigste)
|
||||
6. **Validation**: Check at response er valid JSON før brug
|
||||
|
||||
## Konvertering til template format
|
||||
|
||||
AI returnerer nested format, men template vil have flat format:
|
||||
|
||||
```python
|
||||
ai_result = {
|
||||
"vendor_cvr": {"value": "17630903", "pattern": "DK\\s*(\\d{8})", "group": 1},
|
||||
"invoice_number": {"value": "974733485", "pattern": "Nummer\\s*(\\d+)", "group": 1}
|
||||
}
|
||||
|
||||
# Konverter til template field_mappings
|
||||
field_mappings = {}
|
||||
for field_name, config in ai_result.items():
|
||||
if field_name != 'detection_patterns':
|
||||
field_mappings[field_name] = {
|
||||
'pattern': config['pattern'],
|
||||
'group': config.get('group', 1)
|
||||
}
|
||||
if 'format' in config:
|
||||
field_mappings[field_name]['format'] = config['format']
|
||||
```
|
||||
|
||||
## Forventet output format
|
||||
|
||||
Template systemet forventer:
|
||||
|
||||
```json
|
||||
{
|
||||
"vendor_id": 1,
|
||||
"template_name": "ALSO A/S",
|
||||
"detection_patterns": [
|
||||
{"type": "text", "pattern": "ALSO A/S", "weight": 0.5}
|
||||
],
|
||||
"field_mappings": {
|
||||
"vendor_cvr": {
|
||||
"pattern": "DK\\s*(\\d{8})",
|
||||
"group": 1
|
||||
},
|
||||
"invoice_number": {
|
||||
"pattern": "Nummer\\s*(\\d+)",
|
||||
"group": 1
|
||||
},
|
||||
"lines_start": {
|
||||
"pattern": "Position Varenr"
|
||||
},
|
||||
"lines_end": {
|
||||
"pattern": "Subtotal"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
VIGTIGT: Ingen `line_item` pattern - systemet bruger automatisk multi-line extraction!
|
||||
2
main.py
2
main.py
@ -26,6 +26,7 @@ from app.settings.backend import router as settings_api
|
||||
from app.settings.backend import views as settings_views
|
||||
from app.hardware.backend import router as hardware_api
|
||||
from app.billing.backend import router as billing_api
|
||||
from app.billing.frontend import views as billing_views
|
||||
from app.system.backend import router as system_api
|
||||
from app.dashboard.backend import views as dashboard_views
|
||||
from app.dashboard.backend import router as dashboard_api
|
||||
@ -109,6 +110,7 @@ app.include_router(dashboard_views.router, tags=["Frontend"])
|
||||
app.include_router(customers_views.router, tags=["Frontend"])
|
||||
app.include_router(contacts_views.router, tags=["Frontend"])
|
||||
app.include_router(vendors_views.router, tags=["Frontend"])
|
||||
app.include_router(billing_views.router, tags=["Frontend"])
|
||||
app.include_router(settings_views.router, tags=["Frontend"])
|
||||
app.include_router(devportal_views.router, tags=["Frontend"])
|
||||
|
||||
|
||||
@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS vendors (
|
||||
country VARCHAR(100) DEFAULT 'Danmark',
|
||||
|
||||
-- Integration IDs
|
||||
economic_supplier_number INTEGER,
|
||||
economic_supplier_number INTEGER UNIQUE,
|
||||
|
||||
-- Vendor specific
|
||||
domain VARCHAR(255),
|
||||
|
||||
148
migrations/008_supplier_invoices.sql
Normal file
148
migrations/008_supplier_invoices.sql
Normal file
@ -0,0 +1,148 @@
|
||||
-- Migration 008: Supplier Invoices (Kassekladde / Leverandørfakturaer)
|
||||
-- Tables for tracking invoices WE RECEIVE from vendors (incoming invoices we must pay)
|
||||
-- Different from customer invoices which track invoices WE SEND to customers
|
||||
-- Integrates with e-conomic kassekladde (journals/vouchers API)
|
||||
|
||||
-- Main supplier invoices table
|
||||
CREATE TABLE IF NOT EXISTS supplier_invoices (
|
||||
id SERIAL PRIMARY KEY,
|
||||
invoice_number VARCHAR(100) NOT NULL,
|
||||
vendor_id INTEGER REFERENCES vendors(id),
|
||||
vendor_name VARCHAR(255),
|
||||
invoice_date DATE NOT NULL,
|
||||
due_date DATE,
|
||||
total_amount DECIMAL(15, 2) NOT NULL DEFAULT 0,
|
||||
vat_amount DECIMAL(15, 2) DEFAULT 0,
|
||||
net_amount DECIMAL(15, 2) DEFAULT 0,
|
||||
currency VARCHAR(10) DEFAULT 'DKK',
|
||||
description TEXT,
|
||||
notes TEXT,
|
||||
status VARCHAR(50) DEFAULT 'pending', -- pending, approved, sent_to_economic, paid, overdue, cancelled
|
||||
|
||||
-- e-conomic integration fields
|
||||
economic_supplier_number INTEGER, -- Supplier number in e-conomic
|
||||
economic_journal_number INTEGER, -- Kassekladde number used
|
||||
economic_voucher_number INTEGER, -- Voucher number in e-conomic
|
||||
economic_accounting_year VARCHAR(4), -- Accounting year (e.g., "2025")
|
||||
sent_to_economic_at TIMESTAMP,
|
||||
|
||||
-- Payment tracking
|
||||
paid_date DATE,
|
||||
payment_reference VARCHAR(100),
|
||||
|
||||
-- Approval workflow
|
||||
approved_by VARCHAR(255),
|
||||
approved_at TIMESTAMP,
|
||||
|
||||
-- File attachments
|
||||
file_path VARCHAR(500), -- Path to uploaded PDF/invoice file
|
||||
attachment_url VARCHAR(500), -- URL to external attachment
|
||||
|
||||
-- Metadata
|
||||
created_by VARCHAR(255),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Line items for supplier invoices with VAT breakdown
|
||||
CREATE TABLE IF NOT EXISTS supplier_invoice_lines (
|
||||
id SERIAL PRIMARY KEY,
|
||||
supplier_invoice_id INTEGER NOT NULL REFERENCES supplier_invoices(id) ON DELETE CASCADE,
|
||||
line_number INTEGER,
|
||||
description TEXT,
|
||||
quantity DECIMAL(10, 2) DEFAULT 1,
|
||||
unit_price DECIMAL(15, 2) DEFAULT 0,
|
||||
line_total DECIMAL(15, 2) DEFAULT 0,
|
||||
|
||||
-- VAT details per line
|
||||
vat_code VARCHAR(20) DEFAULT 'I25', -- e-conomic VAT codes: I25, I0, IY25, IYEU, IVEU, etc.
|
||||
vat_rate DECIMAL(5, 2) DEFAULT 25.00,
|
||||
vat_amount DECIMAL(15, 2) DEFAULT 0,
|
||||
|
||||
-- e-conomic account mapping
|
||||
contra_account VARCHAR(10) DEFAULT '5810', -- Default expense account
|
||||
|
||||
-- Product linking (optional)
|
||||
product_id INTEGER,
|
||||
sku VARCHAR(100),
|
||||
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- System settings for e-conomic kassekladde integration
|
||||
-- Add to existing system_settings table if it exists, or track separately
|
||||
CREATE TABLE IF NOT EXISTS supplier_invoice_settings (
|
||||
id SERIAL PRIMARY KEY,
|
||||
setting_key VARCHAR(100) UNIQUE NOT NULL,
|
||||
setting_value TEXT,
|
||||
description TEXT,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Insert default settings for kassekladde
|
||||
INSERT INTO supplier_invoice_settings (setting_key, setting_value, description)
|
||||
VALUES
|
||||
('economic_default_journal', '1', 'Default kassekladde nummer til leverandørfakturaer'),
|
||||
('economic_default_contra_account', '5810', 'Default expense account (drift/materialer)'),
|
||||
('auto_approve_under_amount', '1000', 'Auto-approve invoices under this amount (DKK)'),
|
||||
('require_attachment', 'true', 'Require PDF attachment before sending to e-conomic')
|
||||
ON CONFLICT (setting_key) DO NOTHING;
|
||||
|
||||
-- Indexes for performance
|
||||
CREATE INDEX IF NOT EXISTS idx_supplier_invoices_vendor ON supplier_invoices(vendor_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_supplier_invoices_status ON supplier_invoices(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_supplier_invoices_due_date ON supplier_invoices(due_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_supplier_invoices_economic_voucher ON supplier_invoices(economic_voucher_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_supplier_invoice_lines_invoice ON supplier_invoice_lines(supplier_invoice_id);
|
||||
|
||||
-- Trigger to update updated_at timestamp
|
||||
CREATE OR REPLACE FUNCTION update_supplier_invoice_timestamp()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER supplier_invoices_updated_at
|
||||
BEFORE UPDATE ON supplier_invoices
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_supplier_invoice_timestamp();
|
||||
|
||||
-- View for overdue supplier invoices
|
||||
CREATE OR REPLACE VIEW overdue_supplier_invoices AS
|
||||
SELECT
|
||||
si.*,
|
||||
v.name as vendor_full_name,
|
||||
v.economic_supplier_number as vendor_economic_id,
|
||||
(CURRENT_DATE - si.due_date) as days_overdue
|
||||
FROM supplier_invoices si
|
||||
LEFT JOIN vendors v ON si.vendor_id = v.id
|
||||
WHERE si.status IN ('pending', 'approved')
|
||||
AND si.due_date < CURRENT_DATE
|
||||
ORDER BY si.due_date ASC;
|
||||
|
||||
-- View for pending e-conomic sync
|
||||
CREATE OR REPLACE VIEW pending_economic_sync AS
|
||||
SELECT
|
||||
si.*,
|
||||
v.name as vendor_full_name,
|
||||
v.economic_supplier_number as vendor_economic_id,
|
||||
COUNT(sil.id) as line_count
|
||||
FROM supplier_invoices si
|
||||
LEFT JOIN vendors v ON si.vendor_id = v.id
|
||||
LEFT JOIN supplier_invoice_lines sil ON si.id = sil.supplier_invoice_id
|
||||
WHERE si.status = 'approved'
|
||||
AND si.economic_voucher_number IS NULL
|
||||
AND (
|
||||
(SELECT setting_value FROM supplier_invoice_settings WHERE setting_key = 'require_attachment') = 'false'
|
||||
OR si.file_path IS NOT NULL
|
||||
)
|
||||
GROUP BY si.id, v.name, v.economic_supplier_number
|
||||
ORDER BY si.invoice_date ASC;
|
||||
|
||||
COMMENT ON TABLE supplier_invoices IS 'Leverandørfakturaer - invoices received from vendors that we must pay';
|
||||
COMMENT ON TABLE supplier_invoice_lines IS 'Line items for supplier invoices with VAT and account details';
|
||||
COMMENT ON TABLE supplier_invoice_settings IS 'System settings for supplier invoice and e-conomic kassekladde integration';
|
||||
COMMENT ON VIEW overdue_supplier_invoices IS 'All unpaid supplier invoices past their due date';
|
||||
COMMENT ON VIEW pending_economic_sync IS 'Approved supplier invoices ready to be sent to e-conomic kassekladde';
|
||||
133
migrations/009_document_extraction.sql
Normal file
133
migrations/009_document_extraction.sql
Normal file
@ -0,0 +1,133 @@
|
||||
-- Migration 009: Document Extraction and Upload System
|
||||
-- Adds tables for file upload tracking, AI extraction, and duplicate prevention
|
||||
|
||||
-- Table: incoming_files
|
||||
-- Tracks all uploaded files with SHA256 checksums for duplicate detection
|
||||
CREATE TABLE IF NOT EXISTS incoming_files (
|
||||
file_id SERIAL PRIMARY KEY,
|
||||
filename VARCHAR(500) NOT NULL,
|
||||
original_filename VARCHAR(500) NOT NULL,
|
||||
file_path VARCHAR(1000),
|
||||
file_size INTEGER,
|
||||
mime_type VARCHAR(100),
|
||||
checksum VARCHAR(64) NOT NULL UNIQUE, -- SHA256 hash for duplicate detection
|
||||
status VARCHAR(50) DEFAULT 'pending', -- pending, processing, processed, failed, duplicate
|
||||
uploaded_by INTEGER, -- Future: REFERENCES users(user_id)
|
||||
uploaded_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
processed_at TIMESTAMP,
|
||||
error_message TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX idx_incoming_files_status ON incoming_files(status);
|
||||
CREATE INDEX idx_incoming_files_checksum ON incoming_files(checksum);
|
||||
CREATE INDEX idx_incoming_files_uploaded_at ON incoming_files(uploaded_at DESC);
|
||||
|
||||
-- Table: extractions
|
||||
-- Stores AI-extracted data from uploaded documents
|
||||
CREATE TABLE IF NOT EXISTS extractions (
|
||||
extraction_id SERIAL PRIMARY KEY,
|
||||
file_id INTEGER REFERENCES incoming_files(file_id) ON DELETE CASCADE,
|
||||
|
||||
-- Document metadata
|
||||
document_type VARCHAR(100), -- invoice, purchase_order, delivery_note, credit_note
|
||||
document_id VARCHAR(100), -- Invoice/order number extracted by AI
|
||||
vendor_name VARCHAR(255), -- Vendor name extracted by AI
|
||||
vendor_cvr VARCHAR(20), -- CVR number extracted by AI
|
||||
vendor_matched_id INTEGER REFERENCES vendors(id), -- Matched vendor from database
|
||||
match_confidence DECIMAL(5,4) CHECK (match_confidence >= 0.0 AND match_confidence <= 1.0),
|
||||
|
||||
-- Financial data
|
||||
document_date DATE,
|
||||
due_date DATE,
|
||||
currency VARCHAR(10) DEFAULT 'DKK',
|
||||
total_amount DECIMAL(12,2),
|
||||
vat_amount DECIMAL(12,2),
|
||||
|
||||
-- AI metadata
|
||||
confidence DECIMAL(5,4) CHECK (confidence >= 0.0 AND confidence <= 1.0),
|
||||
raw_text_snippet TEXT,
|
||||
llm_response_json TEXT, -- Full JSON response from Ollama
|
||||
|
||||
-- Processing status
|
||||
status VARCHAR(50) DEFAULT 'extracted', -- extracted, validated, matched, needs_review, converted
|
||||
reviewed_by INTEGER, -- Future: REFERENCES users(user_id)
|
||||
reviewed_at TIMESTAMP,
|
||||
|
||||
-- Timestamps
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX idx_extractions_file_id ON extractions(file_id);
|
||||
CREATE INDEX idx_extractions_vendor_matched_id ON extractions(vendor_matched_id);
|
||||
CREATE INDEX idx_extractions_status ON extractions(status);
|
||||
CREATE INDEX idx_extractions_document_id ON extractions(document_id);
|
||||
CREATE INDEX idx_extractions_vendor_cvr ON extractions(vendor_cvr);
|
||||
|
||||
-- Table: extraction_lines
|
||||
-- Stores individual line items extracted from invoices
|
||||
CREATE TABLE IF NOT EXISTS extraction_lines (
|
||||
line_id SERIAL PRIMARY KEY,
|
||||
extraction_id INTEGER NOT NULL REFERENCES extractions(extraction_id) ON DELETE CASCADE,
|
||||
line_number INTEGER NOT NULL,
|
||||
|
||||
-- Product identification
|
||||
sku VARCHAR(100),
|
||||
ean VARCHAR(13), -- 13-digit EAN barcode
|
||||
description TEXT,
|
||||
|
||||
-- Quantities and pricing
|
||||
quantity DECIMAL(10,2),
|
||||
unit_price DECIMAL(12,4),
|
||||
line_total DECIMAL(12,2),
|
||||
vat_rate DECIMAL(5,2), -- VAT percentage (25.00 for 25%)
|
||||
vat_amount DECIMAL(12,2),
|
||||
|
||||
-- AI metadata
|
||||
confidence DECIMAL(5,4) CHECK (confidence >= 0.0 AND confidence <= 1.0),
|
||||
|
||||
-- Timestamps
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
UNIQUE(extraction_id, line_number)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_extraction_lines_extraction_id ON extraction_lines(extraction_id);
|
||||
|
||||
-- Add file_id column to supplier_invoices (link to uploaded file)
|
||||
ALTER TABLE supplier_invoices
|
||||
ADD COLUMN IF NOT EXISTS extraction_id INTEGER REFERENCES extractions(extraction_id);
|
||||
|
||||
CREATE INDEX idx_supplier_invoices_extraction_id ON supplier_invoices(extraction_id);
|
||||
|
||||
-- Add UNIQUE constraint to prevent duplicate invoices (same vendor + invoice number)
|
||||
-- Drop existing index if it exists
|
||||
DROP INDEX IF EXISTS idx_supplier_invoices_vendor_invoice;
|
||||
|
||||
-- Create UNIQUE index on vendor_id + invoice_number (simple version without WHERE clause)
|
||||
CREATE UNIQUE INDEX idx_supplier_invoices_vendor_invoice
|
||||
ON supplier_invoices(vendor_id, invoice_number);
|
||||
|
||||
-- Trigger to update extractions.updated_at
|
||||
CREATE OR REPLACE FUNCTION update_extractions_updated_at()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trigger_update_extractions_updated_at ON extractions;
|
||||
CREATE TRIGGER trigger_update_extractions_updated_at
|
||||
BEFORE UPDATE ON extractions
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_extractions_updated_at();
|
||||
|
||||
-- Comments
|
||||
COMMENT ON TABLE incoming_files IS 'Tracks all uploaded files with SHA256 checksums for duplicate detection';
|
||||
COMMENT ON TABLE extractions IS 'Stores AI-extracted data from uploaded documents with CVR vendor matching';
|
||||
COMMENT ON TABLE extraction_lines IS 'Stores individual line items extracted from invoices';
|
||||
COMMENT ON COLUMN incoming_files.checksum IS 'SHA256 hash of file content - prevents duplicate uploads';
|
||||
COMMENT ON COLUMN extractions.vendor_cvr IS 'CVR number extracted by AI from invoice';
|
||||
COMMENT ON COLUMN extractions.vendor_matched_id IS 'Matched vendor from vendors table based on CVR number';
|
||||
COMMENT ON COLUMN extractions.match_confidence IS 'Confidence score for vendor matching (0.0-1.0)';
|
||||
60
migrations/010_supplier_invoice_templates.sql
Normal file
60
migrations/010_supplier_invoice_templates.sql
Normal file
@ -0,0 +1,60 @@
|
||||
-- Migration 010: Supplier Invoice Templates
|
||||
-- Template-based invoice recognition (inspired by OmniSync)
|
||||
|
||||
-- Vendor templates for automatic invoice field extraction
|
||||
CREATE TABLE IF NOT EXISTS supplier_invoice_templates (
|
||||
template_id SERIAL PRIMARY KEY,
|
||||
vendor_id INTEGER REFERENCES vendors(id) ON DELETE CASCADE,
|
||||
template_name VARCHAR(255) NOT NULL,
|
||||
|
||||
-- Detection patterns (JSON array of patterns to identify this template)
|
||||
detection_patterns JSONB DEFAULT '[]',
|
||||
-- Example: [
|
||||
-- {"type": "text", "pattern": "BMC Denmark ApS", "weight": 0.5},
|
||||
-- {"type": "cvr", "value": "12345678", "weight": 1.0}
|
||||
-- ]
|
||||
|
||||
-- Field extraction rules (regex patterns with capture groups)
|
||||
field_mappings JSONB DEFAULT '{}',
|
||||
-- Example: {
|
||||
-- "invoice_number": {"pattern": "Faktura\\s*:?\\s*(\\d+)", "group": 1},
|
||||
-- "invoice_date": {"pattern": "(\\d{2}[/-]\\d{2}[/-]\\d{2,4})", "format": "DD/MM/YYYY", "group": 1},
|
||||
-- "total_amount": {"pattern": "Total\\s*:?\\s*([\\d.,]+)", "group": 1}
|
||||
-- }
|
||||
|
||||
-- Statistics
|
||||
usage_count INTEGER DEFAULT 0,
|
||||
success_count INTEGER DEFAULT 0,
|
||||
last_used_at TIMESTAMP,
|
||||
|
||||
-- Metadata
|
||||
is_active BOOLEAN DEFAULT TRUE,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
UNIQUE(vendor_id, template_name)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_supplier_invoice_templates_vendor ON supplier_invoice_templates(vendor_id);
|
||||
CREATE INDEX idx_supplier_invoice_templates_active ON supplier_invoice_templates(is_active);
|
||||
|
||||
-- Add template reference to incoming_files
|
||||
ALTER TABLE incoming_files ADD COLUMN IF NOT EXISTS template_id INTEGER REFERENCES supplier_invoice_templates(template_id);
|
||||
|
||||
-- Template usage tracking
|
||||
CREATE TABLE IF NOT EXISTS template_usage_log (
|
||||
log_id SERIAL PRIMARY KEY,
|
||||
template_id INTEGER REFERENCES supplier_invoice_templates(template_id) ON DELETE CASCADE,
|
||||
file_id INTEGER REFERENCES incoming_files(file_id) ON DELETE CASCADE,
|
||||
matched BOOLEAN DEFAULT FALSE,
|
||||
confidence DECIMAL(3,2),
|
||||
fields_extracted JSONB,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX idx_template_usage_log_template ON template_usage_log(template_id);
|
||||
CREATE INDEX idx_template_usage_log_file ON template_usage_log(file_id);
|
||||
|
||||
COMMENT ON TABLE supplier_invoice_templates IS 'Templates for automatic invoice field extraction per vendor';
|
||||
COMMENT ON COLUMN supplier_invoice_templates.detection_patterns IS 'JSON array of patterns to detect this template in PDF text';
|
||||
COMMENT ON COLUMN supplier_invoice_templates.field_mappings IS 'JSON object with regex patterns to extract invoice fields';
|
||||
2
nohup.out
Normal file
2
nohup.out
Normal file
@ -0,0 +1,2 @@
|
||||
INFO: Will watch for changes in these directories: ['/Users/christianthomas/DEV/bmc_hub_dev']
|
||||
ERROR: [Errno 48] Address already in use
|
||||
@ -8,3 +8,9 @@ python-multipart==0.0.17
|
||||
jinja2==3.1.4
|
||||
pyjwt==2.9.0
|
||||
aiohttp==3.10.10
|
||||
|
||||
# AI & Document Processing
|
||||
httpx==0.27.2
|
||||
PyPDF2==3.0.1
|
||||
pytesseract==0.3.13
|
||||
Pillow==11.0.0
|
||||
|
||||
Loading…
Reference in New Issue
Block a user