310 lines
18 KiB
Python
310 lines
18 KiB
Python
|
"""Local, atomic customer consolidation; never writes to external systems.
|
|||
|
|
|
|||
|
|
Relations are discovered from the actual schema, including legacy integer Hub
|
|||
|
|
references without foreign keys. Conflicts abort the entire operation. Preview
|
|||
|
|
executes the same transaction and rolls it back, including trigger effects.
|
|||
|
|
"""
|
|||
|
|
import hashlib
|
|||
|
|
import json
|
|||
|
|
import logging
|
|||
|
|
from contextlib import contextmanager
|
|||
|
|
|
|||
|
|
import psycopg2
|
|||
|
|
from psycopg2 import sql
|
|||
|
|
from psycopg2.extras import RealDictCursor, Json
|
|||
|
|
from fastapi import APIRouter, Depends, HTTPException
|
|||
|
|
from pydantic import BaseModel, Field
|
|||
|
|
|
|||
|
|
from app.core.database import get_db_connection, release_db_connection
|
|||
|
|
from app.core.auth_dependencies import require_permission
|
|||
|
|
from app.core.config import settings
|
|||
|
|
|
|||
|
|
logger = logging.getLogger(__name__)
|
|||
|
|
router = APIRouter()
|
|||
|
|
|
|||
|
|
|
|||
|
|
class MergeSelection(BaseModel):
|
|||
|
|
target_id: int = Field(gt=0)
|
|||
|
|
source_ids: list[int] = Field(min_length=1, max_length=50)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class MergeConfirmation(MergeSelection):
|
|||
|
|
preview_token: str = Field(min_length=64, max_length=64)
|
|||
|
|
|
|||
|
|
|
|||
|
|
@contextmanager
|
|||
|
|
def transaction(commit=False):
|
|||
|
|
conn = get_db_connection()
|
|||
|
|
try:
|
|||
|
|
with conn.cursor(cursor_factory=RealDictCursor) as cursor:
|
|||
|
|
cursor.execute("SET LOCAL lock_timeout = '5s'")
|
|||
|
|
cursor.execute("SET LOCAL statement_timeout = '30s'")
|
|||
|
|
yield cursor
|
|||
|
|
if commit:
|
|||
|
|
conn.commit()
|
|||
|
|
else:
|
|||
|
|
conn.rollback()
|
|||
|
|
except BaseException:
|
|||
|
|
conn.rollback()
|
|||
|
|
raise
|
|||
|
|
finally:
|
|||
|
|
release_db_connection(conn)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def selections(target, sources):
|
|||
|
|
sources = sorted(set(sources))
|
|||
|
|
if not sources or target in sources or any(i <= 0 for i in sources):
|
|||
|
|
raise HTTPException(422, 'Vælg en hovedkunde og mindst én anden kunde.')
|
|||
|
|
return sources
|
|||
|
|
|
|||
|
|
|
|||
|
|
def relations(cur):
|
|||
|
|
"""Only real tables in current schema; never confuse tmodule customer IDs.
|
|||
|
|
|
|||
|
|
A numeric customer reference with an FK to another entity is excluded.
|
|||
|
|
Partition children are handled through their parent, avoiding double moves.
|
|||
|
|
"""
|
|||
|
|
cur.execute("""
|
|||
|
|
SELECT c.relname AS table_name, a.attname AS column_name,
|
|||
|
|
EXISTS(SELECT 1 FROM pg_attribute x WHERE x.attrelid=c.oid
|
|||
|
|
AND x.attname='entity_type' AND NOT x.attisdropped) AS polymorphic,
|
|||
|
|
EXISTS(SELECT 1 FROM pg_constraint f WHERE f.conrelid=c.oid AND f.contype='f'
|
|||
|
|
AND a.attnum=ANY(f.conkey) AND f.confrelid='customers'::regclass) AS customer_fk,
|
|||
|
|
EXISTS(SELECT 1 FROM pg_constraint f WHERE f.conrelid=c.oid AND f.contype='f'
|
|||
|
|
AND a.attnum=ANY(f.conkey) AND f.confrelid<>'customers'::regclass) AS other_fk
|
|||
|
|
FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
|
|||
|
|
JOIN pg_attribute a ON a.attrelid=c.oid AND a.attnum>0 AND NOT a.attisdropped
|
|||
|
|
WHERE n.nspname=current_schema() AND c.relkind IN ('r','p') AND NOT c.relispartition
|
|||
|
|
AND c.relname<>'customer_merge_history'
|
|||
|
|
AND a.atttypid IN ('int2'::regtype,'int4'::regtype,'int8'::regtype)
|
|||
|
|
AND (a.attname='customer_id' OR a.attname LIKE '%%_customer_id'
|
|||
|
|
OR a.attname='entity_id' OR EXISTS(
|
|||
|
|
SELECT 1 FROM pg_constraint f WHERE f.conrelid=c.oid AND f.contype='f'
|
|||
|
|
AND a.attnum=ANY(f.conkey) AND f.confrelid='customers'::regclass))
|
|||
|
|
ORDER BY c.relname,a.attname
|
|||
|
|
""")
|
|||
|
|
return [r for r in cur.fetchall() if not r['other_fk']
|
|||
|
|
and r['column_name'] != 'merged_into_customer_id'
|
|||
|
|
and (r['column_name'] != 'entity_id' or r['polymorphic'])
|
|||
|
|
# These logs use IDs from the time module, not Hub IDs.
|
|||
|
|
and not (r['table_name'] in ('tmodule_sync_log', 'tticket_audit_log')
|
|||
|
|
and r['column_name'] == 'entity_id')]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def where(ref):
|
|||
|
|
clause = sql.SQL('{} = ANY(%s)').format(sql.Identifier(ref['column_name']))
|
|||
|
|
if ref['column_name'] == 'entity_id':
|
|||
|
|
clause += sql.SQL(" AND entity_type='customer'")
|
|||
|
|
return clause
|
|||
|
|
|
|||
|
|
|
|||
|
|
def signature(value):
|
|||
|
|
return hashlib.sha256(json.dumps(value, sort_keys=True, default=str).encode()).hexdigest()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def mentions_customer(value, sources):
|
|||
|
|
if isinstance(value,list):
|
|||
|
|
return any(mentions_customer(v,sources) for v in value)
|
|||
|
|
if not isinstance(value,dict):
|
|||
|
|
return False
|
|||
|
|
candidates=[value.get('customer_id')]
|
|||
|
|
if value.get('field') == 'customer_id':
|
|||
|
|
candidates.extend(value.get('value') if isinstance(value.get('value'),list) else [value.get('value')])
|
|||
|
|
return any(str(v) in {str(i) for i in sources} for v in candidates if v is not None) or any(
|
|||
|
|
mentions_customer(v,sources) for v in value.values() if isinstance(v,(dict,list)))
|
|||
|
|
|
|||
|
|
|
|||
|
|
IDENTITY_FIELDS = ('economic_customer_number', 'vtiger_id', 'vtiger_account_no', 'cvr_number')
|
|||
|
|
IDENTITY_LABELS = {'economic_customer_number':'e-conomic-kundenumre','vtiger_id':'vTiger-id’er',
|
|||
|
|
'vtiger_account_no':'vTiger-kundenumre','cvr_number':'CVR-numre'}
|
|||
|
|
FILL_FIELDS = ('email', 'phone', 'address', 'email_domain', 'city', 'postal_code',
|
|||
|
|
'country', 'website', 'invoice_email', 'mobile_phone', 'wiki_slug')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def field_plan(customers, target):
|
|||
|
|
changes, differences = {}, []
|
|||
|
|
for field in IDENTITY_FIELDS + FILL_FIELDS:
|
|||
|
|
values = {str(c[field]).strip(): c[field] for c in customers if c.get(field) not in (None, '')}
|
|||
|
|
if field in IDENTITY_FIELDS and len(values) > 1:
|
|||
|
|
raise HTTPException(409, f'Kunderne har forskellige {IDENTITY_LABELS[field]}. Afklar relationen før sammenlægning.')
|
|||
|
|
if not target.get(field) and len(values) == 1:
|
|||
|
|
changes[field] = next(iter(values.values()))
|
|||
|
|
elif len(values) > 1:
|
|||
|
|
differences.append(field)
|
|||
|
|
excluded=set(IDENTITY_FIELDS+FILL_FIELDS) | {'id','created_at','updated_at','deleted_at','last_synced_at',
|
|||
|
|
'merged_into_customer_id','parent_customer_id','billing_customer_id'}
|
|||
|
|
for field in sorted(set().union(*(c.keys() for c in customers))-excluded):
|
|||
|
|
values={json.dumps(c.get(field),default=str,sort_keys=True) for c in customers}
|
|||
|
|
if len(values)>1: differences.append(field)
|
|||
|
|
return changes, differences
|
|||
|
|
|
|||
|
|
|
|||
|
|
def prepare(cur, target_id, source_ids):
|
|||
|
|
cur.execute("SELECT to_regclass('customer_merge_history') AS history")
|
|||
|
|
if not cur.fetchone()['history']:
|
|||
|
|
raise HTTPException(503, 'Kundesammenlægning kræver migration 1074_customer_merge.sql.')
|
|||
|
|
# Consistent locking order for merges; FK inserts are serialized with these locks.
|
|||
|
|
cur.execute('SELECT * FROM customers WHERE id=ANY(%s) ORDER BY id FOR UPDATE',
|
|||
|
|
(sorted([target_id] + source_ids),))
|
|||
|
|
customers = [dict(r) for r in cur.fetchall()]
|
|||
|
|
if len(customers) != len(source_ids) + 1:
|
|||
|
|
raise HTTPException(404, 'En valgt kunde findes ikke længere.')
|
|||
|
|
if any(c.get('merged_into_customer_id') or c.get('deleted_at') or c.get('is_active') is False for c in customers):
|
|||
|
|
raise HTTPException(409, 'En valgt kunde er allerede samlet, slettet eller deaktiveret. Genindlæs listen.')
|
|||
|
|
target = next(c for c in customers if c['id'] == target_id)
|
|||
|
|
changes, differences = field_plan(customers, target)
|
|||
|
|
refs = relations(cur)
|
|||
|
|
# Lock relation tables so legacy references without FKs cannot change mid-merge.
|
|||
|
|
tables = {r['table_name'] for r in refs} | {'customers'}
|
|||
|
|
for table in ('bmc_workflows','bmc_workflow_versions','bmc_workflow_events','bmc_workflow_runtime'):
|
|||
|
|
cur.execute('SELECT to_regclass(%s) AS name',(table,))
|
|||
|
|
if cur.fetchone()['name']: tables.add(table)
|
|||
|
|
for table in sorted(tables):
|
|||
|
|
cur.execute(sql.SQL('LOCK TABLE {} IN SHARE ROW EXCLUSIVE MODE').format(sql.Identifier(table)))
|
|||
|
|
state, movements, origins = [], [], []
|
|||
|
|
for ref in refs:
|
|||
|
|
cur.execute(sql.SQL('SELECT * FROM {} WHERE {}').format(sql.Identifier(ref['table_name']),where(ref)),
|
|||
|
|
([target_id]+source_ids,))
|
|||
|
|
rows = [dict(r) for r in cur.fetchall()]
|
|||
|
|
if ref['table_name']=='ordre_drafts':
|
|||
|
|
economic=changes.get('economic_customer_number',target.get('economic_customer_number'))
|
|||
|
|
if any(r.get('exported_economic_customer_number') not in (None,'') and
|
|||
|
|
str(r['exported_economic_customer_number']).strip()!=str(economic).strip() for r in rows):
|
|||
|
|
raise HTTPException(409,'En ordre eller fakturakladde er eksporteret til et andet e-conomic-kundenummer. Afklar fakturarelationen før sammenlægning.')
|
|||
|
|
if rows:
|
|||
|
|
state.append({'table':ref['table_name'],'column':ref['column_name'],
|
|||
|
|
'rows':sorted(rows,key=lambda r:json.dumps(r,sort_keys=True,default=str))})
|
|||
|
|
cur.execute('''SELECT a.attname FROM pg_index i JOIN pg_attribute a
|
|||
|
|
ON a.attrelid=i.indrelid AND a.attnum=ANY(i.indkey)
|
|||
|
|
WHERE i.indrelid=%s::regclass AND i.indisprimary ORDER BY a.attname''',
|
|||
|
|
(ref['table_name'],))
|
|||
|
|
keys=[r['attname'] for r in cur.fetchall()]
|
|||
|
|
if not keys:
|
|||
|
|
raise HTTPException(409,f"Tabellen {ref['table_name']} mangler en entydig nøgle til flyttehistorikken.")
|
|||
|
|
origins.extend({'table':ref['table_name'],'column':ref['column_name'],
|
|||
|
|
'key':{k:r[k] for k in keys},'original_customer_id':r[ref['column_name']]}
|
|||
|
|
for r in rows if r[ref['column_name']] in source_ids)
|
|||
|
|
if ref['table_name'] in ('contact_companies','entity_tags'):
|
|||
|
|
origins.append({'table':ref['table_name'],'original_memberships':rows})
|
|||
|
|
count = sum(r[ref['column_name']] in source_ids for r in rows)
|
|||
|
|
if count:
|
|||
|
|
movements.append(dict(ref, count=count))
|
|||
|
|
# bmc_workflows stores Hub IDs as text, so numeric FK discovery cannot see it.
|
|||
|
|
cur.execute("SELECT to_regclass('bmc_workflows') AS name")
|
|||
|
|
if cur.fetchone()['name']:
|
|||
|
|
cur.execute('SELECT * FROM bmc_workflows WHERE customer_id=ANY(%s) ORDER BY id',
|
|||
|
|
([str(i) for i in [target_id]+source_ids],))
|
|||
|
|
rows=[dict(r) for r in cur.fetchall()]
|
|||
|
|
state.append({'table':'bmc_workflows','rows':rows})
|
|||
|
|
origins.extend({'table':'bmc_workflows','column':'customer_id','key':{'id':r['id']},
|
|||
|
|
'original_customer_id':r['customer_id']} for r in rows
|
|||
|
|
if r['customer_id'] in [str(i) for i in source_ids])
|
|||
|
|
count=sum(r['customer_id'] in [str(i) for i in source_ids] for r in rows)
|
|||
|
|
if count: movements.append(dict(table_name='bmc_workflows',column_name='customer_id',count=count))
|
|||
|
|
cur.execute("SELECT to_regclass('bmc_workflow_runtime') AS name")
|
|||
|
|
if cur.fetchone()['name']:
|
|||
|
|
cur.execute('SELECT customer_ids FROM bmc_workflow_runtime')
|
|||
|
|
if any(set(r['customer_ids'] or []) & set(source_ids) for r in cur.fetchall()):
|
|||
|
|
raise HTTPException(409, 'En kildekunde er valgt i workflowmotorens kundeliste. Afklar aktiveringen før sammenlægning.')
|
|||
|
|
if set(getattr(settings,'WORKFLOWS_CUSTOMER_IDS',[]) or []) & set(source_ids):
|
|||
|
|
raise HTTPException(409,'En kildekunde er valgt i workflowkonfigurationen. Afklar aktiveringen før sammenlægning.')
|
|||
|
|
for table in ('bmc_workflow_events','bmc_workflow_versions'):
|
|||
|
|
cur.execute('SELECT to_regclass(%s) AS name',(table,))
|
|||
|
|
if not cur.fetchone()['name']: continue
|
|||
|
|
if table == 'bmc_workflow_events':
|
|||
|
|
cur.execute("SELECT context,entity_type,entity_id FROM bmc_workflow_events WHERE status IN ('pending','processing','failed')")
|
|||
|
|
blocked=any(mentions_customer(r['context'],source_ids) or
|
|||
|
|
(r['entity_type']=='customer' and str(r['entity_id']) in {str(i) for i in source_ids}) for r in cur.fetchall())
|
|||
|
|
else:
|
|||
|
|
cur.execute('''SELECT v.definition FROM bmc_workflow_versions v JOIN bmc_workflows w
|
|||
|
|
ON w.active_version_id=v.id WHERE w.enabled''')
|
|||
|
|
blocked=any(mentions_customer(r['definition'],source_ids) for r in cur.fetchall())
|
|||
|
|
if blocked:
|
|||
|
|
raise HTTPException(409,'En kildekunde bruges i aktive workflowregler eller uafsluttede job. Afklar dem før sammenlægning.')
|
|||
|
|
token=signature({'customers':customers,'state':state,'target':target_id,'sources':source_ids})
|
|||
|
|
return customers, changes, differences, movements, token, origins
|
|||
|
|
|
|||
|
|
|
|||
|
|
def move(cur, target, sources, customers, changes, movements):
|
|||
|
|
"""Any unhandled unique constraint is a blocking conflict, never data loss."""
|
|||
|
|
archived=[]
|
|||
|
|
# Transfer unique external IDs only after saving original customer snapshots.
|
|||
|
|
for field in IDENTITY_FIELDS:
|
|||
|
|
if field in customers[0] and (field != 'cvr_number' or field in changes):
|
|||
|
|
cur.execute(sql.SQL('UPDATE customers SET {}=NULL WHERE id=ANY(%s)').format(sql.Identifier(field)),(sources,))
|
|||
|
|
for field,value in changes.items():
|
|||
|
|
cur.execute(sql.SQL('UPDATE customers SET {}=%s WHERE id=%s').format(sql.Identifier(field)),(value,target))
|
|||
|
|
if any(r['table_name']=='customer_relationships' for r in movements):
|
|||
|
|
cur.execute('''DELETE FROM customer_relationships WHERE parent_customer_id=ANY(%s)
|
|||
|
|
AND child_customer_id=ANY(%s) RETURNING *''',([target]+sources,[target]+sources))
|
|||
|
|
archived.extend({'table':'customer_relationships','row':dict(r)} for r in cur.fetchall())
|
|||
|
|
# Shared tag/contact memberships are consolidated; retain their complete old rows in audit.
|
|||
|
|
for table,key,condition in [('contact_companies','contact_id',''),('entity_tags','tag_id'," AND entity_type='customer'")]:
|
|||
|
|
if not any(r['table_name']==table for r in movements): continue
|
|||
|
|
column='entity_id' if table=='entity_tags' else 'customer_id'
|
|||
|
|
cur.execute(sql.SQL('SELECT * FROM {} WHERE {}=ANY(%s)'+condition+' ORDER BY {}').format(
|
|||
|
|
sql.Identifier(table),sql.Identifier(column),sql.Identifier('id')),([target]+sources,))
|
|||
|
|
rows=[dict(r) for r in cur.fetchall()];seen={}
|
|||
|
|
for row in sorted(rows,key=lambda r:(r[column]!=target,r['id'])):
|
|||
|
|
if row[key] not in seen:
|
|||
|
|
seen[row[key]]=row;continue
|
|||
|
|
kept=seen[row[key]]
|
|||
|
|
archived.append({'table':table,'row':row})
|
|||
|
|
if table=='contact_companies':
|
|||
|
|
notes='\n'.join(dict.fromkeys(str(v) for v in (kept.get('notes'),row.get('notes')) if v))
|
|||
|
|
roles=' / '.join(dict.fromkeys(str(v) for v in (kept.get('role'),row.get('role')) if v))
|
|||
|
|
cur.execute('UPDATE contact_companies SET notes=%s,role=%s,is_primary=%s WHERE id=%s',
|
|||
|
|
(notes or None,roles or None,bool(kept.get('is_primary') or row.get('is_primary')),kept['id']))
|
|||
|
|
kept.update(notes=notes,role=roles,is_primary=bool(kept.get('is_primary') or row.get('is_primary')))
|
|||
|
|
cur.execute(sql.SQL('DELETE FROM {} WHERE id=%s').format(sql.Identifier(table)),(row['id'],))
|
|||
|
|
if table=='contact_companies' and any(r[column]==target and r.get('is_primary') for r in rows):
|
|||
|
|
cur.execute('UPDATE contact_companies SET is_primary=FALSE WHERE customer_id=ANY(%s)',(sources,))
|
|||
|
|
for ref in movements:
|
|||
|
|
table,column=ref['table_name'],ref['column_name']
|
|||
|
|
# Avoid introducing self-referential parent/billing relations on merged rows.
|
|||
|
|
if table=='customers':
|
|||
|
|
cur.execute(sql.SQL('UPDATE customers SET {}=NULL WHERE id=ANY(%s) AND {}=ANY(%s)').format(
|
|||
|
|
sql.Identifier(column),sql.Identifier(column)),([target]+sources,sources))
|
|||
|
|
value=str(target) if table=='bmc_workflows' else target
|
|||
|
|
ids=[str(i) for i in sources] if table=='bmc_workflows' else sources
|
|||
|
|
cur.execute(sql.SQL('UPDATE {} SET {}=%s WHERE {}').format(sql.Identifier(table),sql.Identifier(column),where(ref)),(value,ids))
|
|||
|
|
cur.execute('UPDATE customers SET is_active=FALSE,merged_into_customer_id=%s,updated_at=now() WHERE id=ANY(%s)',(target,sources))
|
|||
|
|
return archived
|
|||
|
|
|
|||
|
|
|
|||
|
|
def run(selection, actor_id, preview_token=None):
|
|||
|
|
sources=selections(selection.target_id,selection.source_ids)
|
|||
|
|
try:
|
|||
|
|
with transaction(commit=preview_token is not None) as cur:
|
|||
|
|
customers,changes,differences,movements,token,origins=prepare(cur,selection.target_id,sources)
|
|||
|
|
if preview_token is not None and preview_token != token:
|
|||
|
|
raise HTTPException(409,'Data er ændret siden forhåndsvisningen. Vis sammenlægningen igen.')
|
|||
|
|
archived=move(cur,selection.target_id,sources,customers,changes,movements)
|
|||
|
|
if preview_token is not None:
|
|||
|
|
cur.execute('''INSERT INTO customer_merge_history(target_customer_id,source_customer_ids,actor_id,snapshot,movements)
|
|||
|
|
VALUES(%s,%s,%s,%s,%s) RETURNING id''',
|
|||
|
|
(selection.target_id,sources,actor_id,Json({'customers':customers,'relation_origins':origins,'archived_memberships':archived},dumps=lambda v:json.dumps(v,default=str)),Json(movements)))
|
|||
|
|
history_id=cur.fetchone()['id']
|
|||
|
|
else: history_id=None
|
|||
|
|
return {'target_id':selection.target_id,'source_ids':sources,'preview_token':token,
|
|||
|
|
'customers':[{'id':c['id'],'name':c['name']} for c in customers],
|
|||
|
|
'relations':[{'table':r['table_name'],'column':r['column_name'],'count':r['count']} for r in movements],
|
|||
|
|
'filled_fields':list(changes),'kept_main_fields':differences,'history_id':history_id,
|
|||
|
|
'external_systems_changed':False}
|
|||
|
|
except psycopg2.IntegrityError as exc:
|
|||
|
|
table=exc.diag.table_name or 'en relateret tabel'
|
|||
|
|
raise HTTPException(409,f'Sammenlægningen har en konflikt i {table}. Ingen data er flyttet; afklar de overlappende relationer først.') from exc
|
|||
|
|
except psycopg2.Error as exc:
|
|||
|
|
logger.warning('Customer merge rolled back (%s)',exc.pgcode)
|
|||
|
|
raise HTTPException(409,'Sammenlægningen kunne ikke gennemføres. Ingen data er flyttet. Genindlæs og prøv igen.') from exc
|
|||
|
|
|
|||
|
|
|
|||
|
|
@router.post('/customers/merge/preview')
|
|||
|
|
def preview(selection:MergeSelection,actor=Depends(require_permission('customers.edit'))):
|
|||
|
|
return run(selection,actor['id'])
|
|||
|
|
|
|||
|
|
|
|||
|
|
@router.post('/customers/merge')
|
|||
|
|
def merge(selection:MergeConfirmation,actor=Depends(require_permission('customers.edit'))):
|
|||
|
|
return run(selection,actor['id'],selection.preview_token)
|