Files
Django_TasksApp/main/home_assistant/sync.py
T
2026-10-23 14:30:58 +02:00

363 lines
19 KiB
Python

"""Conservative UID-based grocery reconciliation. Never deletes on either side."""
import hashlib
import json
from datetime import timedelta
from django.db import transaction
from django.db.models import Q
from django.utils import timezone
from groceries.models import GroceryItem
from groceries.services import add as add_grocery, edit as edit_grocery
from .client import ConnectionError, add_item, decrypt_token, get_items, update_item
from .models import HomeAssistantConnection, HomeAssistantItemMap, HomeAssistantSyncRun
def _fingerprint(name, status):
return hashlib.sha256(json.dumps([name, status], ensure_ascii=False).encode()).hexdigest()
def _local_state(item):
return {'name': item.name, 'status': 'completed' if item.status == GroceryItem.Status.PURCHASED else 'needs_action'}
def _hash_state(state):
return _fingerprint(state['name'], state['status'])
def _snapshot(connection):
if not connection.grocery_entity_id:
raise ConnectionError('Choose a Home Assistant grocery list first.')
token = decrypt_token(connection.encrypted_token)
remote = get_items(connection.base_url, token, connection.grocery_entity_id)
local = {str(item.pk): item for item in GroceryItem.objects.filter(user=connection.user)}
maps = list(HomeAssistantItemMap.objects.filter(connection=connection,
entity_id=connection.grocery_entity_id).select_related('grocery'))
return token, local, remote, maps
def build_plan(connection, local, remote, maps):
operations = []
issues = []
pending_recoveries = []
mapped_local = {str(link.grocery_id) for link in maps if link.grocery_id}
mapped_remote = {link.ha_uid for link in maps if link.ha_uid}
can_pull = connection.direction in {'both', 'from_ha'}
can_push = connection.direction in {'both', 'to_ha'}
entity = next((entry for entry in connection.entities if entry.get('id') == connection.grocery_entity_id), {})
features = int(entity.get('features') or 0)
can_create = bool(features & 1)
can_update = bool(features & 4)
for link in sorted(maps, key=lambda item: item.pk):
item = local.get(str(link.grocery_id)) if link.grocery_id else None
ha = remote.get(link.ha_uid) if link.ha_uid else None
if link.pending_create:
if not item:
issues.append('An unfinished Home Assistant addition lost its TasksNest item; review it manually.')
continue
candidates = [value for uid, value in remote.items() if uid not in mapped_remote and value['name'].casefold() == item.name.casefold()]
if len(candidates) == 1:
operations.append({'kind': 'reconcile', 'local_id': str(item.pk), 'uid': candidates[0]['uid'], 'label': item.name})
else:
issues.append(f'Addition of “{item.name}” is uncertain; review Home Assistant before retrying.')
if not candidates:
pending_recoveries.append({'mapping_id': link.pk, 'label': item.name})
continue
if not item or not ha:
issues.append(f'Mapped item {link.ha_uid or "unknown"} is missing on one side; no deletion will occur.')
continue
local_state = _local_state(item)
local_hash = _hash_state(local_state)
remote_hash = _hash_state(ha)
if local_hash == remote_hash:
if link.last_local_hash != local_hash or link.last_remote_hash != remote_hash:
operations.append({'kind': 'baseline', 'local_id': str(item.pk), 'uid': link.ha_uid, 'label': item.name})
continue
local_changed = local_hash != link.last_local_hash
remote_changed = remote_hash != link.last_remote_hash
if connection.direction == 'from_ha':
winner = 'ha'
elif connection.direction == 'to_ha':
winner = 'tasksnest'
elif local_changed and remote_changed:
winner = connection.conflict_winner
elif remote_changed:
winner = 'ha'
elif local_changed:
winner = 'tasksnest'
else:
# Inconsistent prior snapshots must be reviewed, not guessed.
issues.append(f'“{item.name}” has inconsistent sync history; no change was made.')
continue
if winner == 'ha' and can_pull:
operations.append({'kind': 'pull', 'local_id': str(item.pk), 'uid': link.ha_uid, 'label': ha['name']})
elif winner == 'tasksnest' and can_push:
if can_update:
operations.append({'kind': 'push', 'local_id': str(item.pk), 'uid': link.ha_uid, 'label': item.name})
else:
issues.append(f'“{item.name}” cannot be updated: this Home Assistant list is read-only.')
reconciled_uids = {action['uid'] for action in operations if action['kind'] == 'reconcile'}
unmatched_local = [item for key, item in local.items() if key not in mapped_local]
unmatched_remote = [item for uid, item in remote.items() if uid not in mapped_remote and uid not in reconciled_uids]
local_names = {item.name.casefold() for item in local.values()}
remote_names = {item['name'].casefold() for item in remote.values()}
collisions = ({item.name.casefold() for item in unmatched_local} & remote_names) | (
{item['name'].casefold() for item in unmatched_remote} & local_names)
for name in sorted(collisions):
issues.append(f'“{name}” exists on both lists without a UID mapping; review duplicates before syncing it.')
if can_push:
for item in sorted(unmatched_local, key=lambda row: str(row.pk)):
if item.name.casefold() not in collisions:
if can_create and (item.status == GroceryItem.Status.NEEDED or can_update):
operations.append({'kind': 'export', 'local_id': str(item.pk), 'label': item.name})
else:
issues.append(f'“{item.name}” cannot be added: this Home Assistant list lacks create or update support.')
if can_pull:
for uid, item in sorted(remote.items()):
if uid not in mapped_remote and uid not in reconciled_uids and item['name'].casefold() not in collisions:
operations.append({'kind': 'import', 'uid': uid, 'label': item['name']})
state = {
'connection': connection.pk, 'entity': connection.grocery_entity_id,
'direction': connection.direction, 'winner': connection.conflict_winner,
'features': features,
'local': sorted((key, _hash_state(_local_state(item))) for key, item in local.items()),
'remote': sorted((uid, _hash_state(item)) for uid, item in remote.items()),
'maps': sorted((link.pk, str(link.grocery_id), link.ha_uid, link.last_local_hash,
link.last_remote_hash, link.pending_create) for link in maps),
}
digest = hashlib.sha256(json.dumps(state, sort_keys=True).encode()).hexdigest()
return {'operations': operations, 'issues': issues, 'pending_recoveries': pending_recoveries, 'digest': digest,
'limited': len(operations) > 20}
def preview(connection):
_, local, remote, maps = _snapshot(connection)
return build_plan(connection, local, remote, maps)
def _mapping(connection, item, uid='', *, pending=False, local_hash='', remote_hash=''):
return HomeAssistantItemMap.objects.create(
connection=connection, entity_id=connection.grocery_entity_id, grocery=item,
ha_uid=uid, pending_create=pending, last_local_hash=local_hash,
last_remote_hash=remote_hash)
def _record_run(connection, *, succeeded, automatic, changes=0, summary='', error=''):
HomeAssistantSyncRun.objects.create(connection=connection, succeeded=succeeded,
automatic=automatic, changes_applied=changes, summary=str(summary)[:255], error=str(error)[:255])
old_ids = list(connection.sync_runs.order_by('-created_at', '-pk').values_list('pk', flat=True)[20:])
if old_ids:
HomeAssistantSyncRun.objects.filter(pk__in=old_ids).delete()
def _schedule_after_success(connection, now):
connection.consecutive_sync_failures = 0
connection.last_synced_at = now
connection.last_sync_error = ''
if connection.auto_sync_enabled and connection.sync_interval_minutes != 'manual':
connection.next_sync_at = now + timedelta(minutes=int(connection.sync_interval_minutes))
else:
connection.next_sync_at = None
def _schedule_after_failure(connection, error, automatic):
connection.last_sync_error = str(error)[:255]
connection.consecutive_sync_failures = min(connection.consecutive_sync_failures + 1, 16)
exhausted = connection.consecutive_sync_failures >= 5
if exhausted:
connection.auto_sync_enabled = False
connection.next_sync_at = None
elif connection.auto_sync_enabled and connection.sync_interval_minutes != 'manual':
backoff = min(60, 5 * (2 ** min(connection.consecutive_sync_failures - 1, 4)))
connection.next_sync_at = timezone.now() + timedelta(minutes=backoff)
connection.save(update_fields=['last_sync_error', 'consecutive_sync_failures', 'next_sync_at',
'auto_sync_enabled', 'updated_at'])
_record_run(connection, succeeded=False, automatic=automatic, error=error,
summary=('Automatic sync disabled after five consecutive failures.' if exhausted else
'Automatic sync failed.' if automatic else 'Sync stopped after an error.'))
def _apply(connection, token, action, local, remote):
kind = action['kind']
item = local.get(action.get('local_id'))
uid = action.get('uid', '')
ha = remote.get(uid)
link = HomeAssistantItemMap.objects.filter(connection=connection,
entity_id=connection.grocery_entity_id, grocery=item).first() if item else None
if kind == 'import':
if not ha:
raise ConnectionError('The Home Assistant list changed. Preview again.')
key = 'ha:' + hashlib.sha256(f'{connection.pk}:{connection.grocery_entity_id}:{uid}'.encode()).hexdigest()[:64]
with transaction.atomic():
item = add_grocery(connection.user, name=ha['name'], quantity=1, unit='pieces', idempotency_key=key)
if ha['status'] == 'completed' and item.status != GroceryItem.Status.PURCHASED:
edit_grocery(connection.user, item.pk, {'status': GroceryItem.Status.PURCHASED})
_mapping(connection, item, uid, local_hash=_hash_state(ha), remote_hash=_hash_state(ha))
return
if not item:
raise ConnectionError('A TasksNest item changed. Preview again.')
local_state = _local_state(item)
if kind == 'export':
link = _mapping(connection, item, pending=True)
add_item(connection.base_url, token, connection.grocery_entity_id, item.name)
latest = get_items(connection.base_url, token, connection.grocery_entity_id)
candidates = [value for candidate_uid, value in latest.items()
if candidate_uid not in remote and value['name'] == item.name]
if len(candidates) != 1:
raise ConnectionError(f'Added “{item.name}”, but its new UID could not be identified. Review the list before retrying.')
uid = candidates[0]['uid']
link.ha_uid = uid
link.pending_create = False
link.last_remote_hash = _hash_state(candidates[0])
link.last_local_hash = _hash_state(candidates[0])
link.save()
if local_state['status'] == 'completed':
update_item(connection.base_url, token, connection.grocery_entity_id, uid, status='completed')
verified = get_items(connection.base_url, token, connection.grocery_entity_id).get(uid)
if not verified or verified['status'] != 'completed':
raise ConnectionError(f'“{item.name}” was added but its completed state was not confirmed.')
link.last_local_hash = _hash_state(local_state)
link.last_remote_hash = _hash_state(verified)
link.save(update_fields=['last_local_hash', 'last_remote_hash', 'updated_at'])
return
if not link or not ha:
raise ConnectionError('A mapped item changed. Preview again.')
if kind == 'reconcile':
link.ha_uid = uid
link.pending_create = False
link.last_local_hash = _hash_state(ha)
link.last_remote_hash = _hash_state(ha)
link.save()
elif kind == 'pull':
changes = {}
if item.name != ha['name']:
changes['name'] = ha['name']
new_status = GroceryItem.Status.PURCHASED if ha['status'] == 'completed' else GroceryItem.Status.NEEDED
if item.status != new_status:
changes['status'] = new_status
if changes:
edit_grocery(connection.user, item.pk, changes)
link.last_local_hash = _hash_state(ha)
link.last_remote_hash = _hash_state(ha)
link.save(update_fields=['last_local_hash', 'last_remote_hash', 'updated_at'])
elif kind == 'push':
update_item(connection.base_url, token, connection.grocery_entity_id, uid,
name=item.name if item.name != ha['name'] else None,
status=local_state['status'] if local_state['status'] != ha['status'] else None)
verified = get_items(connection.base_url, token, connection.grocery_entity_id).get(uid)
if not verified or _hash_state(verified) != _hash_state(local_state):
raise ConnectionError(f'Home Assistant did not confirm the update to “{item.name}”.')
link.last_local_hash = _hash_state(local_state)
link.last_remote_hash = _hash_state(verified)
link.save(update_fields=['last_local_hash', 'last_remote_hash', 'updated_at'])
elif kind == 'baseline':
link.last_local_hash = _hash_state(local_state)
link.last_remote_hash = _hash_state(ha)
link.save(update_fields=['last_local_hash', 'last_remote_hash', 'updated_at'])
def confirm_and_sync(connection, expected_digest, *, automatic=False):
now = timezone.now()
claimed = HomeAssistantConnection.objects.filter(pk=connection.pk).filter(
Q(sync_lock_until__isnull=True) | Q(sync_lock_until__lt=now)).update(
sync_lock_until=now + timedelta(minutes=3))
if not claimed:
raise ConnectionError('A sync is already running. Try again shortly.')
try:
connection.refresh_from_db()
token, local, remote, maps = _snapshot(connection)
plan = build_plan(connection, local, remote, maps)
if plan['digest'] != expected_digest:
raise ConnectionError('The lists changed after the preview. Review a fresh preview before syncing.')
connection.initial_sync_confirmed = True
connection.save(update_fields=['initial_sync_confirmed', 'updated_at'])
done = 0
for action in plan['operations'][:20]:
_apply(connection, token, action, local, remote)
done += 1
# Refresh state so later actions see UIDs and items created in this run.
token, local, remote, _ = _snapshot(connection)
_schedule_after_success(connection, timezone.now())
connection.save(update_fields=['last_synced_at', 'last_sync_error', 'next_sync_at',
'consecutive_sync_failures', 'updated_at'])
_record_run(connection, succeeded=True, automatic=automatic, changes=done,
summary=f'Applied {done} change(s); {len(plan["issues"])} item(s) need review.')
return done, plan
except ConnectionError as exc:
_schedule_after_failure(connection, exc, automatic)
raise
finally:
HomeAssistantConnection.objects.filter(pk=connection.pk).update(sync_lock_until=None)
def reset_pending_create(connection, expected_digest, mapping_id):
now = timezone.now()
claimed = HomeAssistantConnection.objects.filter(pk=connection.pk).filter(
Q(sync_lock_until__isnull=True) | Q(sync_lock_until__lt=now)).update(
sync_lock_until=now + timedelta(minutes=3))
if not claimed:
raise ConnectionError('A sync is already running. Try again shortly.')
try:
connection.refresh_from_db()
_, local, remote, maps = _snapshot(connection)
plan = build_plan(connection, local, remote, maps)
if plan['digest'] != expected_digest:
raise ConnectionError('The lists changed after the preview. Review a fresh preview.')
mapping = HomeAssistantItemMap.objects.filter(pk=mapping_id, connection=connection,
entity_id=connection.grocery_entity_id, pending_create=True).select_related('grocery').first()
if not mapping or not mapping.grocery_id:
raise ConnectionError('This pending addition can no longer be reset.')
if any(item['name'].casefold() == mapping.grocery.name.casefold() for item in remote.values()):
raise ConnectionError('A matching item is now visible in Home Assistant. Preview again to reconcile it.')
mapping.delete()
_record_run(connection, succeeded=True, automatic=False,
summary=f'Reset uncertain addition for “{mapping.grocery.name}”.')
finally:
HomeAssistantConnection.objects.filter(pk=connection.pk).update(sync_lock_until=None)
def run_due_syncs():
now = timezone.now()
due = HomeAssistantConnection.objects.filter(auto_sync_enabled=True,
initial_sync_confirmed=True).exclude(sync_interval_minutes='manual').filter(
Q(next_sync_at__isnull=True) | Q(next_sync_at__lte=now)).order_by('next_sync_at', 'pk')
attempted = succeeded = failed = 0
for connection in due.iterator():
if connection.sync_lock_until and connection.sync_lock_until > now:
continue
attempted += 1
grocery_failed = False
try:
plan = preview(connection)
confirm_and_sync(connection, plan['digest'], automatic=True)
succeeded += 1
except ConnectionError as exc:
grocery_failed = True
failed += 1
connection.refresh_from_db()
# Preview can fail before confirm_and_sync claims a lock and records history.
recent = connection.sync_runs.filter(created_at__gte=now).exists()
if not recent:
_schedule_after_failure(connection, exc, automatic=True)
# Optional task sync shares the connection's scheduler and lease, but has
# its own first-sync confirmation and item mapping namespace.
connection.refresh_from_db()
if not grocery_failed and connection.task_entity_id and connection.task_initial_sync_confirmed:
runs_before = connection.sync_runs.count()
try:
from .task_sync import confirm_task_sync, preview_tasks
task_plan = preview_tasks(connection)
confirm_task_sync(connection, task_plan['digest'], automatic=True)
succeeded += 1
attempted += 1
except ConnectionError as exc:
failed += 1
connection.refresh_from_db()
if connection.sync_runs.count() == runs_before:
_schedule_after_failure(connection, exc, automatic=True)
return {'attempted': attempted, 'succeeded': succeeded, 'failed': failed}