363 lines
19 KiB
Python
363 lines
19 KiB
Python
"""Conservative UID-based grocery reconciliation. Never deletes on either side."""
|
|
import hashlib
|
|
import json
|
|
from datetime import timedelta
|
|
|
|
from django.db import transaction
|
|
from django.db.models import Q
|
|
from django.utils import timezone
|
|
|
|
from groceries.models import GroceryItem
|
|
from groceries.services import add as add_grocery, edit as edit_grocery
|
|
|
|
from .client import ConnectionError, add_item, decrypt_token, get_items, update_item
|
|
from .models import HomeAssistantConnection, HomeAssistantItemMap, HomeAssistantSyncRun
|
|
|
|
|
|
def _fingerprint(name, status):
|
|
return hashlib.sha256(json.dumps([name, status], ensure_ascii=False).encode()).hexdigest()
|
|
|
|
|
|
def _local_state(item):
|
|
return {'name': item.name, 'status': 'completed' if item.status == GroceryItem.Status.PURCHASED else 'needs_action'}
|
|
|
|
|
|
def _hash_state(state):
|
|
return _fingerprint(state['name'], state['status'])
|
|
|
|
|
|
def _snapshot(connection):
|
|
if not connection.grocery_entity_id:
|
|
raise ConnectionError('Choose a Home Assistant grocery list first.')
|
|
token = decrypt_token(connection.encrypted_token)
|
|
remote = get_items(connection.base_url, token, connection.grocery_entity_id)
|
|
local = {str(item.pk): item for item in GroceryItem.objects.filter(user=connection.user)}
|
|
maps = list(HomeAssistantItemMap.objects.filter(connection=connection,
|
|
entity_id=connection.grocery_entity_id).select_related('grocery'))
|
|
return token, local, remote, maps
|
|
|
|
|
|
def build_plan(connection, local, remote, maps):
|
|
operations = []
|
|
issues = []
|
|
pending_recoveries = []
|
|
mapped_local = {str(link.grocery_id) for link in maps if link.grocery_id}
|
|
mapped_remote = {link.ha_uid for link in maps if link.ha_uid}
|
|
can_pull = connection.direction in {'both', 'from_ha'}
|
|
can_push = connection.direction in {'both', 'to_ha'}
|
|
entity = next((entry for entry in connection.entities if entry.get('id') == connection.grocery_entity_id), {})
|
|
features = int(entity.get('features') or 0)
|
|
can_create = bool(features & 1)
|
|
can_update = bool(features & 4)
|
|
|
|
for link in sorted(maps, key=lambda item: item.pk):
|
|
item = local.get(str(link.grocery_id)) if link.grocery_id else None
|
|
ha = remote.get(link.ha_uid) if link.ha_uid else None
|
|
if link.pending_create:
|
|
if not item:
|
|
issues.append('An unfinished Home Assistant addition lost its TasksNest item; review it manually.')
|
|
continue
|
|
candidates = [value for uid, value in remote.items() if uid not in mapped_remote and value['name'].casefold() == item.name.casefold()]
|
|
if len(candidates) == 1:
|
|
operations.append({'kind': 'reconcile', 'local_id': str(item.pk), 'uid': candidates[0]['uid'], 'label': item.name})
|
|
else:
|
|
issues.append(f'Addition of “{item.name}” is uncertain; review Home Assistant before retrying.')
|
|
if not candidates:
|
|
pending_recoveries.append({'mapping_id': link.pk, 'label': item.name})
|
|
continue
|
|
if not item or not ha:
|
|
issues.append(f'Mapped item {link.ha_uid or "unknown"} is missing on one side; no deletion will occur.')
|
|
continue
|
|
local_state = _local_state(item)
|
|
local_hash = _hash_state(local_state)
|
|
remote_hash = _hash_state(ha)
|
|
if local_hash == remote_hash:
|
|
if link.last_local_hash != local_hash or link.last_remote_hash != remote_hash:
|
|
operations.append({'kind': 'baseline', 'local_id': str(item.pk), 'uid': link.ha_uid, 'label': item.name})
|
|
continue
|
|
local_changed = local_hash != link.last_local_hash
|
|
remote_changed = remote_hash != link.last_remote_hash
|
|
if connection.direction == 'from_ha':
|
|
winner = 'ha'
|
|
elif connection.direction == 'to_ha':
|
|
winner = 'tasksnest'
|
|
elif local_changed and remote_changed:
|
|
winner = connection.conflict_winner
|
|
elif remote_changed:
|
|
winner = 'ha'
|
|
elif local_changed:
|
|
winner = 'tasksnest'
|
|
else:
|
|
# Inconsistent prior snapshots must be reviewed, not guessed.
|
|
issues.append(f'“{item.name}” has inconsistent sync history; no change was made.')
|
|
continue
|
|
if winner == 'ha' and can_pull:
|
|
operations.append({'kind': 'pull', 'local_id': str(item.pk), 'uid': link.ha_uid, 'label': ha['name']})
|
|
elif winner == 'tasksnest' and can_push:
|
|
if can_update:
|
|
operations.append({'kind': 'push', 'local_id': str(item.pk), 'uid': link.ha_uid, 'label': item.name})
|
|
else:
|
|
issues.append(f'“{item.name}” cannot be updated: this Home Assistant list is read-only.')
|
|
|
|
reconciled_uids = {action['uid'] for action in operations if action['kind'] == 'reconcile'}
|
|
unmatched_local = [item for key, item in local.items() if key not in mapped_local]
|
|
unmatched_remote = [item for uid, item in remote.items() if uid not in mapped_remote and uid not in reconciled_uids]
|
|
local_names = {item.name.casefold() for item in local.values()}
|
|
remote_names = {item['name'].casefold() for item in remote.values()}
|
|
collisions = ({item.name.casefold() for item in unmatched_local} & remote_names) | (
|
|
{item['name'].casefold() for item in unmatched_remote} & local_names)
|
|
for name in sorted(collisions):
|
|
issues.append(f'“{name}” exists on both lists without a UID mapping; review duplicates before syncing it.')
|
|
if can_push:
|
|
for item in sorted(unmatched_local, key=lambda row: str(row.pk)):
|
|
if item.name.casefold() not in collisions:
|
|
if can_create and (item.status == GroceryItem.Status.NEEDED or can_update):
|
|
operations.append({'kind': 'export', 'local_id': str(item.pk), 'label': item.name})
|
|
else:
|
|
issues.append(f'“{item.name}” cannot be added: this Home Assistant list lacks create or update support.')
|
|
if can_pull:
|
|
for uid, item in sorted(remote.items()):
|
|
if uid not in mapped_remote and uid not in reconciled_uids and item['name'].casefold() not in collisions:
|
|
operations.append({'kind': 'import', 'uid': uid, 'label': item['name']})
|
|
|
|
state = {
|
|
'connection': connection.pk, 'entity': connection.grocery_entity_id,
|
|
'direction': connection.direction, 'winner': connection.conflict_winner,
|
|
'features': features,
|
|
'local': sorted((key, _hash_state(_local_state(item))) for key, item in local.items()),
|
|
'remote': sorted((uid, _hash_state(item)) for uid, item in remote.items()),
|
|
'maps': sorted((link.pk, str(link.grocery_id), link.ha_uid, link.last_local_hash,
|
|
link.last_remote_hash, link.pending_create) for link in maps),
|
|
}
|
|
digest = hashlib.sha256(json.dumps(state, sort_keys=True).encode()).hexdigest()
|
|
return {'operations': operations, 'issues': issues, 'pending_recoveries': pending_recoveries, 'digest': digest,
|
|
'limited': len(operations) > 20}
|
|
|
|
|
|
def preview(connection):
|
|
_, local, remote, maps = _snapshot(connection)
|
|
return build_plan(connection, local, remote, maps)
|
|
|
|
|
|
def _mapping(connection, item, uid='', *, pending=False, local_hash='', remote_hash=''):
|
|
return HomeAssistantItemMap.objects.create(
|
|
connection=connection, entity_id=connection.grocery_entity_id, grocery=item,
|
|
ha_uid=uid, pending_create=pending, last_local_hash=local_hash,
|
|
last_remote_hash=remote_hash)
|
|
|
|
|
|
def _record_run(connection, *, succeeded, automatic, changes=0, summary='', error=''):
|
|
HomeAssistantSyncRun.objects.create(connection=connection, succeeded=succeeded,
|
|
automatic=automatic, changes_applied=changes, summary=str(summary)[:255], error=str(error)[:255])
|
|
old_ids = list(connection.sync_runs.order_by('-created_at', '-pk').values_list('pk', flat=True)[20:])
|
|
if old_ids:
|
|
HomeAssistantSyncRun.objects.filter(pk__in=old_ids).delete()
|
|
|
|
|
|
def _schedule_after_success(connection, now):
|
|
connection.consecutive_sync_failures = 0
|
|
connection.last_synced_at = now
|
|
connection.last_sync_error = ''
|
|
if connection.auto_sync_enabled and connection.sync_interval_minutes != 'manual':
|
|
connection.next_sync_at = now + timedelta(minutes=int(connection.sync_interval_minutes))
|
|
else:
|
|
connection.next_sync_at = None
|
|
|
|
|
|
def _schedule_after_failure(connection, error, automatic):
|
|
connection.last_sync_error = str(error)[:255]
|
|
connection.consecutive_sync_failures = min(connection.consecutive_sync_failures + 1, 16)
|
|
exhausted = connection.consecutive_sync_failures >= 5
|
|
if exhausted:
|
|
connection.auto_sync_enabled = False
|
|
connection.next_sync_at = None
|
|
elif connection.auto_sync_enabled and connection.sync_interval_minutes != 'manual':
|
|
backoff = min(60, 5 * (2 ** min(connection.consecutive_sync_failures - 1, 4)))
|
|
connection.next_sync_at = timezone.now() + timedelta(minutes=backoff)
|
|
connection.save(update_fields=['last_sync_error', 'consecutive_sync_failures', 'next_sync_at',
|
|
'auto_sync_enabled', 'updated_at'])
|
|
_record_run(connection, succeeded=False, automatic=automatic, error=error,
|
|
summary=('Automatic sync disabled after five consecutive failures.' if exhausted else
|
|
'Automatic sync failed.' if automatic else 'Sync stopped after an error.'))
|
|
|
|
|
|
def _apply(connection, token, action, local, remote):
|
|
kind = action['kind']
|
|
item = local.get(action.get('local_id'))
|
|
uid = action.get('uid', '')
|
|
ha = remote.get(uid)
|
|
link = HomeAssistantItemMap.objects.filter(connection=connection,
|
|
entity_id=connection.grocery_entity_id, grocery=item).first() if item else None
|
|
if kind == 'import':
|
|
if not ha:
|
|
raise ConnectionError('The Home Assistant list changed. Preview again.')
|
|
key = 'ha:' + hashlib.sha256(f'{connection.pk}:{connection.grocery_entity_id}:{uid}'.encode()).hexdigest()[:64]
|
|
with transaction.atomic():
|
|
item = add_grocery(connection.user, name=ha['name'], quantity=1, unit='pieces', idempotency_key=key)
|
|
if ha['status'] == 'completed' and item.status != GroceryItem.Status.PURCHASED:
|
|
edit_grocery(connection.user, item.pk, {'status': GroceryItem.Status.PURCHASED})
|
|
_mapping(connection, item, uid, local_hash=_hash_state(ha), remote_hash=_hash_state(ha))
|
|
return
|
|
if not item:
|
|
raise ConnectionError('A TasksNest item changed. Preview again.')
|
|
local_state = _local_state(item)
|
|
if kind == 'export':
|
|
link = _mapping(connection, item, pending=True)
|
|
add_item(connection.base_url, token, connection.grocery_entity_id, item.name)
|
|
latest = get_items(connection.base_url, token, connection.grocery_entity_id)
|
|
candidates = [value for candidate_uid, value in latest.items()
|
|
if candidate_uid not in remote and value['name'] == item.name]
|
|
if len(candidates) != 1:
|
|
raise ConnectionError(f'Added “{item.name}”, but its new UID could not be identified. Review the list before retrying.')
|
|
uid = candidates[0]['uid']
|
|
link.ha_uid = uid
|
|
link.pending_create = False
|
|
link.last_remote_hash = _hash_state(candidates[0])
|
|
link.last_local_hash = _hash_state(candidates[0])
|
|
link.save()
|
|
if local_state['status'] == 'completed':
|
|
update_item(connection.base_url, token, connection.grocery_entity_id, uid, status='completed')
|
|
verified = get_items(connection.base_url, token, connection.grocery_entity_id).get(uid)
|
|
if not verified or verified['status'] != 'completed':
|
|
raise ConnectionError(f'“{item.name}” was added but its completed state was not confirmed.')
|
|
link.last_local_hash = _hash_state(local_state)
|
|
link.last_remote_hash = _hash_state(verified)
|
|
link.save(update_fields=['last_local_hash', 'last_remote_hash', 'updated_at'])
|
|
return
|
|
if not link or not ha:
|
|
raise ConnectionError('A mapped item changed. Preview again.')
|
|
if kind == 'reconcile':
|
|
link.ha_uid = uid
|
|
link.pending_create = False
|
|
link.last_local_hash = _hash_state(ha)
|
|
link.last_remote_hash = _hash_state(ha)
|
|
link.save()
|
|
elif kind == 'pull':
|
|
changes = {}
|
|
if item.name != ha['name']:
|
|
changes['name'] = ha['name']
|
|
new_status = GroceryItem.Status.PURCHASED if ha['status'] == 'completed' else GroceryItem.Status.NEEDED
|
|
if item.status != new_status:
|
|
changes['status'] = new_status
|
|
if changes:
|
|
edit_grocery(connection.user, item.pk, changes)
|
|
link.last_local_hash = _hash_state(ha)
|
|
link.last_remote_hash = _hash_state(ha)
|
|
link.save(update_fields=['last_local_hash', 'last_remote_hash', 'updated_at'])
|
|
elif kind == 'push':
|
|
update_item(connection.base_url, token, connection.grocery_entity_id, uid,
|
|
name=item.name if item.name != ha['name'] else None,
|
|
status=local_state['status'] if local_state['status'] != ha['status'] else None)
|
|
verified = get_items(connection.base_url, token, connection.grocery_entity_id).get(uid)
|
|
if not verified or _hash_state(verified) != _hash_state(local_state):
|
|
raise ConnectionError(f'Home Assistant did not confirm the update to “{item.name}”.')
|
|
link.last_local_hash = _hash_state(local_state)
|
|
link.last_remote_hash = _hash_state(verified)
|
|
link.save(update_fields=['last_local_hash', 'last_remote_hash', 'updated_at'])
|
|
elif kind == 'baseline':
|
|
link.last_local_hash = _hash_state(local_state)
|
|
link.last_remote_hash = _hash_state(ha)
|
|
link.save(update_fields=['last_local_hash', 'last_remote_hash', 'updated_at'])
|
|
|
|
|
|
def confirm_and_sync(connection, expected_digest, *, automatic=False):
|
|
now = timezone.now()
|
|
claimed = HomeAssistantConnection.objects.filter(pk=connection.pk).filter(
|
|
Q(sync_lock_until__isnull=True) | Q(sync_lock_until__lt=now)).update(
|
|
sync_lock_until=now + timedelta(minutes=3))
|
|
if not claimed:
|
|
raise ConnectionError('A sync is already running. Try again shortly.')
|
|
try:
|
|
connection.refresh_from_db()
|
|
token, local, remote, maps = _snapshot(connection)
|
|
plan = build_plan(connection, local, remote, maps)
|
|
if plan['digest'] != expected_digest:
|
|
raise ConnectionError('The lists changed after the preview. Review a fresh preview before syncing.')
|
|
connection.initial_sync_confirmed = True
|
|
connection.save(update_fields=['initial_sync_confirmed', 'updated_at'])
|
|
done = 0
|
|
for action in plan['operations'][:20]:
|
|
_apply(connection, token, action, local, remote)
|
|
done += 1
|
|
# Refresh state so later actions see UIDs and items created in this run.
|
|
token, local, remote, _ = _snapshot(connection)
|
|
_schedule_after_success(connection, timezone.now())
|
|
connection.save(update_fields=['last_synced_at', 'last_sync_error', 'next_sync_at',
|
|
'consecutive_sync_failures', 'updated_at'])
|
|
_record_run(connection, succeeded=True, automatic=automatic, changes=done,
|
|
summary=f'Applied {done} change(s); {len(plan["issues"])} item(s) need review.')
|
|
return done, plan
|
|
except ConnectionError as exc:
|
|
_schedule_after_failure(connection, exc, automatic)
|
|
raise
|
|
finally:
|
|
HomeAssistantConnection.objects.filter(pk=connection.pk).update(sync_lock_until=None)
|
|
|
|
|
|
def reset_pending_create(connection, expected_digest, mapping_id):
|
|
now = timezone.now()
|
|
claimed = HomeAssistantConnection.objects.filter(pk=connection.pk).filter(
|
|
Q(sync_lock_until__isnull=True) | Q(sync_lock_until__lt=now)).update(
|
|
sync_lock_until=now + timedelta(minutes=3))
|
|
if not claimed:
|
|
raise ConnectionError('A sync is already running. Try again shortly.')
|
|
try:
|
|
connection.refresh_from_db()
|
|
_, local, remote, maps = _snapshot(connection)
|
|
plan = build_plan(connection, local, remote, maps)
|
|
if plan['digest'] != expected_digest:
|
|
raise ConnectionError('The lists changed after the preview. Review a fresh preview.')
|
|
mapping = HomeAssistantItemMap.objects.filter(pk=mapping_id, connection=connection,
|
|
entity_id=connection.grocery_entity_id, pending_create=True).select_related('grocery').first()
|
|
if not mapping or not mapping.grocery_id:
|
|
raise ConnectionError('This pending addition can no longer be reset.')
|
|
if any(item['name'].casefold() == mapping.grocery.name.casefold() for item in remote.values()):
|
|
raise ConnectionError('A matching item is now visible in Home Assistant. Preview again to reconcile it.')
|
|
mapping.delete()
|
|
_record_run(connection, succeeded=True, automatic=False,
|
|
summary=f'Reset uncertain addition for “{mapping.grocery.name}”.')
|
|
finally:
|
|
HomeAssistantConnection.objects.filter(pk=connection.pk).update(sync_lock_until=None)
|
|
|
|
|
|
def run_due_syncs():
|
|
now = timezone.now()
|
|
due = HomeAssistantConnection.objects.filter(auto_sync_enabled=True,
|
|
initial_sync_confirmed=True).exclude(sync_interval_minutes='manual').filter(
|
|
Q(next_sync_at__isnull=True) | Q(next_sync_at__lte=now)).order_by('next_sync_at', 'pk')
|
|
attempted = succeeded = failed = 0
|
|
for connection in due.iterator():
|
|
if connection.sync_lock_until and connection.sync_lock_until > now:
|
|
continue
|
|
attempted += 1
|
|
grocery_failed = False
|
|
try:
|
|
plan = preview(connection)
|
|
confirm_and_sync(connection, plan['digest'], automatic=True)
|
|
succeeded += 1
|
|
except ConnectionError as exc:
|
|
grocery_failed = True
|
|
failed += 1
|
|
connection.refresh_from_db()
|
|
# Preview can fail before confirm_and_sync claims a lock and records history.
|
|
recent = connection.sync_runs.filter(created_at__gte=now).exists()
|
|
if not recent:
|
|
_schedule_after_failure(connection, exc, automatic=True)
|
|
# Optional task sync shares the connection's scheduler and lease, but has
|
|
# its own first-sync confirmation and item mapping namespace.
|
|
connection.refresh_from_db()
|
|
if not grocery_failed and connection.task_entity_id and connection.task_initial_sync_confirmed:
|
|
runs_before = connection.sync_runs.count()
|
|
try:
|
|
from .task_sync import confirm_task_sync, preview_tasks
|
|
task_plan = preview_tasks(connection)
|
|
confirm_task_sync(connection, task_plan['digest'], automatic=True)
|
|
succeeded += 1
|
|
attempted += 1
|
|
except ConnectionError as exc:
|
|
failed += 1
|
|
connection.refresh_from_db()
|
|
if connection.sync_runs.count() == runs_before:
|
|
_schedule_after_failure(connection, exc, automatic=True)
|
|
return {'attempted': attempted, 'succeeded': succeeded, 'failed': failed}
|