Back office: one-way DB -> Sheets mirror

- After each mutation, the affected tab + activity_log are pushed to Sheets by
  a single serialized background worker (Postgres stays source of truth; a
  mirror failure never fails the DB write). Concurrency race fixed by
  serializing through one worker; PYTHONUNBUFFERED for visible logs.
- _cell() formats dates/timestamps/numbers/bools and neutralises formula
  injection. POST /api/sync does a full DB->Sheets resync of every tab.

Verified: add/edit/delete propagate to the Sheet; full resync aligns all tabs;
DB and Sheet consistent after cleanup.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-25 09:26:42 +02:00
parent 9f42e46d6c
commit e59996a385
2 changed files with 93 additions and 3 deletions
+1
View File
@@ -1,5 +1,6 @@
FROM python:3.12-slim FROM python:3.12-slim
ENV PYTHONUNBUFFERED=1
WORKDIR /app WORKDIR /app
COPY requirements.txt . COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt RUN pip install --no-cache-dir -r requirements.txt
+92 -3
View File
@@ -8,18 +8,85 @@ machine-to-machine ingest path (n8n) is gated by the CRM_API_TOKEN header.
import os import os
import re import re
import time import time
from datetime import datetime, date import queue
import threading
import traceback
from datetime import datetime, date, timezone
from decimal import Decimal from decimal import Decimal
from flask import Flask, jsonify, request, Response from flask import Flask, jsonify, request, Response
from waitress import serve from waitress import serve
import db import db
from sheets import Sheets
app = Flask(__name__, static_folder="static", static_url_path="") app = Flask(__name__, static_folder="static", static_url_path="")
CRM_TOKEN = os.environ.get("CRM_API_TOKEN", "") CRM_TOKEN = os.environ.get("CRM_API_TOKEN", "")
# DB -> Sheets one-way mirror. Postgres is the source of truth; the Sheet is a
# best-effort projection. A mirror failure never fails the DB write.
SH = None
try:
SH = Sheets(os.environ["GOOGLE_SA_JSON"], os.environ["SHEET_ID"])
except Exception: # noqa: BLE001
traceback.print_exc()
def _cell(v):
if v is None:
return ""
if isinstance(v, bool):
return "TRUE" if v else "FALSE"
if isinstance(v, datetime):
return v.strftime("%Y-%m-%d %H:%M")
if isinstance(v, date):
return v.strftime("%Y-%m-%d")
if isinstance(v, Decimal):
f = float(v)
return str(int(f)) if f == int(f) else str(f)
s = str(v)
return ("'" + s) if s[:1] in "=+-@" else s # neutralise formula injection
def mirror_entity(entity):
if SH is None:
return 0
spec = db.TABLES[entity]
cols = spec["cols"]
order = LIST_ORDER.get(entity, spec["pk"])
with db.connect() as conn, conn.cursor() as cur:
cur.execute(f"SELECT {', '.join(cols)} FROM {entity} ORDER BY {order}")
rows = cur.fetchall()
grid = [[_cell(r[c]) for c in cols] for r in rows]
SH.overwrite(spec["tab"], cols, grid)
return len(grid)
# Serialize all mirror writes through one worker so concurrent mutations can't
# race on the shared Sheets client / token.
_mirror_q = queue.Queue()
def _mirror_worker():
while True:
entity = _mirror_q.get()
try:
mirror_entity(entity)
except Exception: # noqa: BLE001
print(f"[mirror] {entity} sync failed:", flush=True)
traceback.print_exc()
finally:
_mirror_q.task_done()
threading.Thread(target=_mirror_worker, daemon=True).start()
def mirror_async(entity):
if SH is not None:
_mirror_q.put(entity)
# Static dashboard, with the CRM token injected so the (basic-auth-gated) # Static dashboard, with the CRM token injected so the (basic-auth-gated)
# operator page can call the token-protected mutation endpoints. # operator page can call the token-protected mutation endpoints.
with open(os.path.join(os.path.dirname(__file__), "static", "index.html")) as _f: with open(os.path.join(os.path.dirname(__file__), "static", "index.html")) as _f:
@@ -117,13 +184,13 @@ def add_entity(entity):
with db.connect() as conn, conn.cursor() as cur: with db.connect() as conn, conn.cursor() as cur:
if entity == "leads": if entity == "leads":
row["lead_id"] = row.get("lead_id") or "L-" + str(int(time.time() * 1000)) row["lead_id"] = row.get("lead_id") or "L-" + str(int(time.time() * 1000))
row["received_at"] = row.get("received_at") or datetime.utcnow() row["received_at"] = row.get("received_at") or datetime.now(timezone.utc)
row["status"] = row.get("status") or "new" row["status"] = row.get("status") or "new"
if row.get("notified") is None: if row.get("notified") is None:
row["notified"] = False row["notified"] = False
elif entity == "clients": elif entity == "clients":
row["client_id"] = row.get("client_id") or next_client_id(cur) row["client_id"] = row.get("client_id") or next_client_id(cur)
row["created_at"] = row.get("created_at") or datetime.utcnow() row["created_at"] = row.get("created_at") or datetime.now(timezone.utc)
row["status"] = row.get("status") or "lead" row["status"] = row.get("status") or "lead"
if not row.get("renewal_date") and row.get("start_date"): if not row.get("renewal_date") and row.get("start_date"):
row["renewal_date"] = compute_renewal( row["renewal_date"] = compute_renewal(
@@ -136,6 +203,8 @@ def add_entity(entity):
[row.get(c) for c in cols]) [row.get(c) for c in cols])
log_activity(cur, row.get("client_id"), f"add {entity}", f"{pk}={row.get(pk)}") log_activity(cur, row.get("client_id"), f"add {entity}", f"{pk}={row.get(pk)}")
conn.commit() conn.commit()
mirror_async(entity)
mirror_async("activity_log")
return jsonify({"added": row.get(pk)}), 201 return jsonify({"added": row.get(pk)}), 201
@@ -161,6 +230,8 @@ def edit_entity(entity, ident):
log_activity(cur, ident if entity == "clients" else None, log_activity(cur, ident if entity == "clients" else None,
f"edit {entity}", f"{pk}={ident}: {', '.join(setcols)}") f"edit {entity}", f"{pk}={ident}: {', '.join(setcols)}")
conn.commit() conn.commit()
mirror_async(entity)
mirror_async("activity_log")
return jsonify({"updated": ident, "fields": setcols}) return jsonify({"updated": ident, "fields": setcols})
@@ -178,9 +249,27 @@ def delete_entity(entity, ident):
log_activity(cur, ident if entity == "clients" else None, log_activity(cur, ident if entity == "clients" else None,
f"delete {entity}", f"{pk}={ident}") f"delete {entity}", f"{pk}={ident}")
conn.commit() conn.commit()
mirror_async(entity)
mirror_async("activity_log")
return jsonify({"deleted": ident}) return jsonify({"deleted": ident})
@app.post("/api/sync")
def sync_all():
"""Full DB -> Sheets resync of every tab (manual / alignment)."""
if not authed():
return jsonify({"error": "forbidden"}), 403
if SH is None:
return jsonify({"error": "sheets unavailable"}), 503
out = {}
for entity in db.TABLES:
try:
out[entity] = mirror_entity(entity)
except Exception as e: # noqa: BLE001
out[entity] = f"error: {e}"
return jsonify({"synced": out})
@app.get("/") @app.get("/")
def index(): def index():
return Response(INDEX_HTML, mimetype="text/html") return Response(INDEX_HTML, mimetype="text/html")