Back office: Postgres source-of-truth + read dashboard

Stand up the DB-first CRM backbone (architecture pivot: Postgres is the
source of truth, Google Sheets becomes a one-way downstream mirror).

- backoffice/ stack: smb-db (Postgres 16) + smb-crm (Flask/waitress service).
- Schema mirrors the six Sheet tabs (clients, leads, projects, activity_log,
  bookings, invoices) with typed columns + updated_at triggers.
- Service-account Sheets client (PyJWT) for the one-time import + future mirror.
- import_from_sheets.py: idempotent seed of Postgres from the live Sheets.
- Read dashboard (Leads & Clients tables) at onboard.mivanchenko.de/crm,
  behind the existing Caddy basic-auth; JSON API reads straight from Postgres.

Deployed + verified: import seeded DB, dashboard/API live, no-auth blocked,
onboarding form unaffected. Add/edit/delete + DB->Sheets sync + n8n ingest
swap are the next steps.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-25 09:12:39 +02:00
parent a04ad82e73
commit 18e346d67b
11 changed files with 661 additions and 0 deletions
+2
View File
@@ -7,10 +7,12 @@ dist/
# secrets — never commit credentials # secrets — never commit credentials
.env .env
.env.* .env.*
!.env.example
*.secret *.secret
credentials/ credentials/
.secrets/ .secrets/
stacks/**/.env stacks/**/.env
backoffice/secrets/
# n8n local data # n8n local data
n8n/.n8n/ n8n/.n8n/
+4
View File
@@ -0,0 +1,4 @@
# Copy to .env on the host and fill in. .env and secrets/ are gitignored.
DB_PASSWORD=change-me-strong
CRM_API_TOKEN=change-me-long-random
SHEET_ID=1raMSWRZw_JfHlWqOb3LbhaQ6LWx0VGblxIV4Z2pSzp8
+9
View File
@@ -0,0 +1,9 @@
FROM python:3.12-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY . .
EXPOSE 8080
CMD ["python", "app.py"]
+69
View File
@@ -0,0 +1,69 @@
"""smb-crm back-office service.
Postgres is the source of truth. This serves the operator dashboard + a small
JSON API (read now; add/edit/delete and the DB->Sheets mirror layered on next).
Browser access is gated by Caddy basic-auth on onboard.mivanchenko.de; the
machine-to-machine ingest path (n8n) is gated by the CRM_API_TOKEN header.
"""
import os
from datetime import datetime, date
from decimal import Decimal
from flask import Flask, jsonify, request, Response, send_from_directory
from waitress import serve
import db
app = Flask(__name__, static_folder="static", static_url_path="")
LIST_ORDER = {
"leads": "received_at DESC NULLS LAST",
"clients": "client_id",
"projects": "project_id",
"bookings": "start_time DESC NULLS LAST",
"invoices": "issued_date DESC NULLS LAST",
"activity_log": "ts DESC NULLS LAST",
}
def jsonable(v):
if isinstance(v, (datetime, date)):
return v.isoformat()
if isinstance(v, Decimal):
return float(v)
return v
def rows_json(rows):
return [{k: jsonable(v) for k, v in r.items()} for r in rows]
@app.get("/healthz")
def healthz():
try:
with db.connect() as conn, conn.cursor() as cur:
cur.execute("SELECT 1")
cur.fetchone()
return Response("ok\n", mimetype="text/plain")
except Exception as e: # noqa: BLE001
return Response(f"db error: {e}\n", status=500, mimetype="text/plain")
@app.get("/api/<entity>")
def list_entity(entity):
if entity not in db.TABLES:
return jsonify({"error": "unknown entity"}), 404
order = LIST_ORDER.get(entity, db.TABLES[entity]["pk"])
with db.connect() as conn, conn.cursor() as cur:
cur.execute(f"SELECT * FROM {entity} ORDER BY {order}")
rows = cur.fetchall()
return jsonify({"entity": entity, "count": len(rows), "rows": rows_json(rows)})
@app.get("/")
def index():
return send_from_directory(app.static_folder, "index.html")
if __name__ == "__main__":
serve(app, host="0.0.0.0", port=8080)
+148
View File
@@ -0,0 +1,148 @@
"""Postgres access + the table/column contract shared across the app.
The TABLES map is the single definition of which entities exist, their columns,
and their primary key. Read/CRUD endpoints, the importer and the Sheets mirror
all derive from it so they can never drift apart.
"""
import os
import re
from datetime import datetime, date
import psycopg
from psycopg.rows import dict_row
DATABASE_URL = os.environ["DATABASE_URL"]
# entity -> (sheet tab, primary key, ordered columns)
TABLES = {
"clients": {
"tab": "Clients",
"pk": "client_id",
"cols": ["client_id", "business_name", "owner_name", "email", "phone",
"niche", "tier", "status", "domain", "stack_notes", "vault_ref",
"services", "billing_cycle", "monthly_fee_eur", "start_date",
"renewal_date", "created_at", "notes"],
"dates": ["start_date", "renewal_date"],
"timestamps": ["created_at"],
"numbers": ["monthly_fee_eur"],
"bools": [],
},
"leads": {
"tab": "Leads",
"pk": "lead_id",
"cols": ["lead_id", "received_at", "client_id", "source", "name",
"contact", "service_interest", "message", "status", "notified"],
"dates": [],
"timestamps": ["received_at"],
"numbers": [],
"bools": ["notified"],
},
"projects": {
"tab": "Projects",
"pk": "project_id",
"cols": ["project_id", "client_id", "deliverable", "tier", "checklist",
"go_live_date", "status"],
"dates": ["go_live_date"],
"timestamps": [],
"numbers": [],
"bools": [],
},
"bookings": {
"tab": "Bookings",
"pk": "booking_id",
"cols": ["booking_id", "created_at", "client_id", "customer_name",
"customer_contact", "service", "start_time", "end_time",
"source", "status"],
"dates": [],
"timestamps": ["created_at", "start_time", "end_time"],
"numbers": [],
"bools": [],
},
"invoices": {
"tab": "Invoices",
"pk": "invoice_id",
"cols": ["invoice_id", "client_id", "issued_date", "due_date",
"amount_eur", "period", "status", "paid_date"],
"dates": ["issued_date", "due_date", "paid_date"],
"timestamps": [],
"numbers": ["amount_eur"],
"bools": [],
},
"activity_log": {
"tab": "Activity Log",
"pk": "id",
"cols": ["ts", "workflow", "client_id", "action", "detail", "result"],
"dates": [],
"timestamps": ["ts"],
"numbers": [],
"bools": [],
},
}
def connect():
return psycopg.connect(DATABASE_URL, row_factory=dict_row)
# ---- coercion: Sheet strings / JSON values -> typed Python for Postgres ----
def parse_date(v):
if v in (None, ""):
return None
if isinstance(v, date):
return v
m = re.match(r"(\d{4})-(\d{2})-(\d{2})", str(v))
return date(int(m.group(1)), int(m.group(2)), int(m.group(3))) if m else None
def parse_ts(v):
if v in (None, ""):
return None
if isinstance(v, datetime):
return v
s = str(v).strip()
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%dT%H:%M:%S",
"%Y-%m-%dT%H:%M", "%Y-%m-%d"):
try:
return datetime.strptime(s, fmt)
except ValueError:
continue
return None
def parse_num(v):
if v in (None, ""):
return None
s = str(v).replace("", "").replace(",", ".").strip()
try:
return float(s)
except ValueError:
return None
def parse_bool(v):
if v in (None, ""):
return None
if isinstance(v, bool):
return v
return str(v).strip().lower() in ("true", "1", "yes", "ja", "wahr")
def coerce_row(entity, rec):
"""Return a dict of column -> typed value for the given entity."""
spec = TABLES[entity]
out = {}
for col in spec["cols"]:
v = rec.get(col, None)
if isinstance(v, str):
v = v.strip() or None
if col in spec["dates"]:
v = parse_date(v)
elif col in spec["timestamps"]:
v = parse_ts(v)
elif col in spec["numbers"]:
v = parse_num(v)
elif col in spec["bools"]:
v = parse_bool(v)
out[col] = v
return out
+55
View File
@@ -0,0 +1,55 @@
"""One-time (idempotent) seed of Postgres from the existing Google Sheets.
This is the ONLY place the Sheet is treated as authoritative — to bootstrap the
DB. After this, Postgres is the source of truth and Sheets is a downstream
mirror. Safe to re-run: keyed tables upsert by primary key; activity_log is
replaced wholesale.
Run inside the container: docker compose exec smb-crm python import_from_sheets.py
"""
import os
import sys
import db
from sheets import Sheets
def upsert(cur, entity, records):
spec = db.TABLES[entity]
cols = spec["cols"]
pk = spec["pk"]
n = 0
for rec in records:
row = db.coerce_row(entity, rec)
if entity != "activity_log" and not row.get(pk):
continue # skip rows without a primary key
placeholders = ", ".join(["%s"] * len(cols))
collist = ", ".join(cols)
if entity == "activity_log":
cur.execute(f"INSERT INTO {entity} ({collist}) VALUES ({placeholders})",
[row[c] for c in cols])
else:
updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in cols if c != pk)
cur.execute(
f"INSERT INTO {entity} ({collist}) VALUES ({placeholders}) "
f"ON CONFLICT ({pk}) DO UPDATE SET {updates}",
[row[c] for c in cols])
n += 1
return n
def main():
sh = Sheets(os.environ["GOOGLE_SA_JSON"], os.environ["SHEET_ID"])
with db.connect() as conn:
with conn.cursor() as cur:
cur.execute("TRUNCATE activity_log RESTART IDENTITY")
for entity, spec in db.TABLES.items():
records = sh.read_records(spec["tab"])
n = upsert(cur, entity, records)
print(f" {entity:<13} <- {spec['tab']:<13} {n} row(s)")
conn.commit()
print("import complete.")
if __name__ == "__main__":
sys.exit(main())
+5
View File
@@ -0,0 +1,5 @@
Flask==3.0.3
psycopg[binary]==3.2.1
waitress==3.0.0
PyJWT[crypto]==2.9.0
requests==2.32.3
+98
View File
@@ -0,0 +1,98 @@
"""Minimal Google Sheets v4 client using a service-account JWT.
Used for (a) the one-time import of existing Sheet rows into Postgres and
(b) the one-way DB -> Sheets mirror sync. Postgres stays the source of truth;
nothing here treats the Sheet as authoritative except the explicit import.
"""
import json
import time
import threading
import jwt
import requests
SHEETS_API = "https://sheets.googleapis.com/v4/spreadsheets"
TOKEN_URL = "https://oauth2.googleapis.com/token"
SCOPE = "https://www.googleapis.com/auth/spreadsheets"
# gid map for the live workbook (tab name -> gid), used when clearing/sizing.
TAB_GIDS = {
"Clients": 470934735,
"Leads": 571719114,
"Bookings": 946084008,
"Projects": 619112786,
"Invoices": 413377229,
"Activity Log": 170940481,
}
class Sheets:
def __init__(self, sa_json_path, sheet_id):
with open(sa_json_path) as f:
self.sa = json.load(f)
self.sheet_id = sheet_id
self._tok = None
self._exp = 0
self._lock = threading.Lock()
def _token(self):
with self._lock:
now = int(time.time())
if self._tok and now < self._exp - 60:
return self._tok
claim = {
"iss": self.sa["client_email"],
"scope": SCOPE,
"aud": TOKEN_URL,
"iat": now,
"exp": now + 3600,
}
assertion = jwt.encode(claim, self.sa["private_key"], algorithm="RS256")
r = requests.post(TOKEN_URL, data={
"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
"assertion": assertion,
}, timeout=30)
r.raise_for_status()
self._tok = r.json()["access_token"]
self._exp = now + 3600
return self._tok
def _headers(self):
return {"Authorization": "Bearer " + self._token()}
def read(self, a1_range):
"""Return raw 2D list of cell values for an A1 range (e.g. "Leads!A1:Z")."""
url = f"{SHEETS_API}/{self.sheet_id}/values/{requests.utils.quote(a1_range)}"
r = requests.get(url, headers=self._headers(), timeout=30)
r.raise_for_status()
return r.json().get("values", [])
def read_records(self, tab):
"""Read a whole tab as a list of dicts keyed by the header row."""
rows = self.read(f"{tab}!A1:Z")
if not rows:
return []
header = rows[0]
out = []
for raw in rows[1:]:
if not any(c.strip() for c in raw):
continue
rec = {header[i]: (raw[i] if i < len(raw) else "") for i in range(len(header))}
out.append(rec)
return out
def overwrite(self, tab, header, rows):
"""Replace a tab's contents with header + rows (the DB->Sheets mirror).
Clears the existing value range, then writes the new grid starting at A1.
Postgres is the source of truth; this projects it onto the Sheet.
"""
# clear current values (keeps formatting / the tab itself)
clr = f"{SHEETS_API}/{self.sheet_id}/values/{requests.utils.quote(tab + '!A1:Z')}:clear"
requests.post(clr, headers=self._headers(), timeout=30).raise_for_status()
body = {"values": [header] + rows}
url = (f"{SHEETS_API}/{self.sheet_id}/values/"
f"{requests.utils.quote(tab + '!A1')}?valueInputOption=RAW")
r = requests.put(url, headers=self._headers(), json=body, timeout=60)
r.raise_for_status()
return len(rows)
+124
View File
@@ -0,0 +1,124 @@
<!DOCTYPE html>
<html lang="de">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="robots" content="noindex" />
<title>smb-crm — Back Office</title>
<style>
:root {
--bg: #f6f8f8; --surface: #fff; --ink: #16302f; --muted: #5d716f;
--teal: #0f8a7e; --teal-2: #0b6b62; --teal-soft: #e4f3f1; --line: #e3eae9;
--shadow: 0 10px 30px rgba(16,48,47,.08);
}
* { box-sizing: border-box; }
body { margin: 0; font-family: system-ui, -apple-system, Segoe UI, Roboto, sans-serif;
background: var(--bg); color: var(--ink); }
header { background: var(--surface); border-bottom: 1px solid var(--line); padding: 16px 24px;
display: flex; align-items: center; justify-content: space-between; position: sticky; top: 0; z-index: 5; }
header h1 { font-size: 1.15rem; margin: 0; }
header .src { font-size: .74rem; color: var(--muted); font-weight: 500; }
.wrap { max-width: 1280px; margin: 0 auto; padding: 22px 24px 60px; }
.tabs { display: flex; gap: 8px; margin-bottom: 18px; }
.tab { background: var(--surface); border: 1px solid var(--line); border-radius: 999px;
padding: 9px 18px; font-size: .9rem; font-weight: 600; color: var(--muted); cursor: pointer; }
.tab.active { background: var(--teal); color: #fff; border-color: var(--teal); }
.tab .n { opacity: .7; font-weight: 500; margin-left: 5px; }
.bar { display: flex; align-items: center; gap: 12px; margin-bottom: 14px; }
.bar input { flex: 1; max-width: 340px; padding: 9px 13px; border: 1px solid var(--line);
border-radius: 9px; font: inherit; font-size: .9rem; background: var(--surface); }
.btn { background: var(--teal); color: #fff; border: 0; border-radius: 9px; padding: 9px 16px;
font: inherit; font-size: .88rem; font-weight: 600; cursor: pointer; }
.btn.ghost { background: var(--surface); color: var(--teal-2); border: 1px solid var(--line); }
.card { background: var(--surface); border: 1px solid var(--line); border-radius: 14px;
box-shadow: var(--shadow); overflow: auto; }
table { width: 100%; border-collapse: collapse; font-size: .85rem; }
th, td { text-align: left; padding: 10px 13px; border-bottom: 1px solid var(--line); white-space: nowrap; }
th { background: #f0f5f4; color: var(--muted); font-size: .72rem; text-transform: uppercase;
letter-spacing: .5px; position: sticky; top: 0; }
td { max-width: 280px; overflow: hidden; text-overflow: ellipsis; }
tr:hover td { background: #fafdfc; }
.pillv { display: inline-block; padding: 2px 9px; border-radius: 999px; font-size: .74rem; font-weight: 600;
background: var(--teal-soft); color: var(--teal-2); }
.empty { padding: 40px; text-align: center; color: var(--muted); }
.msg { padding: 10px 14px; border-radius: 9px; font-size: .85rem; margin-bottom: 12px; display: none; }
.msg.err { background: #fdecec; color: #9b2226; border: 1px solid #f3c7c7; display: block; }
</style>
</head>
<body>
<header>
<h1>🗂️ smb-crm · Back Office</h1>
<span class="src">Quelle: Postgres (Sheets = Spiegel) · <span id="now"></span></span>
</header>
<div class="wrap">
<div class="tabs">
<button class="tab active" data-e="leads">Leads<span class="n" id="n-leads"></span></button>
<button class="tab" data-e="clients">Clients<span class="n" id="n-clients"></span></button>
</div>
<div class="bar">
<input id="search" placeholder="Filtern …" />
<button class="btn ghost" id="refresh">↻ Aktualisieren</button>
</div>
<div class="msg" id="msg"></div>
<div class="card"><div id="table"><div class="empty">Lädt …</div></div></div>
</div>
<script>
const API = 'api';
const COLS = {
leads: ['lead_id','received_at','client_id','source','name','contact','service_interest','message','status','notified'],
clients: ['client_id','business_name','owner_name','email','phone','niche','tier','status','billing_cycle','monthly_fee_eur','renewal_date','created_at'],
};
let current = 'leads';
let cache = {};
const esc = s => (s==null?'':String(s)).replace(/[&<>"]/g, c => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;'}[c]));
function showErr(t){ const m=document.getElementById('msg'); m.textContent=t; m.className='msg err'; }
function clearErr(){ document.getElementById('msg').className='msg'; }
async function load(entity){
clearErr();
document.getElementById('table').innerHTML = '<div class="empty">Lädt …</div>';
try {
const r = await fetch(`${API}/${entity}`);
if(!r.ok) throw new Error('HTTP '+r.status);
const d = await r.json();
cache[entity] = d.rows;
document.getElementById('n-'+entity).textContent = d.count;
render();
} catch(e){
showErr('Laden fehlgeschlagen: '+e.message);
document.getElementById('table').innerHTML = '<div class="empty">—</div>';
}
}
function render(){
const cols = COLS[current];
let rows = cache[current] || [];
const q = document.getElementById('search').value.trim().toLowerCase();
if(q) rows = rows.filter(r => cols.some(c => String(r[c]??'').toLowerCase().includes(q)));
if(!rows.length){ document.getElementById('table').innerHTML = '<div class="empty">Keine Einträge.</div>'; return; }
const head = '<tr>' + cols.map(c => `<th>${c}</th>`).join('') + '</tr>';
const body = rows.map(r => '<tr>' + cols.map(c => {
let v = r[c];
if(c==='status' || c==='tier') return `<td><span class="pillv">${esc(v)}</span></td>`;
if((c==='received_at'||c==='created_at') && v) v = String(v).replace('T',' ').slice(0,16);
return `<td title="${esc(v)}">${esc(v)}</td>`;
}).join('') + '</tr>').join('');
document.getElementById('table').innerHTML = `<table><thead>${head}</thead><tbody>${body}</tbody></table>`;
}
document.querySelectorAll('.tab').forEach(t => t.onclick = () => {
document.querySelectorAll('.tab').forEach(x => x.classList.remove('active'));
t.classList.add('active');
current = t.dataset.e;
if(cache[current]) render(); else load(current);
});
document.getElementById('search').oninput = render;
document.getElementById('refresh').onclick = () => load(current);
document.getElementById('now').textContent = new Date().toLocaleString('de-DE');
load('leads'); load('clients');
</script>
</body>
</html>
+105
View File
@@ -0,0 +1,105 @@
-- smb-crm — source-of-truth schema (Postgres).
-- Google Sheets is a one-way downstream mirror fed by the DB->Sheets sync.
-- Column sets mirror the Sheet tabs so the mirror stays 1:1.
CREATE TABLE IF NOT EXISTS clients (
client_id text PRIMARY KEY,
business_name text,
owner_name text,
email text,
phone text,
niche text,
tier text,
status text,
domain text,
stack_notes text,
vault_ref text,
services text,
billing_cycle text,
monthly_fee_eur numeric,
start_date date,
renewal_date date,
created_at timestamptz,
notes text,
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS leads (
lead_id text PRIMARY KEY,
received_at timestamptz,
client_id text,
source text,
name text,
contact text,
service_interest text,
message text,
status text,
notified boolean,
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS projects (
project_id text PRIMARY KEY,
client_id text,
deliverable text,
tier text,
checklist text,
go_live_date date,
status text,
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS activity_log (
id bigserial PRIMARY KEY,
ts timestamptz,
workflow text,
client_id text,
action text,
detail text,
result text
);
CREATE TABLE IF NOT EXISTS bookings (
booking_id text PRIMARY KEY,
created_at timestamptz,
client_id text,
customer_name text,
customer_contact text,
service text,
start_time timestamptz,
end_time timestamptz,
source text,
status text,
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS invoices (
invoice_id text PRIMARY KEY,
client_id text,
issued_date date,
due_date date,
amount_eur numeric,
period text,
status text,
paid_date date,
updated_at timestamptz NOT NULL DEFAULT now()
);
-- keep updated_at fresh on row changes
CREATE OR REPLACE FUNCTION touch_updated_at() RETURNS trigger AS $$
BEGIN NEW.updated_at = now(); RETURN NEW; END;
$$ LANGUAGE plpgsql;
DO $$
DECLARE t text;
BEGIN
FOREACH t IN ARRAY ARRAY['clients','leads','projects','bookings','invoices'] LOOP
EXECUTE format(
'CREATE TRIGGER %I_touch BEFORE UPDATE ON %I FOR EACH ROW EXECUTE FUNCTION touch_updated_at()',
t, t);
END LOOP;
END $$;
CREATE INDEX IF NOT EXISTS leads_received_idx ON leads (received_at DESC);
CREATE INDEX IF NOT EXISTS clients_status_idx ON clients (status);
CREATE INDEX IF NOT EXISTS activity_ts_idx ON activity_log (ts DESC);
+42
View File
@@ -0,0 +1,42 @@
services:
smb-db:
image: postgres:16-alpine
container_name: smb-db
restart: unless-stopped
environment:
POSTGRES_DB: smbcrm
POSTGRES_USER: smbcrm
POSTGRES_PASSWORD: ${DB_PASSWORD}
volumes:
- smb-db-data:/var/lib/postgresql/data
- ./db/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
networks: [smb-net]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U smbcrm -d smbcrm"]
interval: 10s
timeout: 5s
retries: 5
smb-crm:
build: ./app
container_name: smb-crm
restart: unless-stopped
environment:
DATABASE_URL: postgresql://smbcrm:${DB_PASSWORD}@smb-db:5432/smbcrm
CRM_API_TOKEN: ${CRM_API_TOKEN}
SHEET_ID: ${SHEET_ID}
GOOGLE_SA_JSON: /run/secrets/gcp-sa.json
volumes:
- ./secrets/gcp-sa.json:/run/secrets/gcp-sa.json:ro
depends_on:
smb-db:
condition: service_healthy
networks: [smb-net, proxy]
volumes:
smb-db-data:
networks:
smb-net:
proxy:
external: true