From 18e346d67bd84029d12af1c92cfb7a5561b36eeb Mon Sep 17 00:00:00 2001 From: rogalik27 Date: Thu, 25 Jun 2026 09:12:39 +0200 Subject: [PATCH] Back office: Postgres source-of-truth + read dashboard Stand up the DB-first CRM backbone (architecture pivot: Postgres is the source of truth, Google Sheets becomes a one-way downstream mirror). - backoffice/ stack: smb-db (Postgres 16) + smb-crm (Flask/waitress service). - Schema mirrors the six Sheet tabs (clients, leads, projects, activity_log, bookings, invoices) with typed columns + updated_at triggers. - Service-account Sheets client (PyJWT) for the one-time import + future mirror. - import_from_sheets.py: idempotent seed of Postgres from the live Sheets. - Read dashboard (Leads & Clients tables) at onboard.mivanchenko.de/crm, behind the existing Caddy basic-auth; JSON API reads straight from Postgres. Deployed + verified: import seeded DB, dashboard/API live, no-auth blocked, onboarding form unaffected. Add/edit/delete + DB->Sheets sync + n8n ingest swap are the next steps. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 2 + backoffice/.env.example | 4 + backoffice/app/Dockerfile | 9 ++ backoffice/app/app.py | 69 +++++++++++++ backoffice/app/db.py | 148 +++++++++++++++++++++++++++ backoffice/app/import_from_sheets.py | 55 ++++++++++ backoffice/app/requirements.txt | 5 + backoffice/app/sheets.py | 98 ++++++++++++++++++ backoffice/app/static/index.html | 124 ++++++++++++++++++++++ backoffice/db/init.sql | 105 +++++++++++++++++++ backoffice/docker-compose.yml | 42 ++++++++ 11 files changed, 661 insertions(+) create mode 100644 backoffice/.env.example create mode 100644 backoffice/app/Dockerfile create mode 100644 backoffice/app/app.py create mode 100644 backoffice/app/db.py create mode 100644 backoffice/app/import_from_sheets.py create mode 100644 backoffice/app/requirements.txt create mode 100644 backoffice/app/sheets.py create mode 100644 backoffice/app/static/index.html create mode 100644 backoffice/db/init.sql create mode 100644 backoffice/docker-compose.yml diff --git a/.gitignore b/.gitignore index fa7fc29..16d47a0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,10 +7,12 @@ dist/ # secrets — never commit credentials .env .env.* +!.env.example *.secret credentials/ .secrets/ stacks/**/.env +backoffice/secrets/ # n8n local data n8n/.n8n/ diff --git a/backoffice/.env.example b/backoffice/.env.example new file mode 100644 index 0000000..1ab33f0 --- /dev/null +++ b/backoffice/.env.example @@ -0,0 +1,4 @@ +# Copy to .env on the host and fill in. .env and secrets/ are gitignored. +DB_PASSWORD=change-me-strong +CRM_API_TOKEN=change-me-long-random +SHEET_ID=1raMSWRZw_JfHlWqOb3LbhaQ6LWx0VGblxIV4Z2pSzp8 diff --git a/backoffice/app/Dockerfile b/backoffice/app/Dockerfile new file mode 100644 index 0000000..60524c3 --- /dev/null +++ b/backoffice/app/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.12-slim + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY . . + +EXPOSE 8080 +CMD ["python", "app.py"] diff --git a/backoffice/app/app.py b/backoffice/app/app.py new file mode 100644 index 0000000..aa5d186 --- /dev/null +++ b/backoffice/app/app.py @@ -0,0 +1,69 @@ +"""smb-crm back-office service. + +Postgres is the source of truth. This serves the operator dashboard + a small +JSON API (read now; add/edit/delete and the DB->Sheets mirror layered on next). +Browser access is gated by Caddy basic-auth on onboard.mivanchenko.de; the +machine-to-machine ingest path (n8n) is gated by the CRM_API_TOKEN header. +""" +import os +from datetime import datetime, date +from decimal import Decimal + +from flask import Flask, jsonify, request, Response, send_from_directory +from waitress import serve + +import db + +app = Flask(__name__, static_folder="static", static_url_path="") + +LIST_ORDER = { + "leads": "received_at DESC NULLS LAST", + "clients": "client_id", + "projects": "project_id", + "bookings": "start_time DESC NULLS LAST", + "invoices": "issued_date DESC NULLS LAST", + "activity_log": "ts DESC NULLS LAST", +} + + +def jsonable(v): + if isinstance(v, (datetime, date)): + return v.isoformat() + if isinstance(v, Decimal): + return float(v) + return v + + +def rows_json(rows): + return [{k: jsonable(v) for k, v in r.items()} for r in rows] + + +@app.get("/healthz") +def healthz(): + try: + with db.connect() as conn, conn.cursor() as cur: + cur.execute("SELECT 1") + cur.fetchone() + return Response("ok\n", mimetype="text/plain") + except Exception as e: # noqa: BLE001 + return Response(f"db error: {e}\n", status=500, mimetype="text/plain") + + +@app.get("/api/") +def list_entity(entity): + if entity not in db.TABLES: + return jsonify({"error": "unknown entity"}), 404 + order = LIST_ORDER.get(entity, db.TABLES[entity]["pk"]) + with db.connect() as conn, conn.cursor() as cur: + cur.execute(f"SELECT * FROM {entity} ORDER BY {order}") + rows = cur.fetchall() + return jsonify({"entity": entity, "count": len(rows), "rows": rows_json(rows)}) + + +@app.get("/") +def index(): + return send_from_directory(app.static_folder, "index.html") + + +if __name__ == "__main__": + serve(app, host="0.0.0.0", port=8080) diff --git a/backoffice/app/db.py b/backoffice/app/db.py new file mode 100644 index 0000000..58c5213 --- /dev/null +++ b/backoffice/app/db.py @@ -0,0 +1,148 @@ +"""Postgres access + the table/column contract shared across the app. + +The TABLES map is the single definition of which entities exist, their columns, +and their primary key. Read/CRUD endpoints, the importer and the Sheets mirror +all derive from it so they can never drift apart. +""" +import os +import re +from datetime import datetime, date + +import psycopg +from psycopg.rows import dict_row + +DATABASE_URL = os.environ["DATABASE_URL"] + +# entity -> (sheet tab, primary key, ordered columns) +TABLES = { + "clients": { + "tab": "Clients", + "pk": "client_id", + "cols": ["client_id", "business_name", "owner_name", "email", "phone", + "niche", "tier", "status", "domain", "stack_notes", "vault_ref", + "services", "billing_cycle", "monthly_fee_eur", "start_date", + "renewal_date", "created_at", "notes"], + "dates": ["start_date", "renewal_date"], + "timestamps": ["created_at"], + "numbers": ["monthly_fee_eur"], + "bools": [], + }, + "leads": { + "tab": "Leads", + "pk": "lead_id", + "cols": ["lead_id", "received_at", "client_id", "source", "name", + "contact", "service_interest", "message", "status", "notified"], + "dates": [], + "timestamps": ["received_at"], + "numbers": [], + "bools": ["notified"], + }, + "projects": { + "tab": "Projects", + "pk": "project_id", + "cols": ["project_id", "client_id", "deliverable", "tier", "checklist", + "go_live_date", "status"], + "dates": ["go_live_date"], + "timestamps": [], + "numbers": [], + "bools": [], + }, + "bookings": { + "tab": "Bookings", + "pk": "booking_id", + "cols": ["booking_id", "created_at", "client_id", "customer_name", + "customer_contact", "service", "start_time", "end_time", + "source", "status"], + "dates": [], + "timestamps": ["created_at", "start_time", "end_time"], + "numbers": [], + "bools": [], + }, + "invoices": { + "tab": "Invoices", + "pk": "invoice_id", + "cols": ["invoice_id", "client_id", "issued_date", "due_date", + "amount_eur", "period", "status", "paid_date"], + "dates": ["issued_date", "due_date", "paid_date"], + "timestamps": [], + "numbers": ["amount_eur"], + "bools": [], + }, + "activity_log": { + "tab": "Activity Log", + "pk": "id", + "cols": ["ts", "workflow", "client_id", "action", "detail", "result"], + "dates": [], + "timestamps": ["ts"], + "numbers": [], + "bools": [], + }, +} + + +def connect(): + return psycopg.connect(DATABASE_URL, row_factory=dict_row) + + +# ---- coercion: Sheet strings / JSON values -> typed Python for Postgres ---- + +def parse_date(v): + if v in (None, ""): + return None + if isinstance(v, date): + return v + m = re.match(r"(\d{4})-(\d{2})-(\d{2})", str(v)) + return date(int(m.group(1)), int(m.group(2)), int(m.group(3))) if m else None + + +def parse_ts(v): + if v in (None, ""): + return None + if isinstance(v, datetime): + return v + s = str(v).strip() + for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%dT%H:%M", "%Y-%m-%d"): + try: + return datetime.strptime(s, fmt) + except ValueError: + continue + return None + + +def parse_num(v): + if v in (None, ""): + return None + s = str(v).replace("€", "").replace(",", ".").strip() + try: + return float(s) + except ValueError: + return None + + +def parse_bool(v): + if v in (None, ""): + return None + if isinstance(v, bool): + return v + return str(v).strip().lower() in ("true", "1", "yes", "ja", "wahr") + + +def coerce_row(entity, rec): + """Return a dict of column -> typed value for the given entity.""" + spec = TABLES[entity] + out = {} + for col in spec["cols"]: + v = rec.get(col, None) + if isinstance(v, str): + v = v.strip() or None + if col in spec["dates"]: + v = parse_date(v) + elif col in spec["timestamps"]: + v = parse_ts(v) + elif col in spec["numbers"]: + v = parse_num(v) + elif col in spec["bools"]: + v = parse_bool(v) + out[col] = v + return out diff --git a/backoffice/app/import_from_sheets.py b/backoffice/app/import_from_sheets.py new file mode 100644 index 0000000..79c2375 --- /dev/null +++ b/backoffice/app/import_from_sheets.py @@ -0,0 +1,55 @@ +"""One-time (idempotent) seed of Postgres from the existing Google Sheets. + +This is the ONLY place the Sheet is treated as authoritative — to bootstrap the +DB. After this, Postgres is the source of truth and Sheets is a downstream +mirror. Safe to re-run: keyed tables upsert by primary key; activity_log is +replaced wholesale. + +Run inside the container: docker compose exec smb-crm python import_from_sheets.py +""" +import os +import sys + +import db +from sheets import Sheets + + +def upsert(cur, entity, records): + spec = db.TABLES[entity] + cols = spec["cols"] + pk = spec["pk"] + n = 0 + for rec in records: + row = db.coerce_row(entity, rec) + if entity != "activity_log" and not row.get(pk): + continue # skip rows without a primary key + placeholders = ", ".join(["%s"] * len(cols)) + collist = ", ".join(cols) + if entity == "activity_log": + cur.execute(f"INSERT INTO {entity} ({collist}) VALUES ({placeholders})", + [row[c] for c in cols]) + else: + updates = ", ".join(f"{c} = EXCLUDED.{c}" for c in cols if c != pk) + cur.execute( + f"INSERT INTO {entity} ({collist}) VALUES ({placeholders}) " + f"ON CONFLICT ({pk}) DO UPDATE SET {updates}", + [row[c] for c in cols]) + n += 1 + return n + + +def main(): + sh = Sheets(os.environ["GOOGLE_SA_JSON"], os.environ["SHEET_ID"]) + with db.connect() as conn: + with conn.cursor() as cur: + cur.execute("TRUNCATE activity_log RESTART IDENTITY") + for entity, spec in db.TABLES.items(): + records = sh.read_records(spec["tab"]) + n = upsert(cur, entity, records) + print(f" {entity:<13} <- {spec['tab']:<13} {n} row(s)") + conn.commit() + print("import complete.") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backoffice/app/requirements.txt b/backoffice/app/requirements.txt new file mode 100644 index 0000000..2b0347b --- /dev/null +++ b/backoffice/app/requirements.txt @@ -0,0 +1,5 @@ +Flask==3.0.3 +psycopg[binary]==3.2.1 +waitress==3.0.0 +PyJWT[crypto]==2.9.0 +requests==2.32.3 diff --git a/backoffice/app/sheets.py b/backoffice/app/sheets.py new file mode 100644 index 0000000..2ddc3e9 --- /dev/null +++ b/backoffice/app/sheets.py @@ -0,0 +1,98 @@ +"""Minimal Google Sheets v4 client using a service-account JWT. + +Used for (a) the one-time import of existing Sheet rows into Postgres and +(b) the one-way DB -> Sheets mirror sync. Postgres stays the source of truth; +nothing here treats the Sheet as authoritative except the explicit import. +""" +import json +import time +import threading + +import jwt +import requests + +SHEETS_API = "https://sheets.googleapis.com/v4/spreadsheets" +TOKEN_URL = "https://oauth2.googleapis.com/token" +SCOPE = "https://www.googleapis.com/auth/spreadsheets" + +# gid map for the live workbook (tab name -> gid), used when clearing/sizing. +TAB_GIDS = { + "Clients": 470934735, + "Leads": 571719114, + "Bookings": 946084008, + "Projects": 619112786, + "Invoices": 413377229, + "Activity Log": 170940481, +} + + +class Sheets: + def __init__(self, sa_json_path, sheet_id): + with open(sa_json_path) as f: + self.sa = json.load(f) + self.sheet_id = sheet_id + self._tok = None + self._exp = 0 + self._lock = threading.Lock() + + def _token(self): + with self._lock: + now = int(time.time()) + if self._tok and now < self._exp - 60: + return self._tok + claim = { + "iss": self.sa["client_email"], + "scope": SCOPE, + "aud": TOKEN_URL, + "iat": now, + "exp": now + 3600, + } + assertion = jwt.encode(claim, self.sa["private_key"], algorithm="RS256") + r = requests.post(TOKEN_URL, data={ + "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + "assertion": assertion, + }, timeout=30) + r.raise_for_status() + self._tok = r.json()["access_token"] + self._exp = now + 3600 + return self._tok + + def _headers(self): + return {"Authorization": "Bearer " + self._token()} + + def read(self, a1_range): + """Return raw 2D list of cell values for an A1 range (e.g. "Leads!A1:Z").""" + url = f"{SHEETS_API}/{self.sheet_id}/values/{requests.utils.quote(a1_range)}" + r = requests.get(url, headers=self._headers(), timeout=30) + r.raise_for_status() + return r.json().get("values", []) + + def read_records(self, tab): + """Read a whole tab as a list of dicts keyed by the header row.""" + rows = self.read(f"{tab}!A1:Z") + if not rows: + return [] + header = rows[0] + out = [] + for raw in rows[1:]: + if not any(c.strip() for c in raw): + continue + rec = {header[i]: (raw[i] if i < len(raw) else "") for i in range(len(header))} + out.append(rec) + return out + + def overwrite(self, tab, header, rows): + """Replace a tab's contents with header + rows (the DB->Sheets mirror). + + Clears the existing value range, then writes the new grid starting at A1. + Postgres is the source of truth; this projects it onto the Sheet. + """ + # clear current values (keeps formatting / the tab itself) + clr = f"{SHEETS_API}/{self.sheet_id}/values/{requests.utils.quote(tab + '!A1:Z')}:clear" + requests.post(clr, headers=self._headers(), timeout=30).raise_for_status() + body = {"values": [header] + rows} + url = (f"{SHEETS_API}/{self.sheet_id}/values/" + f"{requests.utils.quote(tab + '!A1')}?valueInputOption=RAW") + r = requests.put(url, headers=self._headers(), json=body, timeout=60) + r.raise_for_status() + return len(rows) diff --git a/backoffice/app/static/index.html b/backoffice/app/static/index.html new file mode 100644 index 0000000..ad27ffc --- /dev/null +++ b/backoffice/app/static/index.html @@ -0,0 +1,124 @@ + + + + + + + smb-crm — Back Office + + + +
+

🗂️ smb-crm · Back Office

+ Quelle: Postgres (Sheets = Spiegel) · +
+
+
+ + +
+
+ + +
+
+
Lädt …
+
+ + + + diff --git a/backoffice/db/init.sql b/backoffice/db/init.sql new file mode 100644 index 0000000..5b12ac4 --- /dev/null +++ b/backoffice/db/init.sql @@ -0,0 +1,105 @@ +-- smb-crm — source-of-truth schema (Postgres). +-- Google Sheets is a one-way downstream mirror fed by the DB->Sheets sync. +-- Column sets mirror the Sheet tabs so the mirror stays 1:1. + +CREATE TABLE IF NOT EXISTS clients ( + client_id text PRIMARY KEY, + business_name text, + owner_name text, + email text, + phone text, + niche text, + tier text, + status text, + domain text, + stack_notes text, + vault_ref text, + services text, + billing_cycle text, + monthly_fee_eur numeric, + start_date date, + renewal_date date, + created_at timestamptz, + notes text, + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS leads ( + lead_id text PRIMARY KEY, + received_at timestamptz, + client_id text, + source text, + name text, + contact text, + service_interest text, + message text, + status text, + notified boolean, + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS projects ( + project_id text PRIMARY KEY, + client_id text, + deliverable text, + tier text, + checklist text, + go_live_date date, + status text, + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS activity_log ( + id bigserial PRIMARY KEY, + ts timestamptz, + workflow text, + client_id text, + action text, + detail text, + result text +); + +CREATE TABLE IF NOT EXISTS bookings ( + booking_id text PRIMARY KEY, + created_at timestamptz, + client_id text, + customer_name text, + customer_contact text, + service text, + start_time timestamptz, + end_time timestamptz, + source text, + status text, + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS invoices ( + invoice_id text PRIMARY KEY, + client_id text, + issued_date date, + due_date date, + amount_eur numeric, + period text, + status text, + paid_date date, + updated_at timestamptz NOT NULL DEFAULT now() +); + +-- keep updated_at fresh on row changes +CREATE OR REPLACE FUNCTION touch_updated_at() RETURNS trigger AS $$ +BEGIN NEW.updated_at = now(); RETURN NEW; END; +$$ LANGUAGE plpgsql; + +DO $$ +DECLARE t text; +BEGIN + FOREACH t IN ARRAY ARRAY['clients','leads','projects','bookings','invoices'] LOOP + EXECUTE format( + 'CREATE TRIGGER %I_touch BEFORE UPDATE ON %I FOR EACH ROW EXECUTE FUNCTION touch_updated_at()', + t, t); + END LOOP; +END $$; + +CREATE INDEX IF NOT EXISTS leads_received_idx ON leads (received_at DESC); +CREATE INDEX IF NOT EXISTS clients_status_idx ON clients (status); +CREATE INDEX IF NOT EXISTS activity_ts_idx ON activity_log (ts DESC); diff --git a/backoffice/docker-compose.yml b/backoffice/docker-compose.yml new file mode 100644 index 0000000..e6fb925 --- /dev/null +++ b/backoffice/docker-compose.yml @@ -0,0 +1,42 @@ +services: + smb-db: + image: postgres:16-alpine + container_name: smb-db + restart: unless-stopped + environment: + POSTGRES_DB: smbcrm + POSTGRES_USER: smbcrm + POSTGRES_PASSWORD: ${DB_PASSWORD} + volumes: + - smb-db-data:/var/lib/postgresql/data + - ./db/init.sql:/docker-entrypoint-initdb.d/init.sql:ro + networks: [smb-net] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U smbcrm -d smbcrm"] + interval: 10s + timeout: 5s + retries: 5 + + smb-crm: + build: ./app + container_name: smb-crm + restart: unless-stopped + environment: + DATABASE_URL: postgresql://smbcrm:${DB_PASSWORD}@smb-db:5432/smbcrm + CRM_API_TOKEN: ${CRM_API_TOKEN} + SHEET_ID: ${SHEET_ID} + GOOGLE_SA_JSON: /run/secrets/gcp-sa.json + volumes: + - ./secrets/gcp-sa.json:/run/secrets/gcp-sa.json:ro + depends_on: + smb-db: + condition: service_healthy + networks: [smb-net, proxy] + +volumes: + smb-db-data: + +networks: + smb-net: + proxy: + external: true