diff --git a/.gitignore b/.gitignore index 16d47a0..15d85d7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ node_modules/ dist/ .astro/ *.log +__pycache__/ +*.pyc # secrets — never commit credentials .env diff --git a/Documentation.md b/Documentation.md index c372b1e..aec8bbc 100644 --- a/Documentation.md +++ b/Documentation.md @@ -21,8 +21,9 @@ every client regardless of tier. ``` ┌────────────────────────────────────────────┐ │ Postgres (smb-db) │ - │ clients · leads · projects · bookings · │ - │ invoices · activity_log — SOURCE OF TRUTH │ + │ clients · leads · projects · bookings · │ + │ invoices · activity_log · credentials — │ + │ SOURCE OF TRUTH (credentials never mirror) │ └───────────────▲───────────────┬────────────┘ │ SQL │ one-way mirror JSON API │ ▼ @@ -70,6 +71,7 @@ Six entities, defined once in `backoffice/app/db.py::TABLES` and mirrored 1:1 in | `bookings` | `booking_id` | Every appointment, synced from Easy!Appointments via `booking-sync`. | | `invoices` | `invoice_id` | Billing records (manual today — no automated invoicing workflow yet). | | `activity_log` | `id` (serial) | Append-only audit trail; every mutation (API or workflow) writes one row. | +| `credentials` | `cred_id` (`CR-`) | Per-client login credentials (e.g. the auto-generated Easy!Appointments provider login) — `client_id`, `label`, `username`, `secret`, `notes`. **The one entity excluded from the Sheets mirror** (`mirror: False` in `TABLES`) so secrets never leave Postgres; reading it also requires `X-CRM-Token` (every other entity's list is read-only-open behind Caddy basic-auth alone). Surfaced in the dashboard's **Credentials** tab with a masked value, a reveal toggle, and a copy-to-clipboard button. | `db.coerce_row()` is the single place that types/normalizes incoming values (dates, timestamps, numbers, booleans) so the CRUD API, the n8n ingest path and the one-time Sheets importer can never @@ -91,8 +93,10 @@ Flask app (`app.py`) + `waitress`, backed by Postgres (`smb-db`, `postgres:16-al send custom headers, so this is gated by a separate `?token=$ICS_TOKEN` query param instead of the header token). Supports `?client_id=` to scope to one client. - `GET /healthz` — DB connectivity check. -- `GET /` — the dashboard (`static/index.html`; currently **Leads** and **Clients** tabs only — - read + add (+Neu) + edit (✎) + delete (🗑), all token-gated, all audit-logged). +- `GET /` — the dashboard (`static/index.html`; **Leads**, **Clients** and **Credentials** tabs — + read + add (+Neu) + edit (✎) + delete (🗑), all token-gated, all audit-logged. The Credentials + tab masks the `secret` column by default with a per-row 👁 reveal toggle and a 📋 copy-to- + clipboard button). Every write is audit-logged to `activity_log` and enqueues an async, best-effort mirror of that entity (and of `activity_log` itself) into the linked Google Sheet — mirror failures never fail @@ -154,7 +158,7 @@ keeps the 14 most recent dumps in `/home/mivanchenko/backups/smb-crm/`. | Workflow | Trigger | Does | |---|---|---| | `lead-intake.json` | webhook | Normalize a lead payload → `POST /api/leads` → Telegram notify. Used by every demo/client lead form and the callback widget. | -| `onboarding.json` | webhook (`onboard.mivanchenko.de` form) | Compute client+project rows → `POST /api/clients` → `POST /api/projects` → Telegram notify → **provision Easy!Appointments** (create service, create provider with a generated login, build the booking embed URL) → `PATCH` the client's `stack_notes` with that embed URL + EA login. Fully automates "sign a client" end to end. | +| `onboarding.json` | webhook (`onboard.mivanchenko.de` form) | Compute client+project rows → `POST /api/clients` → `POST /api/projects` → Telegram notify → **provision Easy!Appointments** (create service, create provider with a generated login, build the booking embed URL) → `PATCH` the client's `stack_notes` with that embed URL + EA username → `POST /api/credentials` with the EA username **and password**. Fully automates "sign a client" end to end, including capturing the generated password so it isn't lost (it used to be discarded after the EA API call — see `TODO.md`). | | `booking-sync.json` | webhook (EA) | Normalize a booking event → `POST /api/bookings` → Telegram notify. | | `renewal-reminder.json` | daily 08:00 schedule | Read `Clients` from Sheets → find renewals due soon → Telegram notify → append a row to `Activity Log`. **Note:** still reads from the Sheets mirror rather than the DB directly — safe today because the mirror is kept current, but a re-point to the DB would remove that indirection. | @@ -207,8 +211,13 @@ see `deploy/clients/README.md` and each group's own compose file for the exact r feed URL leaks. - Sheet cell values are defended against formula injection (`_cell()` in `app.py` prefixes values starting with `=+-@` with a `'`). -- Credentials for client-owned accounts are never stored directly — `clients.vault_ref` stores a - pointer into Vaultwarden only. +- Credentials for client-owned accounts are recorded two ways: `clients.vault_ref` points to a + Vaultwarden item for anything the operator manually stashes there; the `credentials` table + holds secrets the *system itself* generates (currently: the Easy!Appointments provider login + created during onboarding), gated by `X-CRM-Token` even to read and deliberately excluded from + the Sheets mirror. Stored as plaintext in Postgres today — same trust boundary as the rest of + the CRM (Caddy basic-auth + host security); revisit with column-level encryption (pgcrypto) if + the dashboard is ever exposed more broadly (see `TODO.md`). - The DB→Sheets mirror is clear-then-write, not atomic — a reader can theoretically catch a cleared tab mid-sync. Accepted as low-risk (human overview, not a system of record) — see `TODO.md`. diff --git a/TODO.md b/TODO.md index 025d9a1..001ab47 100644 --- a/TODO.md +++ b/TODO.md @@ -13,12 +13,15 @@ Implications to work through: "Online-Terminbuchung"/"Sichere Buchung", Google-Kalender/Cal.com/self-hosted/Tier labels removed from rendered copy *and* HTML comments, demo booking dates now render on the current week via JS so nothing looks stale. -- [ ] After a client signs, **I decide hosting**: self-host the page on the homelab, or stand it - up on the client's free Google account. The CRM already has `tier` + `stack_notes` columns - to record that decision per client. -- [ ] Revisit the `Clients.tier` field semantics: keep A/B as an internal effort/price band, or +- [x] After a client signs, **I decide hosting**: self-host the page on the homelab, or stand it + up on the client's free Google account. Already how it works in practice — the CRM has + `tier` + `stack_notes` columns recording that per-client decision, and every deploy this + session (happynails' credential, the client stacks in `deploy/clients/`) went through that + path rather than a fixed per-tier template. Nothing left to build here. +- [ ] **Needs your call:** Revisit the `Clients.tier` field semantics — keep A/B as an internal effort/price band, or replace with something hosting-agnostic (e.g. `hosting = google | selfhosted`, `plan = …`). -- [ ] Onboarding form + workflow currently ask for Tier A/B — realign once the model is settled. +- [ ] **Blocked on the above:** Onboarding form + workflow currently ask for Tier A/B — realign + once the field semantics are settled. ## Onboard dashboard → manage Leads & Clients (CRUD) — DONE 2026-06-25 Built as a DB-first back-office (architecture pivot: **Postgres is the source of truth**, Sheets @@ -41,18 +44,54 @@ Follow-ups (not yet done): - [ ] Make the DB→Sheets overwrite atomic (currently clear-then-write; brief mid-write window a reader could catch the cleared sheet — harmless for a human overview). +## Credentials in the CRM — copy-to-clipboard — DONE 2026-07-15 +**Goal:** be able to grab a client's login (e.g. their Easy!Appointments provider password) from +the CRM dashboard itself instead of digging through Vaultwarden or a live n8n execution log — +prompted by not being able to find the happynails booking password anywhere. New `credentials` +entity (`backoffice/db/init.sql`, `backoffice/app/db.py`), **deliberately never mirrored to +Sheets** (`mirror: False`, enforced in `mirror_async`/`mirror_entity`/`/api/sync` and skipped by +`import_from_sheets.py`) and gated by `X-CRM-Token` even to read. Dashboard has a new +**Credentials** tab with a masked value, "👁 zeigen" reveal toggle, "📋 Kopieren" copy button. +`n8n/onboarding.json` now saves the auto-generated EA provider password instead of discarding it +(root cause of the happynails password being unrecoverable — it was never persisted anywhere). +Migrated + shipped to the live homelab (`smb-db` schema, `smb-crm` rebuilt, confirmed healthy). +Happynails' (`C-0002`) reset EA password is recorded as `CR-1784116056728`. + +Follow-ups (not yet done — all need you, not more code): +- [ ] Re-import the updated `n8n/onboarding.json` into the live n8n instance (the new "Save + credential" node). Needs the real `__CRM_TOKEN__`/`__EA_AUTH__` credential bindings re-attached + in the n8n UI — not safely scriptable from outside n8n. +- [ ] The happynails credential's `username` is still blank — confirm it in the EA admin UI and + fill it in via the dashboard's ✎ edit on the Credentials tab. +- [ ] Your call, no rush: encrypt `credentials.secret` at rest (e.g. pgcrypto) instead of + plaintext if the dashboard is ever exposed more broadly than Caddy basic-auth + host security. + +## Deploy pipeline — trigger from the Gitea UI +**Goal:** stop hand-running scp/ssh/docker-compose-build for every change (that's how the +credentials feature above shipped). `git.mivanchenko.de` (Gitea, already the `origin` remote for +every repo here, running alongside the other homelab containers) supports **Gitea Actions** — +workflows in `.gitea/workflows/` that run on a self-hosted runner and can be started with a +manual "Run workflow" button in the repo UI, not just on push. +- [ ] Register a Gitea Actions runner (can run directly on the homelab host, or in its own + container with the Docker socket mounted so it can `docker compose build`/`up -d`). +- [ ] `.gitea/workflows/deploy.yml`: `workflow_dispatch` trigger (the UI button), optionally also + on push to `main`. Steps: checkout → sync changed paths to `/home/mivanchenko/smb-crm/` (and + whichever other `deploy/*` groups changed) → apply any pending SQL migrations → `docker compose + build` + `up -d --no-deps ` for just the affected Compose group(s) → hit `/healthz` + (or the equivalent) to confirm before declaring success. +- [ ] Needs real secrets in Gitea's repo/runner secrets store (SSH deploy key or a runner already + on the host, `CRM_API_TOKEN`, etc.) — **your call** on which, since it's a homelab access + decision. +- [ ] Once trustworthy, this replaces the manual migration step described in the credentials + section above and the `new-client.sh` → scp → ssh flow in `deploy/clients/README.md`. + ## Other - [ ] Harden n8n auth: compose still has deprecated `N8N_BASIC_AUTH_*` with password `password` - (legacy/ignored by modern n8n owner-login). Confirm the owner account is strong; remove the - dead env. + (legacy/ignored by modern n8n owner-login). **Needs your call** — I can remove the dead env, but + whether the owner account itself is strong enough is a judgment only you can make. - [ ] Orders: the pizzeria demo posts orders into the `Leads` tab via the lead-intake webhook as a stop-gap. Build a proper **Orders** flow + sheet tab (items, total, mode, status) when the order-management product is real. -- [x] Booking-sync workflow — DONE, but not as originally scoped: the plan to split - Tier-B-Cal.com-webhook / Tier-A-GCal-poll was superseded by the tier-decoupling above. Instead, - one shared **Easy!Appointments** instance handles booking for every client (`deploy/booking/`), - wired via `n8n/booking-sync.json` (webhook → `POST /api/bookings` → Telegram), and - `n8n/onboarding.json` auto-provisions each new client's EA service+provider. ## Done - [x] 2026-06-25 — Latency fix (n8n executions 1–4 min → ~6 s; DNS + PostHog telemetry). @@ -67,3 +106,7 @@ Follow-ups (not yet done): to the CRM. - [x] 2026-07-14 — Per-client deploy tooling: `deploy/clients/` isolated-stack model + `new-client.sh` scaffolder; back-office updates. +- [x] 2026-07-14 — Booking-sync workflow, done differently than originally scoped: the planned + Tier-B-Cal.com-webhook / Tier-A-GCal-poll split was superseded by the tier-decoupling above — + one shared Easy!Appointments instance handles booking for every client instead. +- [x] 2026-07-15 — Credentials-in-the-CRM feature built and shipped live (see above). diff --git a/backoffice/app/app.py b/backoffice/app/app.py index 637446a..a38afe4 100644 --- a/backoffice/app/app.py +++ b/backoffice/app/app.py @@ -53,9 +53,9 @@ def _cell(v): def mirror_entity(entity): - if SH is None: - return 0 spec = db.TABLES[entity] + if SH is None or spec.get("mirror") is False: + return 0 cols = spec["cols"] order = LIST_ORDER.get(entity, spec["pk"]) with db.connect() as conn, conn.cursor() as cur: @@ -87,7 +87,7 @@ threading.Thread(target=_mirror_worker, daemon=True).start() def mirror_async(entity): - if SH is not None: + if SH is not None and db.TABLES[entity].get("mirror") is not False: _mirror_q.put(entity) # Static dashboard, with the CRM token injected so the (basic-auth-gated) @@ -113,6 +113,7 @@ LIST_ORDER = { "bookings": "start_time DESC NULLS LAST", "invoices": "issued_date DESC NULLS LAST", "activity_log": "ts DESC NULLS LAST", + "credentials": "client_id", } @@ -143,6 +144,11 @@ def healthz(): def list_entity(entity): if entity not in db.TABLES: return jsonify({"error": "unknown entity"}), 404 + # Credentials carry secrets, not just business metadata — require the + # mutation-level token even to read, on top of the Caddy basic-auth every + # other entity's (read-only) list relies on alone. + if entity == "credentials" and not authed(): + return jsonify({"error": "forbidden"}), 403 order = LIST_ORDER.get(entity, db.TABLES[entity]["pk"]) with db.connect() as conn, conn.cursor() as cur: cur.execute(f"SELECT * FROM {entity} ORDER BY {order}") @@ -198,6 +204,9 @@ def add_entity(entity): if not row.get("renewal_date") and row.get("start_date"): row["renewal_date"] = compute_renewal( row["start_date"], row.get("billing_cycle")) + elif entity == "credentials": + row["cred_id"] = row.get("cred_id") or "CR-" + str(int(time.time() * 1000)) + row["created_at"] = row.get("created_at") or datetime.now(timezone.utc) elif not row.get(pk): return jsonify({"error": f"{pk} required"}), 400 cols = spec["cols"] diff --git a/backoffice/app/db.py b/backoffice/app/db.py index 554933a..5dd345f 100644 --- a/backoffice/app/db.py +++ b/backoffice/app/db.py @@ -77,6 +77,16 @@ TABLES = { "numbers": [], "bools": [], }, + "credentials": { + # No "tab": never mirrored to Sheets (see "mirror" below) — secrets stay in Postgres only. + "pk": "cred_id", + "cols": ["cred_id", "client_id", "label", "username", "secret", "notes", "created_at"], + "dates": [], + "timestamps": ["created_at"], + "numbers": [], + "bools": [], + "mirror": False, + }, } diff --git a/backoffice/app/import_from_sheets.py b/backoffice/app/import_from_sheets.py index 79c2375..52f581a 100644 --- a/backoffice/app/import_from_sheets.py +++ b/backoffice/app/import_from_sheets.py @@ -44,6 +44,8 @@ def main(): with conn.cursor() as cur: cur.execute("TRUNCATE activity_log RESTART IDENTITY") for entity, spec in db.TABLES.items(): + if spec.get("mirror") is False: + continue # e.g. credentials — never lived in the Sheet, nothing to import records = sh.read_records(spec["tab"]) n = upsert(cur, entity, records) print(f" {entity:<13} <- {spec['tab']:<13} {n} row(s)") diff --git a/backoffice/app/static/index.html b/backoffice/app/static/index.html index 52938cb..56b4ac1 100644 --- a/backoffice/app/static/index.html +++ b/backoffice/app/static/index.html @@ -81,6 +81,7 @@
+
@@ -112,20 +113,25 @@ const COLS = { leads: ['lead_id','received_at','client_id','source','name','contact','service_interest','message','status','notified'], clients: ['client_id','business_name','owner_name','email','phone','niche','tier','status','billing_cycle','monthly_fee_eur','renewal_date','created_at'], + credentials: ['cred_id','client_id','label','username','secret','created_at'], }; - const PK = { leads: 'lead_id', clients: 'client_id' }; + const PK = { leads: 'lead_id', clients: 'client_id', credentials: 'cred_id' }; let current = 'leads'; let cache = {}; + let revealed = new Set(); // cred_ids currently shown in plaintext (resets on tab switch/reload) const esc = s => (s==null?'':String(s)).replace(/[&<>"]/g, c => ({'&':'&','<':'<','>':'>','"':'"'}[c])); function showErr(t){ const m=document.getElementById('msg'); m.textContent=t; m.className='msg err'; } - function clearErr(){ document.getElementById('msg').className='msg'; } + function showOk(t){ const m=document.getElementById('msg'); m.textContent=t; m.className='msg'; m.style.cssText='display:block;background:#e3f6ec;color:#0c6b3c;border:1px solid #b8e6cc;padding:10px 14px;border-radius:9px;font-size:.85rem;margin-bottom:12px;'; setTimeout(clearErr, 1800); } + function clearErr(){ const m=document.getElementById('msg'); m.className='msg'; m.style.cssText=''; } async function load(entity){ clearErr(); + revealed.clear(); document.getElementById('table').innerHTML = '
Lädt …
'; try { - const r = await fetch(`${API}/${entity}`); + // Credentials require the token to read (see app.py); harmless to send it always. + const r = await fetch(`${API}/${entity}`, { headers: { 'X-CRM-Token': TOKEN } }); if(!r.ok) throw new Error('HTTP '+r.status); const d = await r.json(); cache[entity] = d.rows; @@ -150,6 +156,13 @@ let v = r[c]; if(c==='status' || c==='tier') return `${esc(v)}`; if((c==='received_at'||c==='created_at') && v) v = String(v).replace('T',' ').slice(0,16); + if(c==='secret') { + const shown = revealed.has(id); + const display = shown ? esc(v) : '••••••••'; + return `${display} ` + + `` + + ``; + } return `${esc(v)}`; }).join(''); return `${cells}`; @@ -157,6 +170,19 @@ document.getElementById('table').innerHTML = `${head}${body}
`; } + function toggleSecret(id){ + if(revealed.has(id)) revealed.delete(id); else revealed.add(id); + render(); + } + async function copySecret(id){ + const row = (cache.credentials||[]).find(r => r.cred_id === id); + if(!row) return; + try { + await navigator.clipboard.writeText(row.secret || ''); + showOk('✓ Passwort kopiert (' + (row.label || row.cred_id) + ')'); + } catch(e){ showErr('Kopieren fehlgeschlagen: '+e.message); } + } + async function del(id){ if(!confirm(`${current.slice(0,-1).toUpperCase()} „${id}" wirklich löschen?`)) return; clearErr(); @@ -189,6 +215,10 @@ {k:'services',wide:true},{k:'stack_notes',t:'textarea',wide:true}, {k:'vault_ref'},{k:'notes',t:'textarea',wide:true}, ], + credentials: [ + {k:'client_id'},{k:'label'},{k:'username'},{k:'secret'}, + {k:'notes',t:'textarea',wide:true}, + ], }; let editId = null; diff --git a/backoffice/db/init.sql b/backoffice/db/init.sql index eade021..6750914 100644 --- a/backoffice/db/init.sql +++ b/backoffice/db/init.sql @@ -86,6 +86,20 @@ CREATE TABLE IF NOT EXISTS invoices ( updated_at timestamptz NOT NULL DEFAULT now() ); +-- Per-client login credentials (e.g. the auto-generated Easy!Appointments +-- provider login). Deliberately NOT mirrored to Sheets — see db.py TABLES +-- ("mirror": False) — so secrets never leave Postgres. +CREATE TABLE IF NOT EXISTS credentials ( + cred_id text PRIMARY KEY, + client_id text, + label text, + username text, + secret text, + notes text, + created_at timestamptz, + updated_at timestamptz NOT NULL DEFAULT now() +); + -- keep updated_at fresh on row changes CREATE OR REPLACE FUNCTION touch_updated_at() RETURNS trigger AS $$ BEGIN NEW.updated_at = now(); RETURN NEW; END; @@ -94,7 +108,7 @@ $$ LANGUAGE plpgsql; DO $$ DECLARE t text; BEGIN - FOREACH t IN ARRAY ARRAY['clients','leads','projects','bookings','invoices'] LOOP + FOREACH t IN ARRAY ARRAY['clients','leads','projects','bookings','invoices','credentials'] LOOP EXECUTE format( 'CREATE TRIGGER %I_touch BEFORE UPDATE ON %I FOR EACH ROW EXECUTE FUNCTION touch_updated_at()', t, t); @@ -104,3 +118,4 @@ END $$; CREATE INDEX IF NOT EXISTS leads_received_idx ON leads (received_at DESC); CREATE INDEX IF NOT EXISTS clients_status_idx ON clients (status); CREATE INDEX IF NOT EXISTS activity_ts_idx ON activity_log (ts DESC); +CREATE INDEX IF NOT EXISTS credentials_client_idx ON credentials (client_id); diff --git a/n8n/onboarding.json b/n8n/onboarding.json index d5bf2ed..6974236 100644 --- a/n8n/onboarding.json +++ b/n8n/onboarding.json @@ -221,7 +221,7 @@ }, { "parameters": { - "jsCode": "const bp=$('EA build provider').first().json;\nconst provId=$json.id;\nconst embed='https://booking.mivanchenko.de/index.php/booking?service='+bp.svcId+'&provider='+provId;\nconst stack_notes='Buchung-Embed: '+embed+' | EA provider '+provId+' / svc '+bp.svcId+' / login '+bp.username;\nreturn [{json:{cid:bp.cid,embed,provId,patch:{stack_notes}}}];" + "jsCode": "const bp=$('EA build provider').first().json;\nconst provId=$json.id;\nconst embed='https://booking.mivanchenko.de/index.php/booking?service='+bp.svcId+'&provider='+provId;\nconst stack_notes='Buchung-Embed: '+embed+' | EA provider '+provId+' / svc '+bp.svcId+' / login '+bp.username;\nconst password=bp.providerBody.settings.password;\nreturn [{json:{cid:bp.cid,embed,provId,username:bp.username,password,patch:{stack_notes}}}];" }, "id": "ea-ea-booking-info", "name": "EA booking info", @@ -262,6 +262,37 @@ "maxTries": 3, "waitBetweenTries": 2000, "continueOnFail": true + }, + { + "parameters": { + "options": {}, + "method": "POST", + "url": "http://smb-crm:8080/api/credentials", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "X-CRM-Token", + "value": "__CRM_TOKEN__" + } + ] + }, + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ JSON.stringify({ client_id: $json.cid, label: 'Easy!Appointments Provider-Login', username: $json.username, secret: $json.password }) }}" + }, + "id": "ea-save-credential", + "name": "Save credential", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [ + 1780, + 680 + ], + "retryOnFail": true, + "maxTries": 3, + "waitBetweenTries": 2000, + "continueOnFail": true } ], "connections": { @@ -376,6 +407,11 @@ "node": "EA update client", "type": "main", "index": 0 + }, + { + "node": "Save credential", + "type": "main", + "index": 0 } ] ]