Add enterprise appliance experience Phase 1 foundation

ansible/roles/cezen-backend/files/main.py

@@ -67,6 +67,12 @@ JWT_ALGORITHM  = "HS256"
 JWT_EXPIRE_HRS = 8
 COOKIE_NAME    = "cezen_token"
 OLLAMA_URL     = os.environ.get("OLLAMA_URL", "http://localhost:11434")
+SCHEMA_VERSION = 1
+
+CUSTOMER_STATES = {"loading", "empty", "ready", "degraded", "failed", "restricted", "unavailable", "stale"}
+OPERATION_STATES = {"requested", "validating", "queued", "running", "awaiting_reboot", "succeeded", "failed", "cancelled", "recovery_required"}
+TERMINAL_OPERATION_STATES = {"succeeded", "failed", "cancelled", "recovery_required"}
+_AUDIT_SENSITIVE_KEYS = {"password", "token", "secret", "signature", "license_key", "license_json", "override_json", "content"}
 
 # Tier is set by Cezen at deployment time via environment variable.
 # When set, it cannot be changed from the portal — customers get read-only visibility.
@@ -223,6 +229,47 @@ def get_db():
     conn.execute("PRAGMA journal_mode=WAL")
     return conn
 
+def _column_names(db, table: str) -> set[str]:
+    return {row[1] for row in db.execute(f"PRAGMA table_info({table})").fetchall()}
+
+def run_migrations(db) -> int:
+    """Apply additive, idempotent appliance migrations in one transaction."""
+    db.execute("CREATE TABLE IF NOT EXISTS schema_metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
+    row = db.execute("SELECT value FROM schema_metadata WHERE key='schema_version'").fetchone()
+    current = int(row[0]) if row else 0
+    if current > SCHEMA_VERSION:
+        raise RuntimeError(f"Database schema {current} is newer than supported {SCHEMA_VERSION}")
+    if current < 1:
+        db.execute("""
+            CREATE TABLE IF NOT EXISTS operations (
+                id TEXT PRIMARY KEY,
+                correlation_id TEXT NOT NULL UNIQUE,
+                kind TEXT NOT NULL,
+                actor_id INTEGER,
+                actor_name TEXT NOT NULL DEFAULT '',
+                scope TEXT NOT NULL DEFAULT '',
+                state TEXT NOT NULL,
+                progress_percent INTEGER,
+                progress_label TEXT NOT NULL DEFAULT '',
+                requested_at TEXT NOT NULL,
+                started_at TEXT,
+                updated_at TEXT NOT NULL,
+                finished_at TEXT,
+                result_code TEXT NOT NULL DEFAULT '',
+                customer_message TEXT NOT NULL DEFAULT '',
+                recovery_actions TEXT NOT NULL DEFAULT '[]',
+                metadata_json TEXT NOT NULL DEFAULT '{}',
+                idempotency_key TEXT,
+                parent_id TEXT,
+                UNIQUE(kind, idempotency_key)
+            )
+        """)
+        if "correlation_id" not in _column_names(db, "audit_log"):
+            db.execute("ALTER TABLE audit_log ADD COLUMN correlation_id TEXT NOT NULL DEFAULT ''")
+        db.execute("INSERT OR REPLACE INTO schema_metadata (key,value) VALUES ('schema_version','1')")
+        current = 1
+    return current
+
 def init_db():
     db = get_db()
     db.executescript("""
@@ -691,6 +738,12 @@ def init_db():
         created_at  TEXT    NOT NULL
     );
     """)
+    try:
+        with db:
+            run_migrations(db)
+    except Exception:
+        db.close()
+        raise
 
     # Seed default admin if no users exist
     row = db.execute("SELECT COUNT(*) as n FROM users").fetchone()
@@ -868,10 +921,36 @@ async def unhandled_exception_handler(request: Request, exc: Exception):
 # the client. This logs the real error server-side and raises a structured,
 # actionable HTTPException instead — {code, message, remediation} rather than
 # a bare string, so the portal can show "what happened / why / how to fix it".
-def raise_api_error(status_code: int, code: str, message: str, remediation: str = "", *, log_context: str = "", log_exc=None):
+def customer_state(state: str, summary: str, *, severity: str = "info", observed_at: str = "", next_actions=None, code: str = "") -> dict:
+    normalized = str(state or "unavailable").lower().replace("_", "-")
+    if normalized not in CUSTOMER_STATES:
+        normalized = "unavailable"
+    if severity not in {"info", "warning", "critical"}:
+        severity = "info"
+    return {
+        "schema": "cezen.customer_state.v1",
+        "state": normalized,
+        "severity": severity,
+        "summary": str(summary or "Status unavailable"),
+        "observed_at": observed_at or utcnow(),
+        "next_actions": list(next_actions or []),
+        "code": str(code or normalized),
+    }
+
+def error_envelope(code: str, message: str, remediation: str = "", *, affected_scope: str = "", correlation_id: str = "") -> dict:
+    return {
+        "schema": "cezen.error.v1",
+        "code": str(code),
+        "message": str(message),
+        "remediation": str(remediation),
+        "affected_scope": str(affected_scope),
+        "correlation_id": str(correlation_id),
+    }
+
+def raise_api_error(status_code: int, code: str, message: str, remediation: str = "", *, log_context: str = "", log_exc=None, affected_scope: str = "", correlation_id: str = ""):
     if log_exc is not None:
         logger.error("%s: %s", log_context or code, log_exc, exc_info=True)
-    raise HTTPException(status_code=status_code, detail={"code": code, "message": message, "remediation": remediation})
+    raise HTTPException(status_code=status_code, detail=error_envelope(code, message, remediation, affected_scope=affected_scope, correlation_id=correlation_id))
 
 # ── Auth helpers ──────────────────────────────────────────────────────────────
 
@@ -924,25 +1003,85 @@ def admin_only(user: dict = Depends(current_user)) -> dict:
         raise HTTPException(status_code=403, detail="Admin access required")
     return user
 
-def audit(db, user_id, username, action, detail="", ip="", result="success"):
+def _redact_audit_value(value):
+    if isinstance(value, dict):
+        return {str(k): ("[REDACTED]" if str(k).lower() in _AUDIT_SENSITIVE_KEYS else _redact_audit_value(v)) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_redact_audit_value(v) for v in value]
+    return value
+
+def safe_audit_detail(detail) -> str:
+    if isinstance(detail, (dict, list)):
+        return json.dumps(_redact_audit_value(detail), separators=(",", ":"), sort_keys=True)
+    text = str(detail or "")
+    for key in _AUDIT_SENSITIVE_KEYS:
+        text = __import__("re").sub(rf"(?i)({key}\s*[=:]\s*)([^\s,;]+)", rf"\1[REDACTED]", text)
+    return text[:2000]
+
+def audit(db, user_id, username, action, detail="", ip="", result="success", correlation_id=""):
     db.execute(
-        "INSERT INTO audit_log (timestamp,user_id,username,action,detail,ip_address,result) VALUES (?,?,?,?,?,?,?)",
-        (utcnow(), user_id, username, action, detail, ip, result)
+        "INSERT INTO audit_log (timestamp,user_id,username,action,detail,ip_address,result,correlation_id) VALUES (?,?,?,?,?,?,?,?)",
+        (utcnow(), user_id, username, action, safe_audit_detail(detail), ip, result, correlation_id)
     )
 
-def _audit(db, username, action, detail="", ip="", result="success", user_id=None):
+def _audit(db, username, action, detail="", ip="", result="success", user_id=None, correlation_id=""):
     """Audit helper for API sections that do not already hold a DB connection."""
     own_db = db is None
     if own_db:
         db = get_db()
     try:
-        audit(db, user_id, username, action, detail, ip, result)
+        audit(db, user_id, username, action, detail, ip, result, correlation_id)
         if own_db:
             db.commit()
     finally:
         if own_db:
             db.close()
 
+def create_operation(kind: str, actor: dict, scope: str = "", *, idempotency_key: str = "", metadata=None) -> dict:
+    if not actor or actor.get("role") != "admin":
+        raise HTTPException(status_code=403, detail=error_envelope("admin_required", "Admin access required", "Sign in with an authorized administrator account."))
+    now = utcnow()
+    db = get_db()
+    try:
+        if idempotency_key:
+            existing = db.execute("SELECT * FROM operations WHERE kind=? AND idempotency_key=?", (kind, idempotency_key)).fetchone()
+            if existing:
+                return dict(existing)
+        operation_id = str(uuid.uuid4())
+        correlation_id = str(uuid.uuid4())
+        db.execute("""INSERT INTO operations
+            (id,correlation_id,kind,actor_id,actor_name,scope,state,requested_at,updated_at,metadata_json,idempotency_key)
+            VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
+            (operation_id, correlation_id, kind, actor.get("id") or actor.get("sub"), actor.get("username", ""), scope,
+             "requested", now, now, json.dumps(_redact_audit_value(metadata or {})), idempotency_key or None))
+        audit(db, actor.get("id") or actor.get("sub"), actor.get("username", ""), "operation_create",
+              {"kind": kind, "scope": scope}, result="success", correlation_id=correlation_id)
+        db.commit()
+        return dict(db.execute("SELECT * FROM operations WHERE id=?", (operation_id,)).fetchone())
+    finally:
+        db.close()
+
+def transition_operation(operation_id: str, new_state: str, *, progress_percent=None, progress_label: str = "", result_code: str = "", customer_message: str = "", recovery_actions=None) -> dict:
+    if new_state not in OPERATION_STATES:
+        raise ValueError(f"Invalid operation state: {new_state}")
+    db = get_db()
+    try:
+        row = db.execute("SELECT * FROM operations WHERE id=?", (operation_id,)).fetchone()
+        if not row:
+            raise KeyError(operation_id)
+        if row["state"] in TERMINAL_OPERATION_STATES:
+            raise ValueError("Terminal operation cannot transition; create a linked retry")
+        now = utcnow()
+        started_at = row["started_at"] or (now if new_state in {"running", "awaiting_reboot"} else None)
+        finished_at = now if new_state in TERMINAL_OPERATION_STATES else None
+        db.execute("""UPDATE operations SET state=?,progress_percent=?,progress_label=?,started_at=?,updated_at=?,finished_at=?,result_code=?,customer_message=?,recovery_actions=? WHERE id=?""",
+                   (new_state, progress_percent, progress_label, started_at, now, finished_at, result_code, customer_message,
+                    json.dumps(recovery_actions or []), operation_id))
+        db.commit()
+        return dict(db.execute("SELECT * FROM operations WHERE id=?", (operation_id,)).fetchone())
+    finally:
+        db.close()
+
 def _setting_value(key: str, default: str = "") -> str:
     db = get_db()
     try:
@@ -2036,6 +2175,7 @@ async def reconcile_license(request: Request, admin: dict = Depends(admin_only))
 
 @app.get("/api/system/readiness-report")
 async def readiness_report(admin: dict = Depends(admin_only)):
+    observed_at = utcnow()
     feasibility = await system_feasibility(admin)
     license_payload = _tier_payload()
     entitlement = _entitlement_summary(feasibility)
@@ -2050,9 +2190,27 @@ async def readiness_report(admin: dict = Depends(admin_only)):
         recommended_rank = list(TIER_MATRIX.keys()).index(_normalize_tier(recommended_tier))
         commercial_fit = "matched" if current_rank >= recommended_rank else "license_upgrade_recommended"
         commercial_fit_note = "Current license covers the recommended deployment." if commercial_fit == "matched" else "Quote a higher tier or reduce enabled features for this hardware."
+    raw_status = readiness.get("status")
+    state = "ready" if raw_status == "ready" else "degraded" if raw_status == "limited" else "failed"
+    affected = [str(name) for name in readiness.get("blockers", [])]
+    capability_reasons = {
+        "not_licensed": "Capability is not included in the current license.",
+        "not_provisioned": "Capability is licensed but not installed on this appliance.",
+        "unsupported_hardware": "Capability is not supported by the detected hardware.",
+        "degraded": "Capability is installed but currently degraded.",
+        "unavailable": "Capability is currently unavailable.",
+    }
     return {
-        "schema": "cezen.readiness_report.v1",
-        "generated_at": utcnow(),
+        "schema": "cezen.readiness_report.v2",
+        "generated_at": observed_at,
+        "observed_at": observed_at,
+        "freshness_seconds": 0,
+        "state": state,
+        "severity": "info" if state == "ready" else "warning" if state == "degraded" else "critical",
+        "customer_impact": "Appliance is ready for supported use." if state == "ready" else "Some appliance capabilities need attention.",
+        "affected_capabilities": affected,
+        "next_actions": ([{"kind": "review_health", "label": "Review Health Center", "href": "health.html"}] if affected else []),
+        "capability_reasons": capability_reasons,
         "customer_mode": "software_only" if (feasibility.get("features") or {}).get("software_only", True) else "appliance",
         "license": license_payload,
         "entitlement": entitlement,

ansible/roles/cezen-backend/tasks/main.yml

@@ -43,6 +43,21 @@
     mode: "0640"
   when: cezen_license_file.stat.exists
 
+- name: Check for existing appliance database before schema migration
+  stat:
+    path: /opt/cezen/data/cezen.db
+  register: cezen_existing_db
+
+- name: Create pre-deploy database safety copy
+  copy:
+    src: /opt/cezen/data/cezen.db
+    dest: /opt/cezen/data/cezen.db.pre-deploy
+    remote_src: yes
+    owner: "{{ cezen_user }}"
+    group: "{{ cezen_user }}"
+    mode: "0600"
+  when: cezen_existing_db.stat.exists
+
 - name: Copy FastAPI application
   copy:
     src: main.py