Add enterprise appliance experience Phase 1 foundation

autoinstall/firstboot-setup.sh

@@ -12,10 +12,50 @@ LICENSE_CHECK_SCRIPT="$AIPACKAGE_DIR/scripts/cezen-license-check.py"
 PUBLIC_KEY_PATH="${CEZEN_LICENSE_PUBLIC_KEY:-$AIPACKAGE_DIR/autoinstall/keys/cezen-license-public.pem}"
 LOG_FILE="/var/log/cezen-setup.log"
 INSTALL_LOG_FILE="/var/log/cezen-install.log"
+SETUP_OPERATION_FILE="/opt/cezen/setup-operation.json"
 export TERM="${TERM:-linux}"
 touch "$LOG_FILE" "$INSTALL_LOG_FILE"
 chmod 0644 "$LOG_FILE" "$INSTALL_LOG_FILE"
 
+write_setup_operation() {
+  local state="$1" progress="$2" label="$3" message="$4"
+  mkdir -p /opt/cezen
+  STATE="$state" PROGRESS="$progress" LABEL="$label" MESSAGE="$message" SETUP_OPERATION_FILE="$SETUP_OPERATION_FILE" python3 - <<'PY'
+import json, os, time, uuid
+path = os.environ["SETUP_OPERATION_FILE"]
+try:
+    with open(path) as fh:
+        payload = json.load(fh)
+except Exception:
+    payload = {
+        "schema": "cezen.setup_operation.v1",
+        "operation_id": str(uuid.uuid4()),
+        "correlation_id": str(uuid.uuid4()),
+        "idempotency_key": str(uuid.uuid4()),
+        "kind": "install",
+        "acknowledged_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+    }
+payload.update({
+    "state": os.environ["STATE"],
+    "progress_percent": int(os.environ["PROGRESS"]),
+    "progress_label": os.environ["LABEL"],
+    "customer_message": os.environ["MESSAGE"],
+    "updated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+    "recovery_actions": ["retry", "view_log", "escalate"] if os.environ["STATE"] == "recovery_required" else [],
+})
+tmp = path + ".tmp"
+with open(tmp, "w") as fh:
+    json.dump(payload, fh, indent=2)
+os.replace(tmp, path)
+os.chmod(path, 0o640)
+PY
+}
+
+if [ -f "$SETUP_OPERATION_FILE" ] && grep -q '"state": "recovery_required"' "$SETUP_OPERATION_FILE"; then
+  whiptail --title "Nexus One AI Setup Recovery" --yesno "A previous setup attempt needs recovery.\n\nChoose Yes to resume setup safely. Choose No to leave the existing state unchanged and inspect $INSTALL_LOG_FILE." 14 72 || exit 1
+fi
+write_setup_operation "validating" 0 "Collecting setup choices" "Setup is ready to resume safely if interrupted."
+
 detect_iface() {
   ip route show default 2>/dev/null | awk '/default/ {print $5; exit}'
 }
@@ -387,6 +427,7 @@ whiptail --title "$TITLE" \
 
 clear
 echo ""
+write_setup_operation "running" 5 "Starting appliance installation" "Installation has started."
 echo "╔══════════════════════════════════════════╗"
 echo "║   Nexus One AI — Installing...         ║"
 echo "║   Check progress: journalctl -f           ║"
@@ -449,10 +490,12 @@ whiptail --title "$TITLE" \
 if bash "$AIPACKAGE_DIR/install.sh" --tier="$TIER" >> "$INSTALL_LOG_FILE" 2>&1; then
   # Mark as configured only after the installer finishes successfully.
   touch /opt/cezen/.setup-done
+  write_setup_operation "succeeded" 100 "Installation completed" "Installation completed successfully."
   whiptail --title "$TITLE" \
     --msgbox "\nInstaller command finished successfully.\n\nPortal      -> http://localhost\nOllama API  -> http://localhost:11434\nGrafana     -> http://localhost:3000\n\nAdmin login -> admin / Cezen@2024!\n(you will be required to change this on first login)\n\nFor detailed logs, run:\n  sudo tail -f $INSTALL_LOG_FILE" \
     $H $W
 else
+  write_setup_operation "recovery_required" 0 "Installation needs attention" "Installation did not complete. Review the local log, then retry or contact support."
   whiptail --title "$TITLE" \
     --msgbox "\nInstaller command failed.\n\nThe setup wizard will run again on next boot.\n\nCheck the log with:\n  sudo tail -n 120 $INSTALL_LOG_FILE" \
     $H $W

autoinstall/websetup/server.py

@@ -24,8 +24,50 @@ FEASIBILITY_JSON = "/opt/cezen/feasibility.json"
 LICENSE_JSON = "/opt/cezen/license.json"
 OVERRIDE_JSON = "/opt/cezen/license.override.json"
 INSTALL_RECORD_JSON = "/opt/cezen/install-record.json"
+INSTALL_OPERATION_JSON = os.environ.get("CEZEN_SETUP_OPERATION_JSON", "/opt/cezen/setup-operation.json")
 PUBLIC_KEY_PATH = os.environ.get("CEZEN_LICENSE_PUBLIC_KEY", f"{AIPACKAGE_DIR}/autoinstall/keys/cezen-license-public.pem")
 
+def read_install_operation():
+    return read_json_file(INSTALL_OPERATION_JSON)
+
+def write_install_operation(operation):
+    operation = dict(operation or {})
+    operation["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    write_json_file(INSTALL_OPERATION_JSON, operation)
+    try:
+        os.chmod(INSTALL_OPERATION_JSON, 0o640)
+    except OSError:
+        pass
+    return operation
+
+def start_install_operation(tier, skip_tools, idempotency_key):
+    existing = read_install_operation()
+    if idempotency_key and existing.get("idempotency_key") == idempotency_key:
+        existing["duplicate_request"] = True
+        return existing, False
+    if existing.get("state") in {"validating", "running", "awaiting_reboot"}:
+        existing["duplicate_request"] = True
+        return existing, False
+    now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    operation = {
+        "schema": "cezen.setup_operation.v1",
+        "operation_id": str(__import__("uuid").uuid4()),
+        "correlation_id": str(__import__("uuid").uuid4()),
+        "idempotency_key": idempotency_key or str(__import__("uuid").uuid4()),
+        "kind": "install",
+        "state": "validating",
+        "tier": tier,
+        "skip_tools": list(skip_tools or []),
+        "progress_percent": 0,
+        "progress_label": "Validating installation",
+        "acknowledged_at": now,
+        "updated_at": now,
+        "duplicate_request": False,
+        "customer_message": "Installation request accepted.",
+        "recovery_actions": [],
+    }
+    return write_install_operation(operation), True
+
 # ─── Helpers ──────────────────────────────────────────────
 def get_ip():
     try:
@@ -173,10 +215,14 @@ def write_install_record(license_data, tier, skip_tools, preview):
     subprocess.run(["chown", "root:cezen", INSTALL_RECORD_JSON], check=False)
     os.chmod(INSTALL_RECORD_JSON, 0o640)
 
-def run_install(tier, skip_tools, license_data=None, preview=None):
+def run_install(tier, skip_tools, license_data=None, preview=None, operation_id=""):
     global install_status
     install_status = {"running": True, "done": False, "error": None}
     try:
+        operation = read_install_operation()
+        if operation.get("operation_id") == operation_id:
+            operation.update({"state": "running", "progress_percent": 5, "progress_label": "Preparing appliance"})
+            write_install_operation(operation)
         # Write config so phase 2 (post-reboot) knows what to skip
         os.makedirs("/opt/cezen", exist_ok=True)
         skip_str = ",".join(skip_tools) if skip_tools else ""
@@ -192,14 +238,25 @@ def run_install(tier, skip_tools, license_data=None, preview=None):
         # Fresh NVIDIA servers do not have nvidia-smi yet, so detect the PCI
         # device and run phase 1 to install drivers before the AI stack.
         phase = "1" if has_nvidia_gpu() else "2"
+        if operation.get("operation_id") == operation_id and phase == "1":
+            operation.update({"state": "awaiting_reboot", "progress_percent": 40, "progress_label": "Installing drivers before reboot"})
+            write_install_operation(operation)
         cmd = ["bash", f"{AIPACKAGE_DIR}/install.sh", f"--phase={phase}", f"--tier={tier}"]
         with open(INSTALL_LOG, "w") as log:
             proc = subprocess.Popen(cmd, stdout=log, stderr=log, env=env)
             proc.wait()
         # Reaches here only if no reboot happened (e.g. no GPU / drivers already installed)
         install_status = {"running": False, "done": True, "error": None}
+        operation = read_install_operation()
+        if operation.get("operation_id") == operation_id:
+            operation.update({"state": "succeeded", "progress_percent": 100, "progress_label": "Installation command completed", "customer_message": "Installation completed successfully."})
+            write_install_operation(operation)
     except Exception as e:
         install_status = {"running": False, "done": False, "error": str(e)}
+        operation = read_install_operation()
+        if operation.get("operation_id") == operation_id:
+            operation.update({"state": "recovery_required", "progress_label": "Installation needs attention", "result_code": "install_failed", "customer_message": "Installation did not complete. Review the local log and retry or contact support.", "recovery_actions": ["retry", "view_log", "escalate"]})
+            write_install_operation(operation)
 
 # ─── HTML UI ──────────────────────────────────────────────
 HTML = r"""<!DOCTYPE html>
@@ -592,6 +649,7 @@ let tools = {
 window.onload = () => {
   fetch('/api/status').then(r=>r.json()).then(d=>{
     document.getElementById('current-ip').textContent = d.ip || 'unknown';
+    if (d.operation && ['validating','running','awaiting_reboot','recovery_required'].includes(d.operation.state)) resumeOperation(d.operation);
   });
   renderTools();
   renderTierAvailability(null);
@@ -749,10 +807,11 @@ function startInstall() {
   fetch('/api/install', {
     method: 'POST',
     headers: {'Content-Type':'application/json'},
-    body: JSON.stringify({ tier: selectedTier, skip_tools: skip, license: collectLicense() })
+    body: JSON.stringify({ tier: selectedTier, skip_tools: skip, license: collectLicense(), idempotency_key: sessionStorage.getItem('cezen-install-key') || (() => { const k = (crypto.randomUUID ? crypto.randomUUID() : String(Date.now())); sessionStorage.setItem('cezen-install-key', k); return k; })() })
   }).then(async r => {
     const data = await r.json();
     if (!r.ok || !data.ok) throw new Error(data.error || 'Install start failed');
+    window.currentOperationId = data.operation_id;
   }).catch(err => {
     showBanner(err.message, 'error');
     document.getElementById('install-btn-row').classList.remove('hidden');
@@ -763,6 +822,17 @@ function startInstall() {
   streamLog();
 }
 
+function resumeOperation(operation) {
+  window.currentOperationId = operation.operation_id;
+  document.getElementById('install-btn-row').classList.add('hidden');
+  document.getElementById('summary-card').classList.add('hidden');
+  document.getElementById('progress-wrap').classList.add('show');
+  document.getElementById('progress-label').textContent = operation.progress_label || 'Resuming installation status…';
+  document.getElementById('progress-bar').style.width = (operation.progress_percent || 0) + '%';
+  if (operation.state === 'recovery_required') showBanner(operation.customer_message || 'Installation needs attention. Retry or contact support.', 'error');
+  else streamLog();
+}
+
 function streamLog() {
   const log = document.getElementById('log-box');
   const bar = document.getElementById('progress-bar');
@@ -890,6 +960,7 @@ class Handler(BaseHTTPRequestHandler):
                 "interfaces": get_interfaces(),
                 "setup_done": os.path.exists(SETUP_DONE_FILE),
                 "install_status": install_status,
+                "operation": read_install_operation(),
             })
 
         elif path == "/api/progress":
@@ -961,6 +1032,7 @@ class Handler(BaseHTTPRequestHandler):
             tier = body.get("tier", "basic")
             skip = body.get("skip_tools", [])
             license_data = body.get("license", {})
+            idempotency_key = str(body.get("idempotency_key") or "").strip()
             try:
                 preview = evaluate_install_constraints(license_data or {})
             except Exception as e:
@@ -970,12 +1042,13 @@ class Handler(BaseHTTPRequestHandler):
             if not selected or not selected.get("selectable"):
                 self.send_json({"ok": False, "error": f"Tier '{tier}' is not allowed for this license/hardware."}, 400)
                 return
-            if not install_status["running"]:
-                t = threading.Thread(target=run_install, args=(tier, skip, license_data, preview), daemon=True)
+            operation, created = start_install_operation(tier, skip, idempotency_key)
+            if created and not install_status["running"]:
+                t = threading.Thread(target=run_install, args=(tier, skip, license_data, preview, operation["operation_id"]), daemon=True)
                 t.start()
-                self.send_json({"ok": True})
+                self.send_json({"ok": True, "operation_id": operation["operation_id"], "acknowledged_at": operation["acknowledged_at"], "duplicate_request": False})
             else:
-                self.send_json({"ok": False, "error": "Install already running"})
+                self.send_json({"ok": True, "operation_id": operation.get("operation_id"), "acknowledged_at": operation.get("acknowledged_at"), "duplicate_request": True})
 
         else:
             self.send_response(404)