Add enterprise appliance experience Phase 1 foundation

autoinstall/websetup/server.py

@@ -24,8 +24,50 @@ FEASIBILITY_JSON = "/opt/cezen/feasibility.json"
 LICENSE_JSON = "/opt/cezen/license.json"
 OVERRIDE_JSON = "/opt/cezen/license.override.json"
 INSTALL_RECORD_JSON = "/opt/cezen/install-record.json"
+INSTALL_OPERATION_JSON = os.environ.get("CEZEN_SETUP_OPERATION_JSON", "/opt/cezen/setup-operation.json")
 PUBLIC_KEY_PATH = os.environ.get("CEZEN_LICENSE_PUBLIC_KEY", f"{AIPACKAGE_DIR}/autoinstall/keys/cezen-license-public.pem")
 
+def read_install_operation():
+    return read_json_file(INSTALL_OPERATION_JSON)
+
+def write_install_operation(operation):
+    operation = dict(operation or {})
+    operation["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    write_json_file(INSTALL_OPERATION_JSON, operation)
+    try:
+        os.chmod(INSTALL_OPERATION_JSON, 0o640)
+    except OSError:
+        pass
+    return operation
+
+def start_install_operation(tier, skip_tools, idempotency_key):
+    existing = read_install_operation()
+    if idempotency_key and existing.get("idempotency_key") == idempotency_key:
+        existing["duplicate_request"] = True
+        return existing, False
+    if existing.get("state") in {"validating", "running", "awaiting_reboot"}:
+        existing["duplicate_request"] = True
+        return existing, False
+    now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    operation = {
+        "schema": "cezen.setup_operation.v1",
+        "operation_id": str(__import__("uuid").uuid4()),
+        "correlation_id": str(__import__("uuid").uuid4()),
+        "idempotency_key": idempotency_key or str(__import__("uuid").uuid4()),
+        "kind": "install",
+        "state": "validating",
+        "tier": tier,
+        "skip_tools": list(skip_tools or []),
+        "progress_percent": 0,
+        "progress_label": "Validating installation",
+        "acknowledged_at": now,
+        "updated_at": now,
+        "duplicate_request": False,
+        "customer_message": "Installation request accepted.",
+        "recovery_actions": [],
+    }
+    return write_install_operation(operation), True
+
 # ─── Helpers ──────────────────────────────────────────────
 def get_ip():
     try:
@@ -173,10 +215,14 @@ def write_install_record(license_data, tier, skip_tools, preview):
     subprocess.run(["chown", "root:cezen", INSTALL_RECORD_JSON], check=False)
     os.chmod(INSTALL_RECORD_JSON, 0o640)
 
-def run_install(tier, skip_tools, license_data=None, preview=None):
+def run_install(tier, skip_tools, license_data=None, preview=None, operation_id=""):
     global install_status
     install_status = {"running": True, "done": False, "error": None}
     try:
+        operation = read_install_operation()
+        if operation.get("operation_id") == operation_id:
+            operation.update({"state": "running", "progress_percent": 5, "progress_label": "Preparing appliance"})
+            write_install_operation(operation)
         # Write config so phase 2 (post-reboot) knows what to skip
         os.makedirs("/opt/cezen", exist_ok=True)
         skip_str = ",".join(skip_tools) if skip_tools else ""
@@ -192,14 +238,25 @@ def run_install(tier, skip_tools, license_data=None, preview=None):
         # Fresh NVIDIA servers do not have nvidia-smi yet, so detect the PCI
         # device and run phase 1 to install drivers before the AI stack.
         phase = "1" if has_nvidia_gpu() else "2"
+        if operation.get("operation_id") == operation_id and phase == "1":
+            operation.update({"state": "awaiting_reboot", "progress_percent": 40, "progress_label": "Installing drivers before reboot"})
+            write_install_operation(operation)
         cmd = ["bash", f"{AIPACKAGE_DIR}/install.sh", f"--phase={phase}", f"--tier={tier}"]
         with open(INSTALL_LOG, "w") as log:
             proc = subprocess.Popen(cmd, stdout=log, stderr=log, env=env)
             proc.wait()
         # Reaches here only if no reboot happened (e.g. no GPU / drivers already installed)
         install_status = {"running": False, "done": True, "error": None}
+        operation = read_install_operation()
+        if operation.get("operation_id") == operation_id:
+            operation.update({"state": "succeeded", "progress_percent": 100, "progress_label": "Installation command completed", "customer_message": "Installation completed successfully."})
+            write_install_operation(operation)
     except Exception as e:
         install_status = {"running": False, "done": False, "error": str(e)}
+        operation = read_install_operation()
+        if operation.get("operation_id") == operation_id:
+            operation.update({"state": "recovery_required", "progress_label": "Installation needs attention", "result_code": "install_failed", "customer_message": "Installation did not complete. Review the local log and retry or contact support.", "recovery_actions": ["retry", "view_log", "escalate"]})
+            write_install_operation(operation)
 
 # ─── HTML UI ──────────────────────────────────────────────
 HTML = r"""<!DOCTYPE html>
@@ -592,6 +649,7 @@ let tools = {
 window.onload = () => {
   fetch('/api/status').then(r=>r.json()).then(d=>{
     document.getElementById('current-ip').textContent = d.ip || 'unknown';
+    if (d.operation && ['validating','running','awaiting_reboot','recovery_required'].includes(d.operation.state)) resumeOperation(d.operation);
   });
   renderTools();
   renderTierAvailability(null);
@@ -749,10 +807,11 @@ function startInstall() {
   fetch('/api/install', {
     method: 'POST',
     headers: {'Content-Type':'application/json'},
-    body: JSON.stringify({ tier: selectedTier, skip_tools: skip, license: collectLicense() })
+    body: JSON.stringify({ tier: selectedTier, skip_tools: skip, license: collectLicense(), idempotency_key: sessionStorage.getItem('cezen-install-key') || (() => { const k = (crypto.randomUUID ? crypto.randomUUID() : String(Date.now())); sessionStorage.setItem('cezen-install-key', k); return k; })() })
   }).then(async r => {
     const data = await r.json();
     if (!r.ok || !data.ok) throw new Error(data.error || 'Install start failed');
+    window.currentOperationId = data.operation_id;
   }).catch(err => {
     showBanner(err.message, 'error');
     document.getElementById('install-btn-row').classList.remove('hidden');
@@ -763,6 +822,17 @@ function startInstall() {
   streamLog();
 }
 
+function resumeOperation(operation) {
+  window.currentOperationId = operation.operation_id;
+  document.getElementById('install-btn-row').classList.add('hidden');
+  document.getElementById('summary-card').classList.add('hidden');
+  document.getElementById('progress-wrap').classList.add('show');
+  document.getElementById('progress-label').textContent = operation.progress_label || 'Resuming installation status…';
+  document.getElementById('progress-bar').style.width = (operation.progress_percent || 0) + '%';
+  if (operation.state === 'recovery_required') showBanner(operation.customer_message || 'Installation needs attention. Retry or contact support.', 'error');
+  else streamLog();
+}
+
 function streamLog() {
   const log = document.getElementById('log-box');
   const bar = document.getElementById('progress-bar');
@@ -890,6 +960,7 @@ class Handler(BaseHTTPRequestHandler):
                 "interfaces": get_interfaces(),
                 "setup_done": os.path.exists(SETUP_DONE_FILE),
                 "install_status": install_status,
+                "operation": read_install_operation(),
             })
 
         elif path == "/api/progress":
@@ -961,6 +1032,7 @@ class Handler(BaseHTTPRequestHandler):
             tier = body.get("tier", "basic")
             skip = body.get("skip_tools", [])
             license_data = body.get("license", {})
+            idempotency_key = str(body.get("idempotency_key") or "").strip()
             try:
                 preview = evaluate_install_constraints(license_data or {})
             except Exception as e:
@@ -970,12 +1042,13 @@ class Handler(BaseHTTPRequestHandler):
             if not selected or not selected.get("selectable"):
                 self.send_json({"ok": False, "error": f"Tier '{tier}' is not allowed for this license/hardware."}, 400)
                 return
-            if not install_status["running"]:
-                t = threading.Thread(target=run_install, args=(tier, skip, license_data, preview), daemon=True)
+            operation, created = start_install_operation(tier, skip, idempotency_key)
+            if created and not install_status["running"]:
+                t = threading.Thread(target=run_install, args=(tier, skip, license_data, preview, operation["operation_id"]), daemon=True)
                 t.start()
-                self.send_json({"ok": True})
+                self.send_json({"ok": True, "operation_id": operation["operation_id"], "acknowledged_at": operation["acknowledged_at"], "duplicate_request": False})
             else:
-                self.send_json({"ok": False, "error": "Install already running"})
+                self.send_json({"ok": True, "operation_id": operation.get("operation_id"), "acknowledged_at": operation.get("acknowledged_at"), "duplicate_request": True})
 
         else:
             self.send_response(404)