Fix Phase 1 server validation defects

ansible/roles/cezen-backend/files/cezen-api.service

@@ -9,6 +9,7 @@ User=cezen
 WorkingDirectory=/opt/cezen/backend
 Environment="CEZEN_DATA=/opt/cezen/data"
 Environment="OLLAMA_URL=http://localhost:11434"
+Environment="CHROMA_URL=http://localhost:8100"
 Environment="PATH=/opt/cezen/backend/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin"
 ExecStart=/opt/cezen/backend/venv/bin/uvicorn main:app --host 0.0.0.0 --port 8080 --workers 2
 Restart=always

ansible/roles/cezen-backend/files/main.py

@@ -2412,6 +2412,11 @@ def _ollama_embed(texts: list[str], model: str = OLLAMA_EMBED) -> list[list[floa
                 log_context=f"_ollama_embed(model={model})", log_exc=e)
     return embeddings
 
+def _chroma_collection_id(name: str) -> str:
+    """Resolve a stable Chroma collection name to the UUID required by data routes."""
+    collection = _chroma_req("GET", f"/api/v1/collections/{name}")
+    return collection["id"]
+
 # Collections
 @app.get("/api/rag/collections")
 async def list_collections(admin: dict = Depends(admin_only)):
@@ -2554,7 +2559,8 @@ async def delete_document(cid: int, doc_id: int, admin: dict = Depends(admin_onl
     # Remove chunks from ChromaDB
     if col:
         try:
-            _chroma_req("POST", f"/api/v1/collections/{col['chroma_name']}/delete",
+            chroma_id = _chroma_collection_id(col["chroma_name"])
+            _chroma_req("POST", f"/api/v1/collections/{chroma_id}/delete",
                         body={"where": {"doc_id": {"$eq": doc_id}}})
         except Exception:
             pass
@@ -2598,7 +2604,8 @@ async def rag_query(body: dict, user: dict = Depends(current_user)):
         raise HTTPException(status_code=503, detail="Failed to embed query")
 
     # Query ChromaDB
-    result = _chroma_req("POST", f"/api/v1/collections/{col['chroma_name']}/query", body={
+    chroma_id = _chroma_collection_id(col["chroma_name"])
+    result = _chroma_req("POST", f"/api/v1/collections/{chroma_id}/query", body={
         "query_embeddings": embed,
         "n_results": n_results,
         "include": ["documents", "metadatas", "distances"]

ansible/roles/cezen-backend/files/rag_ingest.py

@@ -162,9 +162,9 @@ def chunk_text(text: str, chunk_size: int = CHUNK_SIZE, overlap: int = CHUNK_OVE
                     chunk = chunk[:idx + len(sep)]
                     break
         chunks.append(chunk.strip())
-        start += len(chunk) - overlap
-        if start >= len(text):
+        if end >= len(text):
             break
+        start += max(1, len(chunk) - overlap)
     return [c for c in chunks if c]
 
 # ── Embedding ─────────────────────────────────────────────────────────────────
@@ -197,8 +197,10 @@ def chroma_upsert(ids, embeddings, documents, metadatas):
         "documents":  documents,
         "metadatas":  metadatas,
     }).encode()
+    with urllib.request.urlopen(f"{args.chroma_url}/api/v1/collections/{args.collection}", timeout=30) as response:
+        collection_id = json.loads(response.read().decode())["id"]
     req = urllib.request.Request(
-        f"{args.chroma_url}/api/v1/collections/{args.collection}/upsert",
+        f"{args.chroma_url}/api/v1/collections/{collection_id}/upsert",
         data=body, method="POST"
     )
     req.add_header("Content-Type", "application/json")

ansible/roles/cezen-ttyd/files/cezen-ttyd.service

@@ -7,7 +7,6 @@ After=network.target
 ExecStart=/usr/bin/ttyd \
     --port 7681 \
     --interface 127.0.0.1 \
-    --writable \
     login -f cezen-console
 Restart=always
 RestartSec=5

ansible/roles/cezen-ttyd/tasks/main.yml

@@ -7,6 +7,13 @@
     state: present
     update_cache: yes
 
+- name: Disable distribution ttyd service to reserve the portal console port
+  systemd:
+    name: ttyd
+    enabled: no
+    state: stopped
+  failed_when: false
+
 - name: Create cezen-console restricted user
   user:
     name: cezen-console

ansible/roles/chromadb/tasks/main.yml

@@ -1,9 +1,14 @@
 ---
 # ChromaDB — vector database for RAG pipelines
-- name: Install ChromaDB in cezen conda env
-  become_user: cezen
-  shell: |
-    /opt/cezen/miniconda/bin/conda run -n cezen pip install chromadb
+- name: Create isolated ChromaDB virtual environment
+  command: /usr/bin/python3 -m venv /opt/cezen/chromadb-venv
+  args:
+    creates: /opt/cezen/chromadb-venv/bin/python
+
+- name: Install pinned ChromaDB in its isolated environment
+  pip:
+    name: chromadb==0.5.23
+    virtualenv: /opt/cezen/chromadb-venv
   retries: 3
   delay: 10
 
@@ -27,13 +32,13 @@
       User=cezen
       Group=cezen
       WorkingDirectory=/opt/cezen/data/chromadb
-      ExecStart=/opt/cezen/miniconda/envs/cezen/bin/chroma run \
+      ExecStart=/opt/cezen/chromadb-venv/bin/chroma run \
         --host 0.0.0.0 \
         --port 8100 \
         --path /opt/cezen/data/chromadb
       Restart=always
       RestartSec=5
-      Environment="PATH=/opt/cezen/miniconda/envs/cezen/bin:/usr/local/bin:/usr/bin:/bin"
+      Environment="PATH=/opt/cezen/chromadb-venv/bin:/usr/local/bin:/usr/bin:/bin"
 
       [Install]
       WantedBy=multi-user.target

ansible/roles/vllm/tasks/main.yml

@@ -22,7 +22,7 @@
   when: gpu_check.stdout == "" or gpu_check.rc != 0
 
 - name: Pull vLLM Docker image
-  shell: docker pull vllm/vllm-openai:latest
+  shell: docker pull vllm/vllm-openai:v0.10.2
   retries: 3
   delay: 15
   when: gpu_check.stdout != "" and gpu_check.rc == 0
@@ -53,12 +53,11 @@
         -p 8000:8000 \
         -v /opt/cezen/models:/root/.cache/huggingface \
         -e HF_HOME=/root/.cache/huggingface \
-        vllm/vllm-openai:latest \
+        vllm/vllm-openai:v0.10.2 \
         --model {{ vllm_model }} \
         --gpu-memory-utilization {{ vllm_gpu_memory_util }} \
         --max-model-len {{ vllm_max_model_len }} \
-        --tensor-parallel-size {{ vllm_tensor_parallel }} \
-        {{ vllm_quant_flag }}
+        --tensor-parallel-size {{ vllm_tensor_parallel }}{{ ' ' + vllm_quant_flag if vllm_quant_flag else '' }}
       ExecStop=/usr/bin/docker stop vllm
       TimeoutStartSec=300
 
@@ -92,5 +91,6 @@
   systemd:
     name: vllm
     enabled: true
+    state: started
     daemon_reload: true
   when: gpu_check.stdout != "" and gpu_check.rc == 0

ansible/starter.yml

@@ -32,7 +32,7 @@
     vllm_tensor_parallel: 1
     vllm_gpu_memory_util: "0.85"
     vllm_max_model_len: 4096
-    vllm_quantization: "awq"
+    vllm_quantization: ""          # Phi-3 source model is not AWQ-quantized
 
     # ── Ollama — lightweight models ───────────────
     ollama_default_model: "phi3:mini"

specs/001-enterprise-experience/evidence/server-phase1.md

@@ -0,0 +1,75 @@
+# Server Phase 1 Validation Evidence
+
+**Validation date:** 2026-07-12 to 2026-07-13
+
+**Target:** Vast.ai full Ubuntu 22.04 VM, RTX 3090 24 GB, 23 vCPU, 52.8 GB usable RAM, 155.9 GB root filesystem
+
+**Package revision at initial install:** `55a2a41` plus the uncommitted corrections listed below
+
+**Install mode:** `install.sh --software-only --tier=starter`
+
+## Executive result
+
+The software package installs and the tested Phase 1 Server workflows operate on the representative
+cloud VM after correcting defects found during validation. This is evidence for a controlled Server
+pilot only. It is not ISO boot evidence, physical hardware evidence, Workstation evidence, a customer
+pilot, WCAG manual approval, or general-release approval.
+
+## Results
+
+| Area | Result | Evidence |
+|---|---|---|
+| Starter installation | PASS after Docker repository fix | Ansible recap: 86 ok, 51 changed, 0 failed, 2 ignored |
+| Source contract suite | PASS | 19 tests passed |
+| Shell/Python/static validation | PASS | `bash -n`, `py_compile`, `git diff --check` |
+| Tier playbook syntax | PASS | starter, entry, pro, max syntax checks |
+| Portal/backend | PASS | HTTP 200 after install and reboot |
+| Authenticated read endpoints | PASS | 51 HTTP 200 responses |
+| Entitlement restrictions | PASS | model router and two fine-tuning routes correctly returned 403 |
+| Authentication and RBAC | PASS | admin login; viewer permitted metrics and denied admin user list |
+| User/session lifecycle | PASS | create, login, revoke by deletion, expired-session rejection, self-delete protection |
+| API key lifecycle | PASS | create, verify, revoke, revoked-key rejection |
+| Guardrails | PASS | temporary keyword rule blocked matching input and was removed |
+| Prompt library | PASS | create, update and delete |
+| Backup/restore | PASS | ZIP created; listed; restored; pre-restore safety snapshot created |
+| Ollama inference | PASS | Qwen 2.5 1.5B GPU inference; model persisted across reboot |
+| Open WebUI | PASS after network correction | healthy container and model visible from WebUI |
+| RAG | PASS after corrections | collection, TXT upload, embedding, ready status, semantic retrieval, delete/cleanup |
+| vLLM | PASS after corrections | Phi-3 readiness and OpenAI-compatible chat completion |
+| Monitoring | PASS | Grafana health 200, Prometheus ready 200, DCGM and node exporter running |
+| Web console | PASS after correction | service active and `/console/` HTTP 200 |
+| Reboot persistence | PASS | all six services and five containers auto-started; models and backup persisted |
+| Concurrent inference | PASS | vLLM used 21.3 GB and Ollama 1.8 GB on RTX 3090; both completed inference |
+
+Post-reboot vLLM cold readiness occurred at approximately 144 seconds after boot.
+
+## Defects discovered and corrected in source
+
+1. Ubuntu ttyd does not support `--writable`, and the distribution ttyd service can occupy port 7681.
+2. Open WebUI could not reach an Ollama process that had not reloaded the appliance service environment.
+3. Backend targeted ChromaDB port 8000 while the service intentionally runs on 8100.
+4. Unpinned latest ChromaDB removed the v1 API used by the backend.
+5. Compatible ChromaDB dependencies conflicted with the shared AI environment; ChromaDB is now isolated.
+6. Short RAG documents caused an infinite chunking loop.
+7. Chroma query/upsert/delete data routes require collection UUIDs rather than collection names.
+8. Starter Phi-3 was incorrectly configured as AWQ.
+9. Blank optional vLLM arguments caused systemd command continuation into `ExecStop`.
+10. `vllm:latest` required an incompatible CUDA/driver combination; the image is now pinned to v0.10.2.
+11. vLLM was enabled but not explicitly started during installation.
+
+## Expected warnings / environment gaps
+
+- No final signed customer license was installed; field-staging mode was used.
+- TLS was not configured; access used SSH tunnels.
+- The VM began without a backup or models; both were created during testing.
+- Direct public port 80 was blocked by the provider; this is not an appliance Nginx failure.
+- Clean ISO boot, disk partitioning, BIOS/UEFI, USB media, physical NICs and offline ISO installation
+  cannot be validated on this rented VM.
+- Workstation validation (T036), manual WCAG 2.2 AA evidence, interruption/restricted-network trials,
+  controlled customer pilot and release approvals remain pending.
+
+## Release decision
+
+**Not ready for customer shipment yet.** The corrected package is suitable for another clean Server
+installation validation and controlled pilot preparation. A new ISO must be built from the corrected
+revision and both Server and Workstation ISO paths must be tested before release consideration.

specs/001-enterprise-experience/tasks.md

@@ -87,7 +87,7 @@ verify the same durable operation, accurate readiness, and safe next action rema
 - [x] T034 [US1] Add local setup, interruption, readiness, and escalation guidance in `cezen-portal/quickstart.html` and `cezen-portal/troubleshooting.html`
 - [x] T035 [US1] Add schema migration, operation recovery, and portal deployment steps to `ansible/roles/cezen-backend/tasks/main.yml` and `ansible/roles/cezen-nginx/tasks/main.yml`
 - [ ] T036 [US1] Verify clean install, interruption, restricted-network guidance, and WCAG evidence on a representative Workstation and record it in `specs/001-enterprise-experience/evidence/workstation-phase1.md`
-- [ ] T037 [US1] Verify clean install, interruption, restricted-network guidance, and WCAG evidence on a representative Server and record it in `specs/001-enterprise-experience/evidence/server-phase1.md`
+- [ ] T037 [US1] Verify clean install, interruption, restricted-network guidance, and WCAG evidence on a representative Server and record it in `specs/001-enterprise-experience/evidence/server-phase1.md` (software-package clean install and reboot evidence recorded; interruption, restricted-network, ISO boot, and manual WCAG evidence remain)
 - [x] T038 [US1] Verify source-to-package and source-to-ISO provenance for all Phase 1 surfaces and record checksums in `specs/001-enterprise-experience/evidence/phase1-manifest.md`
 
 **Checkpoint**: User Story 1 and product Phase 1 pass internal evidence review on Workstation and