diff --git a/ansible/roles/cezen-backend/files/cezen-api.service b/ansible/roles/cezen-backend/files/cezen-api.service index bf25b49..63c01d4 100644 --- a/ansible/roles/cezen-backend/files/cezen-api.service +++ b/ansible/roles/cezen-backend/files/cezen-api.service @@ -9,6 +9,7 @@ User=cezen WorkingDirectory=/opt/cezen/backend Environment="CEZEN_DATA=/opt/cezen/data" Environment="OLLAMA_URL=http://localhost:11434" +Environment="CHROMA_URL=http://localhost:8100" Environment="PATH=/opt/cezen/backend/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin" ExecStart=/opt/cezen/backend/venv/bin/uvicorn main:app --host 0.0.0.0 --port 8080 --workers 2 Restart=always diff --git a/ansible/roles/cezen-backend/files/main.py b/ansible/roles/cezen-backend/files/main.py index 732b0ab..98f3fbd 100644 --- a/ansible/roles/cezen-backend/files/main.py +++ b/ansible/roles/cezen-backend/files/main.py @@ -2412,6 +2412,11 @@ def _ollama_embed(texts: list[str], model: str = OLLAMA_EMBED) -> list[list[floa log_context=f"_ollama_embed(model={model})", log_exc=e) return embeddings +def _chroma_collection_id(name: str) -> str: + """Resolve a stable Chroma collection name to the UUID required by data routes.""" + collection = _chroma_req("GET", f"/api/v1/collections/{name}") + return collection["id"] + # Collections @app.get("/api/rag/collections") async def list_collections(admin: dict = Depends(admin_only)): @@ -2554,7 +2559,8 @@ async def delete_document(cid: int, doc_id: int, admin: dict = Depends(admin_onl # Remove chunks from ChromaDB if col: try: - _chroma_req("POST", f"/api/v1/collections/{col['chroma_name']}/delete", + chroma_id = _chroma_collection_id(col["chroma_name"]) + _chroma_req("POST", f"/api/v1/collections/{chroma_id}/delete", body={"where": {"doc_id": {"$eq": doc_id}}}) except Exception: pass @@ -2598,7 +2604,8 @@ async def rag_query(body: dict, user: dict = Depends(current_user)): raise HTTPException(status_code=503, detail="Failed to embed query") # Query ChromaDB - result = _chroma_req("POST", f"/api/v1/collections/{col['chroma_name']}/query", body={ + chroma_id = _chroma_collection_id(col["chroma_name"]) + result = _chroma_req("POST", f"/api/v1/collections/{chroma_id}/query", body={ "query_embeddings": embed, "n_results": n_results, "include": ["documents", "metadatas", "distances"] diff --git a/ansible/roles/cezen-backend/files/rag_ingest.py b/ansible/roles/cezen-backend/files/rag_ingest.py index 4f48269..1c900ab 100644 --- a/ansible/roles/cezen-backend/files/rag_ingest.py +++ b/ansible/roles/cezen-backend/files/rag_ingest.py @@ -162,9 +162,9 @@ def chunk_text(text: str, chunk_size: int = CHUNK_SIZE, overlap: int = CHUNK_OVE chunk = chunk[:idx + len(sep)] break chunks.append(chunk.strip()) - start += len(chunk) - overlap - if start >= len(text): + if end >= len(text): break + start += max(1, len(chunk) - overlap) return [c for c in chunks if c] # ── Embedding ───────────────────────────────────────────────────────────────── @@ -197,8 +197,10 @@ def chroma_upsert(ids, embeddings, documents, metadatas): "documents": documents, "metadatas": metadatas, }).encode() + with urllib.request.urlopen(f"{args.chroma_url}/api/v1/collections/{args.collection}", timeout=30) as response: + collection_id = json.loads(response.read().decode())["id"] req = urllib.request.Request( - f"{args.chroma_url}/api/v1/collections/{args.collection}/upsert", + f"{args.chroma_url}/api/v1/collections/{collection_id}/upsert", data=body, method="POST" ) req.add_header("Content-Type", "application/json") diff --git a/ansible/roles/cezen-ttyd/files/cezen-ttyd.service b/ansible/roles/cezen-ttyd/files/cezen-ttyd.service index 3a0132e..ba147b3 100644 --- a/ansible/roles/cezen-ttyd/files/cezen-ttyd.service +++ b/ansible/roles/cezen-ttyd/files/cezen-ttyd.service @@ -7,7 +7,6 @@ After=network.target ExecStart=/usr/bin/ttyd \ --port 7681 \ --interface 127.0.0.1 \ - --writable \ login -f cezen-console Restart=always RestartSec=5 diff --git a/ansible/roles/cezen-ttyd/tasks/main.yml b/ansible/roles/cezen-ttyd/tasks/main.yml index a6c9ecd..6cdbba8 100644 --- a/ansible/roles/cezen-ttyd/tasks/main.yml +++ b/ansible/roles/cezen-ttyd/tasks/main.yml @@ -7,6 +7,13 @@ state: present update_cache: yes +- name: Disable distribution ttyd service to reserve the portal console port + systemd: + name: ttyd + enabled: no + state: stopped + failed_when: false + - name: Create cezen-console restricted user user: name: cezen-console diff --git a/ansible/roles/chromadb/tasks/main.yml b/ansible/roles/chromadb/tasks/main.yml index 541856e..55de55b 100644 --- a/ansible/roles/chromadb/tasks/main.yml +++ b/ansible/roles/chromadb/tasks/main.yml @@ -1,9 +1,14 @@ --- # ChromaDB — vector database for RAG pipelines -- name: Install ChromaDB in cezen conda env - become_user: cezen - shell: | - /opt/cezen/miniconda/bin/conda run -n cezen pip install chromadb +- name: Create isolated ChromaDB virtual environment + command: /usr/bin/python3 -m venv /opt/cezen/chromadb-venv + args: + creates: /opt/cezen/chromadb-venv/bin/python + +- name: Install pinned ChromaDB in its isolated environment + pip: + name: chromadb==0.5.23 + virtualenv: /opt/cezen/chromadb-venv retries: 3 delay: 10 @@ -27,13 +32,13 @@ User=cezen Group=cezen WorkingDirectory=/opt/cezen/data/chromadb - ExecStart=/opt/cezen/miniconda/envs/cezen/bin/chroma run \ + ExecStart=/opt/cezen/chromadb-venv/bin/chroma run \ --host 0.0.0.0 \ --port 8100 \ --path /opt/cezen/data/chromadb Restart=always RestartSec=5 - Environment="PATH=/opt/cezen/miniconda/envs/cezen/bin:/usr/local/bin:/usr/bin:/bin" + Environment="PATH=/opt/cezen/chromadb-venv/bin:/usr/local/bin:/usr/bin:/bin" [Install] WantedBy=multi-user.target diff --git a/ansible/roles/vllm/tasks/main.yml b/ansible/roles/vllm/tasks/main.yml index 65a74b4..394a847 100644 --- a/ansible/roles/vllm/tasks/main.yml +++ b/ansible/roles/vllm/tasks/main.yml @@ -22,7 +22,7 @@ when: gpu_check.stdout == "" or gpu_check.rc != 0 - name: Pull vLLM Docker image - shell: docker pull vllm/vllm-openai:latest + shell: docker pull vllm/vllm-openai:v0.10.2 retries: 3 delay: 15 when: gpu_check.stdout != "" and gpu_check.rc == 0 @@ -53,12 +53,11 @@ -p 8000:8000 \ -v /opt/cezen/models:/root/.cache/huggingface \ -e HF_HOME=/root/.cache/huggingface \ - vllm/vllm-openai:latest \ + vllm/vllm-openai:v0.10.2 \ --model {{ vllm_model }} \ --gpu-memory-utilization {{ vllm_gpu_memory_util }} \ --max-model-len {{ vllm_max_model_len }} \ - --tensor-parallel-size {{ vllm_tensor_parallel }} \ - {{ vllm_quant_flag }} + --tensor-parallel-size {{ vllm_tensor_parallel }}{{ ' ' + vllm_quant_flag if vllm_quant_flag else '' }} ExecStop=/usr/bin/docker stop vllm TimeoutStartSec=300 @@ -92,5 +91,6 @@ systemd: name: vllm enabled: true + state: started daemon_reload: true when: gpu_check.stdout != "" and gpu_check.rc == 0 diff --git a/ansible/starter.yml b/ansible/starter.yml index 4a60c39..de04bff 100644 --- a/ansible/starter.yml +++ b/ansible/starter.yml @@ -32,7 +32,7 @@ vllm_tensor_parallel: 1 vllm_gpu_memory_util: "0.85" vllm_max_model_len: 4096 - vllm_quantization: "awq" + vllm_quantization: "" # Phi-3 source model is not AWQ-quantized # ── Ollama — lightweight models ─────────────── ollama_default_model: "phi3:mini" diff --git a/specs/001-enterprise-experience/evidence/server-phase1.md b/specs/001-enterprise-experience/evidence/server-phase1.md new file mode 100644 index 0000000..c87ce28 --- /dev/null +++ b/specs/001-enterprise-experience/evidence/server-phase1.md @@ -0,0 +1,75 @@ +# Server Phase 1 Validation Evidence + +**Validation date:** 2026-07-12 to 2026-07-13 + +**Target:** Vast.ai full Ubuntu 22.04 VM, RTX 3090 24 GB, 23 vCPU, 52.8 GB usable RAM, 155.9 GB root filesystem + +**Package revision at initial install:** `55a2a41` plus the uncommitted corrections listed below + +**Install mode:** `install.sh --software-only --tier=starter` + +## Executive result + +The software package installs and the tested Phase 1 Server workflows operate on the representative +cloud VM after correcting defects found during validation. This is evidence for a controlled Server +pilot only. It is not ISO boot evidence, physical hardware evidence, Workstation evidence, a customer +pilot, WCAG manual approval, or general-release approval. + +## Results + +| Area | Result | Evidence | +|---|---|---| +| Starter installation | PASS after Docker repository fix | Ansible recap: 86 ok, 51 changed, 0 failed, 2 ignored | +| Source contract suite | PASS | 19 tests passed | +| Shell/Python/static validation | PASS | `bash -n`, `py_compile`, `git diff --check` | +| Tier playbook syntax | PASS | starter, entry, pro, max syntax checks | +| Portal/backend | PASS | HTTP 200 after install and reboot | +| Authenticated read endpoints | PASS | 51 HTTP 200 responses | +| Entitlement restrictions | PASS | model router and two fine-tuning routes correctly returned 403 | +| Authentication and RBAC | PASS | admin login; viewer permitted metrics and denied admin user list | +| User/session lifecycle | PASS | create, login, revoke by deletion, expired-session rejection, self-delete protection | +| API key lifecycle | PASS | create, verify, revoke, revoked-key rejection | +| Guardrails | PASS | temporary keyword rule blocked matching input and was removed | +| Prompt library | PASS | create, update and delete | +| Backup/restore | PASS | ZIP created; listed; restored; pre-restore safety snapshot created | +| Ollama inference | PASS | Qwen 2.5 1.5B GPU inference; model persisted across reboot | +| Open WebUI | PASS after network correction | healthy container and model visible from WebUI | +| RAG | PASS after corrections | collection, TXT upload, embedding, ready status, semantic retrieval, delete/cleanup | +| vLLM | PASS after corrections | Phi-3 readiness and OpenAI-compatible chat completion | +| Monitoring | PASS | Grafana health 200, Prometheus ready 200, DCGM and node exporter running | +| Web console | PASS after correction | service active and `/console/` HTTP 200 | +| Reboot persistence | PASS | all six services and five containers auto-started; models and backup persisted | +| Concurrent inference | PASS | vLLM used 21.3 GB and Ollama 1.8 GB on RTX 3090; both completed inference | + +Post-reboot vLLM cold readiness occurred at approximately 144 seconds after boot. + +## Defects discovered and corrected in source + +1. Ubuntu ttyd does not support `--writable`, and the distribution ttyd service can occupy port 7681. +2. Open WebUI could not reach an Ollama process that had not reloaded the appliance service environment. +3. Backend targeted ChromaDB port 8000 while the service intentionally runs on 8100. +4. Unpinned latest ChromaDB removed the v1 API used by the backend. +5. Compatible ChromaDB dependencies conflicted with the shared AI environment; ChromaDB is now isolated. +6. Short RAG documents caused an infinite chunking loop. +7. Chroma query/upsert/delete data routes require collection UUIDs rather than collection names. +8. Starter Phi-3 was incorrectly configured as AWQ. +9. Blank optional vLLM arguments caused systemd command continuation into `ExecStop`. +10. `vllm:latest` required an incompatible CUDA/driver combination; the image is now pinned to v0.10.2. +11. vLLM was enabled but not explicitly started during installation. + +## Expected warnings / environment gaps + +- No final signed customer license was installed; field-staging mode was used. +- TLS was not configured; access used SSH tunnels. +- The VM began without a backup or models; both were created during testing. +- Direct public port 80 was blocked by the provider; this is not an appliance Nginx failure. +- Clean ISO boot, disk partitioning, BIOS/UEFI, USB media, physical NICs and offline ISO installation + cannot be validated on this rented VM. +- Workstation validation (T036), manual WCAG 2.2 AA evidence, interruption/restricted-network trials, + controlled customer pilot and release approvals remain pending. + +## Release decision + +**Not ready for customer shipment yet.** The corrected package is suitable for another clean Server +installation validation and controlled pilot preparation. A new ISO must be built from the corrected +revision and both Server and Workstation ISO paths must be tested before release consideration. diff --git a/specs/001-enterprise-experience/tasks.md b/specs/001-enterprise-experience/tasks.md index 688ae55..1109896 100644 --- a/specs/001-enterprise-experience/tasks.md +++ b/specs/001-enterprise-experience/tasks.md @@ -87,7 +87,7 @@ verify the same durable operation, accurate readiness, and safe next action rema - [x] T034 [US1] Add local setup, interruption, readiness, and escalation guidance in `cezen-portal/quickstart.html` and `cezen-portal/troubleshooting.html` - [x] T035 [US1] Add schema migration, operation recovery, and portal deployment steps to `ansible/roles/cezen-backend/tasks/main.yml` and `ansible/roles/cezen-nginx/tasks/main.yml` - [ ] T036 [US1] Verify clean install, interruption, restricted-network guidance, and WCAG evidence on a representative Workstation and record it in `specs/001-enterprise-experience/evidence/workstation-phase1.md` -- [ ] T037 [US1] Verify clean install, interruption, restricted-network guidance, and WCAG evidence on a representative Server and record it in `specs/001-enterprise-experience/evidence/server-phase1.md` +- [ ] T037 [US1] Verify clean install, interruption, restricted-network guidance, and WCAG evidence on a representative Server and record it in `specs/001-enterprise-experience/evidence/server-phase1.md` (software-package clean install and reboot evidence recorded; interruption, restricted-network, ISO boot, and manual WCAG evidence remain) - [x] T038 [US1] Verify source-to-package and source-to-ISO provenance for all Phase 1 surfaces and record checksums in `specs/001-enterprise-experience/evidence/phase1-manifest.md` **Checkpoint**: User Story 1 and product Phase 1 pass internal evidence review on Workstation and