Update ISO installer tiers and backend deployment

ansible/entry.yml

@@ -1,7 +1,7 @@
 ---
-# Phase 2: Full Cezen AI Suite — Entry Tier
+# Phase 2: Full Nexus One AI — Entry Tier
 # Runs after NVIDIA driver reboot
-- name: Cezen AI — Entry Tier Stack
+- name: Nexus One AI — Entry Tier Stack
   hosts: localhost
   connection: local
   become: true
@@ -35,3 +35,9 @@
       when: "'minio' not in skip_roles.split(',')"
     - role: monitoring
       when: "'monitoring' not in skip_roles.split(',')"
+    - role: cezen-backend
+      when: "'cezen-backend' not in skip_roles.split(',')"
+    - role: cezen-ttyd
+      when: "'cezen-ttyd' not in skip_roles.split(',')"
+    - role: cezen-nginx
+      when: "'cezen-nginx' not in skip_roles.split(',')"

ansible/max.yml

@@ -0,0 +1,83 @@
+---
+# Nexus One AI — Max Tier Stack
+# Hardware: 4–8× NVIDIA H100/A100/RTX 5090 (80–320 GB VRAM total), 256–512 GB DDR5, 8 TB+ NVMe, 100 GbE
+# Capacity: 100+ concurrent users
+# Runs after NVIDIA driver reboot (phase1_nvidia.yml)
+#
+# Differences from Pro tier:
+#   - vLLM tensor-parallel across 4+ GPUs (set vllm_tensor_parallel to GPU count)
+#   - Full precision models (no quantization required)
+#   - Advanced fine-tuning (QLoRA + DeepSpeed ZeRO-3 for multi-GPU training)
+#   - Full MLflow + MinIO stack for experiment tracking and artifact storage
+#   - All optional services enabled by default
+
+- name: Nexus One AI — Max Tier Stack
+  hosts: localhost
+  connection: local
+  become: true
+  vars:
+    cezen_user: "cezen"
+    cezen_home: "/opt/cezen"
+    cezen_login_home: "/home/cezen"
+    python_version: "3.11"
+    cuda_version: "12.6"
+    skip_roles: ""                # comma-separated list of roles to skip
+    gpu_available: false
+    tier: "max"
+
+    # ── vLLM — Max defaults ──────────────────────
+    # Full-precision Llama-3.1-70B across 4 GPUs by default.
+    # For HGX/DGX-class systems with 8 GPUs set vllm_tensor_parallel: 8
+    # and switch to Llama-3.1-405B or Mixtral-8x22B.
+    vllm_model: "meta-llama/Meta-Llama-3.1-70B-Instruct"
+    vllm_tensor_parallel: 4
+    vllm_gpu_memory_util: "0.90"
+    vllm_max_model_len: 32768
+    vllm_quantization: ""          # full precision at Max tier
+
+    # ── Ollama — large model defaults ────────────
+    ollama_default_model: "llama3.1:70b"
+
+    # ── DeepSpeed — multi-GPU fine-tuning ────────
+    deepspeed_enabled: true
+    deepspeed_zero_stage: 3        # ZeRO-3 for large model training
+
+  roles:
+    - role: base
+      when: "'base' not in skip_roles.split(',')"
+
+    - role: docker
+      when: "'docker' not in skip_roles.split(',')"
+
+    - role: k3s
+      when: "'k3s' not in skip_roles.split(',')"
+
+    - role: ollama
+      when: "'ollama' not in skip_roles.split(',')"
+
+    - role: vllm
+      when: "'vllm' not in skip_roles.split(',')"
+
+    - role: jupyterlab
+      when: "'jupyterlab' not in skip_roles.split(',')"
+
+    - role: chromadb
+      when: "'chromadb' not in skip_roles.split(',')"
+
+    - role: mlflow
+      when: "'mlflow' not in skip_roles.split(',')"
+
+    - role: minio
+      when: "'minio' not in skip_roles.split(',')"
+
+    - role: monitoring
+      when: "'monitoring' not in skip_roles.split(',')"
+
+    - role: cezen-backend
+      when: "'cezen-backend' not in skip_roles.split(',')"
+
+    - role: cezen-ttyd
+      when: "'cezen-ttyd' not in skip_roles.split(',')"
+
+    - role: cezen-nginx
+      when: "'cezen-nginx' not in skip_roles.split(',')"

ansible/phase1_nvidia.yml

@@ -1,6 +1,6 @@
 ---
 # Phase 1: NVIDIA drivers only. Server reboots after this.
-- name: Cezen AI — Phase 1 NVIDIA Drivers
+- name: Nexus One AI — Phase 1 NVIDIA Drivers
   hosts: localhost
   connection: local
   become: true

ansible/pro.yml

@@ -0,0 +1,79 @@
+---
+# Nexus One AI — Pro Tier Stack
+# Hardware: 2× NVIDIA RTX 5090 (32 GB GDDR7 each / 64 GB total), 128 GB DDR5, 4 TB NVMe, 10 GbE
+# Capacity: 20–100 concurrent users
+# Runs after NVIDIA driver reboot (phase1_nvidia.yml)
+#
+# Differences from Basic tier:
+#   - k3s included (multi-service orchestration at this scale)
+#   - MLflow included (fine-tuning tracking needed at Pro)
+#   - MinIO included (model + data storage at scale)
+#   - vLLM runs tensor-parallel across 2 GPUs
+#   - QLoRA fine-tuning available via portal
+
+- name: Nexus One AI — Pro Tier Stack
+  hosts: localhost
+  connection: local
+  become: true
+  vars:
+    cezen_user: "cezen"
+    cezen_home: "/opt/cezen"
+    cezen_login_home: "/home/cezen"
+    python_version: "3.11"
+    cuda_version: "12.6"
+    skip_roles: ""                # comma-separated list of roles to skip
+    gpu_available: false
+    tier: "pro"
+
+    # ── vLLM — Pro defaults ──────────────────────
+    # Tensor-parallel across 2× RTX 5090 (64 GB combined GDDR7).
+    # Llama-3.1-70B at 4-bit fits comfortably; switch to full-precision
+    # smaller models via the portal Model Manager.
+    vllm_model: "meta-llama/Meta-Llama-3.1-70B-Instruct"
+    vllm_tensor_parallel: 2
+    vllm_gpu_memory_util: "0.85"
+    vllm_max_model_len: 8192
+    vllm_quantization: "awq"
+
+    # ── Ollama — full-size models ─────────────────
+    ollama_default_model: "llama3.1:70b"
+
+  roles:
+    - role: base
+      when: "'base' not in skip_roles.split(',')"
+
+    - role: docker
+      when: "'docker' not in skip_roles.split(',')"
+
+    - role: k3s
+      when: "'k3s' not in skip_roles.split(',')"
+
+    - role: ollama
+      when: "'ollama' not in skip_roles.split(',')"
+
+    - role: vllm
+      when: "'vllm' not in skip_roles.split(',')"
+
+    - role: jupyterlab
+      when: "'jupyterlab' not in skip_roles.split(',')"
+
+    - role: chromadb
+      when: "'chromadb' not in skip_roles.split(',')"
+
+    - role: mlflow
+      when: "'mlflow' not in skip_roles.split(',')"
+
+    - role: minio
+      when: "'minio' not in skip_roles.split(',')"
+
+    - role: monitoring
+      when: "'monitoring' not in skip_roles.split(',')"
+
+    - role: cezen-backend
+      when: "'cezen-backend' not in skip_roles.split(',')"
+
+    - role: cezen-ttyd
+      when: "'cezen-ttyd' not in skip_roles.split(',')"
+
+    - role: cezen-nginx
+      when: "'cezen-nginx' not in skip_roles.split(',')"

ansible/roles/base/tasks/main.yml

@@ -50,6 +50,18 @@
     - "{{ cezen_home }}/models"
     - "{{ cezen_home }}/data"
     - "{{ cezen_home }}/logs"
+    - "{{ cezen_home }}/scripts"
+
+- name: Install Cezen operational helper scripts
+  copy:
+    src: "{{ playbook_dir }}/../scripts/{{ item }}"
+    dest: "{{ cezen_home }}/scripts/{{ item }}"
+    owner: "{{ cezen_user }}"
+    group: "{{ cezen_user }}"
+    mode: "0755"
+  loop:
+    - cezen-backup.sh
+    - cezen-feasibility.sh
 
 - name: Download Miniconda
   get_url:

ansible/roles/cezen-backend/files/cezen-api.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=Nexus One AI Management API
+After=network-online.target ollama.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=cezen
+WorkingDirectory=/opt/cezen/backend
+Environment="CEZEN_DATA=/opt/cezen/data"
+Environment="OLLAMA_URL=http://localhost:11434"
+Environment="PATH=/opt/cezen/backend/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin"
+ExecStart=/opt/cezen/backend/venv/bin/uvicorn main:app --host 0.0.0.0 --port 8080 --workers 2
+Restart=always
+RestartSec=5
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/cezen-backend/files/requirements.txt

@@ -0,0 +1,13 @@
+fastapi>=0.111.0
+uvicorn[standard]>=0.29.0
+python-jose[cryptography]>=3.3.0
+passlib[bcrypt]>=1.7.4
+bcrypt<4.0.0
+psutil>=5.9.0
+python-multipart>=0.0.9
+aiofiles>=23.0.0
+# Document Intelligence
+pymupdf>=1.24.0          # PDF text extraction (fitz)
+python-docx>=1.1.0       # Word document extraction
+# Scheduled Jobs
+apscheduler>=3.10.0      # In-process cron/interval scheduler

ansible/roles/cezen-backend/files/train_qlora.py

@@ -0,0 +1,309 @@
+#!/usr/bin/env python3
+"""
+Nexus One AI — QLoRA Fine-Tuning Runner
+Launched as a subprocess by the FastAPI backend.
+
+Writes structured JSONL log lines to --log-path so the UI can stream
+live loss curves and progress. Updates training_jobs.status in SQLite.
+
+Requires (install on the training node):
+    pip install torch transformers datasets peft bitsandbytes trl
+
+Optional (faster, lower VRAM):
+    pip install unsloth
+
+Usage (called by main.py — do not run manually in production):
+    python3 train_qlora.py --job-id 1 --db-path /opt/cezen/data/cezen.db \
+        --dataset /opt/cezen/data/datasets/abc.jsonl \
+        --base-model mistral:7b --output-dir /opt/cezen/data/finetuned/mymodel \
+        --log-path /opt/cezen/data/job_logs/abc.jsonl \
+        --epochs 3 --lr 2e-4 --batch-size 4 --lora-r 16 --lora-alpha 32 \
+        --output-name mymodel
+"""
+
+import argparse, json, os, sqlite3, sys, time
+from datetime import datetime, timezone
+from pathlib import Path
+
+# ── Argument parsing ──────────────────────────────────────────────────────────
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--job-id",     type=int, required=True)
+parser.add_argument("--db-path",    required=True)
+parser.add_argument("--dataset",    required=True)
+parser.add_argument("--base-model", required=True)
+parser.add_argument("--output-dir", required=True)
+parser.add_argument("--log-path",   required=True)
+parser.add_argument("--output-name", required=True)
+parser.add_argument("--epochs",     type=int,   default=3)
+parser.add_argument("--lr",         type=float, default=2e-4)
+parser.add_argument("--batch-size", type=int,   default=4)
+parser.add_argument("--lora-r",     type=int,   default=16)
+parser.add_argument("--lora-alpha", type=int,   default=32)
+args = parser.parse_args()
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+def utcnow():
+    return datetime.now(timezone.utc).isoformat()
+
+def db_connect():
+    conn = sqlite3.connect(args.db_path)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+log_file = open(args.log_path, "a", buffering=1)
+
+def log(type_: str, **kwargs):
+    entry = {"ts": utcnow(), "type": type_, **kwargs}
+    log_file.write(json.dumps(entry) + "\n")
+
+def set_status(status: str):
+    db = db_connect()
+    if status in ("completed", "failed", "cancelled"):
+        db.execute(
+            "UPDATE training_jobs SET status=?, finished_at=? WHERE id=?",
+            (status, utcnow(), args.job_id)
+        )
+    else:
+        db.execute("UPDATE training_jobs SET status=? WHERE id=?", (status, args.job_id))
+    db.commit()
+    db.close()
+
+# ── Dataset loading ───────────────────────────────────────────────────────────
+
+def load_dataset_from_file(path: str):
+    """Load JSONL or CSV dataset into a list of dicts with 'text' or 'prompt'/'completion' keys."""
+    p = Path(path)
+    rows = []
+    if p.suffix.lower() == ".csv":
+        import csv
+        with open(path, newline="", encoding="utf-8", errors="replace") as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                rows.append(dict(row))
+    else:
+        with open(path, encoding="utf-8", errors="replace") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    try:
+                        rows.append(json.loads(line))
+                    except Exception:
+                        pass
+    return rows
+
+def format_row(row: dict) -> str:
+    """Convert a dataset row to a plain text training string."""
+    if "text" in row:
+        return row["text"]
+    if "prompt" in row and "completion" in row:
+        return f"### Instruction:\n{row['prompt']}\n\n### Response:\n{row['completion']}"
+    if "instruction" in row and "output" in row:
+        inp = row.get("input", "")
+        return (f"### Instruction:\n{row['instruction']}\n\n### Input:\n{inp}\n\n### Response:\n{row['output']}"
+                if inp else
+                f"### Instruction:\n{row['instruction']}\n\n### Response:\n{row['output']}")
+    # Fallback: concatenate all values
+    return " ".join(str(v) for v in row.values() if v)
+
+# ── Main training routine ─────────────────────────────────────────────────────
+
+def main():
+    log("start", job_id=args.job_id, base_model=args.base_model,
+        epochs=args.epochs, lr=args.lr, batch_size=args.batch_size,
+        lora_r=args.lora_r, lora_alpha=args.lora_alpha)
+    set_status("running")
+
+    # Resolve model name (Ollama uses "mistral:7b" style — strip the tag for HF)
+    hf_model = args.base_model
+    if ":" in hf_model and "/" not in hf_model:
+        # e.g. "mistral:7b" → try to map to HF repo
+        name_map = {
+            "mistral":  "mistralai/Mistral-7B-v0.1",
+            "llama2":   "meta-llama/Llama-2-7b-hf",
+            "llama3":   "meta-llama/Meta-Llama-3-8B",
+            "phi3":     "microsoft/Phi-3-mini-4k-instruct",
+            "gemma":    "google/gemma-7b",
+            "codellama":"codellama/CodeLlama-7b-hf",
+            "qwen2":    "Qwen/Qwen2-7B",
+        }
+        base_name = hf_model.split(":")[0].lower()
+        hf_model = name_map.get(base_name, hf_model)
+        log("info", msg=f"Mapped '{args.base_model}' → '{hf_model}' (HuggingFace)")
+
+    # Load dataset
+    log("info", msg="Loading dataset...")
+    raw_rows = load_dataset_from_file(args.dataset)
+    if not raw_rows:
+        log("error", msg="Dataset is empty or could not be parsed")
+        set_status("failed")
+        sys.exit(1)
+
+    texts = [format_row(r) for r in raw_rows]
+    log("info", msg=f"Loaded {len(texts)} training examples")
+
+    # Try Unsloth first (faster), fall back to HF PEFT
+    use_unsloth = False
+    try:
+        from unsloth import FastLanguageModel
+        use_unsloth = True
+        log("info", msg="Using Unsloth for accelerated training")
+    except ImportError:
+        log("info", msg="Unsloth not available — using HuggingFace PEFT + BitsAndBytes")
+
+    try:
+        import torch
+        from transformers import TrainingArguments, TrainerCallback
+        from datasets import Dataset as HFDataset
+
+        if use_unsloth:
+            model, tokenizer = FastLanguageModel.from_pretrained(
+                model_name=hf_model,
+                max_seq_length=2048,
+                dtype=None,
+                load_in_4bit=True,
+            )
+            model = FastLanguageModel.get_peft_model(
+                model,
+                r=args.lora_r,
+                lora_alpha=args.lora_alpha,
+                target_modules=["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj","down_proj"],
+                lora_dropout=0,
+                bias="none",
+                use_gradient_checkpointing="unsloth",
+            )
+        else:
+            from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+            from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+
+            bnb_cfg = BitsAndBytesConfig(
+                load_in_4bit=True,
+                bnb_4bit_quant_type="nf4",
+                bnb_4bit_compute_dtype=torch.float16,
+            )
+            tokenizer = AutoTokenizer.from_pretrained(hf_model, trust_remote_code=True)
+            if tokenizer.pad_token is None:
+                tokenizer.pad_token = tokenizer.eos_token
+
+            model = AutoModelForCausalLM.from_pretrained(
+                hf_model,
+                quantization_config=bnb_cfg,
+                device_map="auto",
+                trust_remote_code=True,
+            )
+            model = prepare_model_for_kbit_training(model)
+
+            lora_cfg = LoraConfig(
+                r=args.lora_r,
+                lora_alpha=args.lora_alpha,
+                target_modules=["q_proj","k_proj","v_proj","o_proj"],
+                lora_dropout=0.05,
+                bias="none",
+                task_type="CAUSAL_LM",
+            )
+            model = get_peft_model(model, lora_cfg)
+
+        # Tokenise
+        def tokenise(examples):
+            return tokenizer(
+                examples["text"],
+                truncation=True,
+                max_length=2048,
+                padding="max_length",
+            )
+
+        hf_ds = HFDataset.from_dict({"text": texts})
+        hf_ds = hf_ds.map(tokenise, batched=True, remove_columns=["text"])
+
+        # Custom callback to stream loss to our log
+        class LossLogger(TrainerCallback):
+            def on_log(self, _args, state, control, logs=None, **kwargs):
+                if logs and "loss" in logs:
+                    log("loss",
+                        step=state.global_step,
+                        loss=round(float(logs["loss"]), 6),
+                        epoch=round(float(logs.get("epoch", 0)), 3),
+                        lr=float(logs.get("learning_rate", args.lr)))
+
+        output_dir = args.output_dir
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+
+        from trl import SFTTrainer
+        trainer = SFTTrainer(
+            model=model,
+            tokenizer=tokenizer,
+            train_dataset=hf_ds,
+            dataset_text_field="input_ids",
+            max_seq_length=2048,
+            args=TrainingArguments(
+                output_dir=output_dir,
+                num_train_epochs=args.epochs,
+                per_device_train_batch_size=args.batch_size,
+                gradient_accumulation_steps=4,
+                warmup_steps=5,
+                learning_rate=args.lr,
+                fp16=not torch.cuda.is_bf16_supported(),
+                bf16=torch.cuda.is_bf16_supported(),
+                logging_steps=1,
+                save_strategy="epoch",
+                report_to="none",
+            ),
+            callbacks=[LossLogger()],
+        )
+
+        log("info", msg="Training started")
+        trainer.train()
+        log("info", msg="Training complete — saving model")
+        trainer.save_model(output_dir)
+        tokenizer.save_pretrained(output_dir)
+
+    except Exception as e:
+        import traceback
+        log("error", msg=str(e), traceback=traceback.format_exc())
+        set_status("failed")
+        sys.exit(1)
+
+    # Auto-register with Ollama via Modelfile
+    try:
+        _register_with_ollama(output_dir, args.output_name)
+    except Exception as e:
+        log("warning", msg=f"Could not auto-register with Ollama: {e}")
+
+    log("complete", msg="Job finished successfully", output_dir=output_dir)
+    set_status("completed")
+
+
+def _register_with_ollama(model_dir: str, model_name: str):
+    """Create an Ollama Modelfile and register the fine-tuned model."""
+    modelfile_path = Path(model_dir) / "Modelfile"
+    modelfile_path.write_text(
+        f'FROM {model_dir}\n'
+        f'PARAMETER stop "<|im_end|>"\n'
+        f'SYSTEM "This is a Nexus One AI fine-tuned model."\n'
+    )
+    import subprocess
+    result = subprocess.run(
+        ["ollama", "create", model_name, "-f", str(modelfile_path)],
+        capture_output=True, text=True, timeout=300
+    )
+    if result.returncode == 0:
+        log("info", msg=f"Model '{model_name}' registered with Ollama")
+    else:
+        log("warning", msg=f"Ollama registration failed: {result.stderr}")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        log("error", msg="Job interrupted (SIGTERM/SIGINT)")
+        set_status("cancelled")
+        sys.exit(130)
+    except Exception as e:
+        import traceback
+        log("error", msg=str(e), traceback=traceback.format_exc())
+        set_status("failed")
+        sys.exit(1)
+    finally:
+        log_file.close()

ansible/roles/cezen-backend/handlers/main.yml

@@ -0,0 +1,9 @@
+---
+- name: Reload systemd
+  systemd:
+    daemon_reload: yes
+
+- name: Restart cezen-api
+  systemd:
+    name: cezen-api
+    state: restarted

ansible/roles/cezen-backend/tasks/main.yml

@@ -0,0 +1,113 @@
+---
+# cezen-backend role: installs the Nexus One AI FastAPI management API
+
+- name: Install system Python deps
+  apt:
+    name:
+      - python3-pip
+      - python3-venv
+      - python3.11
+      - python3.11-venv
+      - libmupdf-dev      # required by pymupdf (Document Intelligence)
+      - mupdf-tools
+    state: present
+    update_cache: yes
+
+- name: Create backend directory
+  file:
+    path: /opt/cezen/backend
+    state: directory
+    owner: "{{ cezen_user }}"
+    group: "{{ cezen_user }}"
+    mode: "0755"
+
+- name: Create data directory (JWT secret + SQLite DB)
+  file:
+    path: /opt/cezen/data
+    state: directory
+    owner: "{{ cezen_user }}"
+    group: "{{ cezen_user }}"
+    mode: "0750"
+
+- name: Copy FastAPI application
+  copy:
+    src: main.py
+    dest: /opt/cezen/backend/main.py
+    owner: "{{ cezen_user }}"
+    group: "{{ cezen_user }}"
+    mode: "0644"
+  notify: Restart cezen-api
+
+- name: Copy QLoRA training runner
+  copy:
+    src: train_qlora.py
+    dest: /opt/cezen/backend/train_qlora.py
+    owner: "{{ cezen_user }}"
+    group: "{{ cezen_user }}"
+    mode: "0755"
+
+- name: Copy requirements.txt
+  copy:
+    src: requirements.txt
+    dest: /opt/cezen/backend/requirements.txt
+    owner: "{{ cezen_user }}"
+    group: "{{ cezen_user }}"
+    mode: "0644"
+
+- name: Create Python virtual environment (Python 3.11)
+  become_user: "{{ cezen_user }}"
+  command: python3.11 -m venv /opt/cezen/backend/venv
+  args:
+    creates: /opt/cezen/backend/venv/bin/activate
+
+- name: Install Python dependencies
+  become_user: "{{ cezen_user }}"
+  pip:
+    requirements: /opt/cezen/backend/requirements.txt
+    virtualenv: /opt/cezen/backend/venv
+  notify: Restart cezen-api
+
+- name: Install Pro/Max fine-tuning dependencies
+  become_user: "{{ cezen_user }}"
+  pip:
+    name:
+      - torch
+      - transformers
+      - datasets
+      - peft
+      - bitsandbytes
+      - accelerate
+      - trl
+      - sentencepiece
+    virtualenv: /opt/cezen/backend/venv
+  retries: 3
+  delay: 15
+  when: (tier | default('basic')) in ['pro', 'max']
+
+- name: Install Max multi-GPU training dependencies
+  become_user: "{{ cezen_user }}"
+  pip:
+    name:
+      - deepspeed
+    virtualenv: /opt/cezen/backend/venv
+  retries: 3
+  delay: 15
+  when: (tier | default('basic')) == 'max'
+
+- name: Install systemd service unit
+  copy:
+    src: cezen-api.service
+    dest: /etc/systemd/system/cezen-api.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify:
+    - Reload systemd
+    - Restart cezen-api
+
+- name: Enable and start cezen-api service
+  systemd:
+    name: cezen-api
+    enabled: yes
+    state: started
+    daemon_reload: yes

ansible/roles/cezen-nginx/files/cezen.conf

@@ -0,0 +1,135 @@
+# /etc/nginx/sites-available/cezen
+# Nexus One AI Portal — serves static portal, proxies API and console terminal
+#
+# Install:
+#   sudo cp cezen.conf /etc/nginx/sites-available/cezen
+#   sudo ln -sf /etc/nginx/sites-available/cezen /etc/nginx/sites-enabled/cezen
+#   sudo rm -f /etc/nginx/sites-enabled/default
+#   sudo nginx -t && sudo systemctl reload nginx
+
+# ─── Rate limiting zones (must be outside server block) ──────────────────────
+# Login: 5 requests/min per IP, burst of 3 queued, then 429
+limit_req_zone $binary_remote_addr zone=cezen_login:10m rate=5r/m;
+# General API: 60 req/min per IP (generous for dashboard polling)
+limit_req_zone $binary_remote_addr zone=cezen_api:10m   rate=60r/m;
+
+server {
+    listen 80 default_server;
+    listen [::]:80 default_server;
+
+    server_name _;
+
+    # Hide server version
+    server_tokens off;
+
+    # Logging
+    access_log /var/log/nginx/cezen-access.log;
+    error_log  /var/log/nginx/cezen-error.log;
+
+    # ─── Global security headers ──────────────────────────────────────────────
+    add_header X-Content-Type-Options  "nosniff"                   always;
+    add_header X-Frame-Options         "SAMEORIGIN"                always;
+    add_header X-XSS-Protection        "1; mode=block"             always;
+    add_header Referrer-Policy         "strict-origin-when-cross-origin" always;
+    add_header Permissions-Policy      "geolocation=(), camera=(), microphone=()" always;
+    add_header Content-Security-Policy
+        "default-src 'self'; "
+        "script-src 'self' 'unsafe-inline'; "
+        "style-src 'self' 'unsafe-inline'; "
+        "img-src 'self' data:; "
+        "connect-src 'self'; "
+        "frame-src 'self'; "
+        "font-src 'self'; "
+        "object-src 'none'; "
+        "base-uri 'self';"
+        always;
+
+    # ─── robots.txt — block all indexing (air-gapped / private portal) ────────
+    location = /robots.txt {
+        return 200 "User-agent: *\nDisallow: /\n";
+        add_header Content-Type text/plain;
+    }
+
+    # ─── Static Portal ───────────────────────────────────────────────────────
+    root /opt/cezen/portal;
+    index index.html;
+
+    location / {
+        try_files $uri $uri/ /index.html;
+    }
+
+    # Cache static assets aggressively
+    location ~* \.(css|js|png|jpg|jpeg|gif|ico|svg|woff2?)$ {
+        expires 7d;
+        add_header Cache-Control "public, immutable";
+    }
+
+    # ─── Model upload (large files — no size limit, extended timeout) ────────
+    location = /api/models/upload {
+        client_max_body_size    0;         # unlimited — GGUF files can be 70 GB+
+        proxy_request_buffering off;       # stream directly to backend, don't buffer in Nginx
+        proxy_read_timeout      7200s;     # 2 hours for slow transfers
+        proxy_send_timeout      7200s;
+
+        proxy_pass         http://127.0.0.1:8080;
+        proxy_http_version 1.1;
+        proxy_set_header   Host              $host;
+        proxy_set_header   X-Real-IP         $remote_addr;
+        proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+    }
+
+    # ─── Login rate limit (tight) ─────────────────────────────────────────────
+    location = /api/auth/login {
+        limit_req zone=cezen_login burst=3 nodelay;
+        limit_req_status 429;
+
+        proxy_pass         http://127.0.0.1:8080;
+        proxy_http_version 1.1;
+        proxy_set_header   Host              $host;
+        proxy_set_header   X-Real-IP         $remote_addr;
+        proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+        proxy_set_header   X-Forwarded-Proto $scheme;
+        proxy_read_timeout 30s;
+    }
+
+    # ─── FastAPI Backend (/api/) ──────────────────────────────────────────────
+    location /api/ {
+        limit_req zone=cezen_api burst=20 nodelay;
+        limit_req_status 429;
+
+        proxy_pass         http://127.0.0.1:8080;
+        proxy_http_version 1.1;
+        proxy_set_header   Host              $host;
+        proxy_set_header   X-Real-IP         $remote_addr;
+        proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
+        proxy_set_header   X-Forwarded-Proto $scheme;
+        proxy_read_timeout 120s;
+    }
+
+    # ─── Web Console (ttyd) (/console/) ──────────────────────────────────────
+    location /console/ {
+        proxy_pass         http://127.0.0.1:7681/;
+        proxy_http_version 1.1;
+        proxy_set_header   Upgrade           $http_upgrade;
+        proxy_set_header   Connection        "upgrade";
+        proxy_set_header   Host              $host;
+        proxy_set_header   X-Real-IP         $remote_addr;
+        proxy_read_timeout 86400s;
+
+        # Rewrite paths so ttyd JS/CSS assets load correctly
+        proxy_redirect     / /console/;
+        sub_filter         'href="/'  'href="/console/';
+        sub_filter         'src="/'   'src="/console/';
+        sub_filter_once    off;
+        sub_filter_types   text/html;
+    }
+
+    # ─── Block dotfiles and common attack paths ───────────────────────────────
+    location ~ /\. {
+        deny all;
+    }
+
+    location ~* \.(env|git|sql|bak|sh|py)$ {
+        deny all;
+    }
+}

ansible/roles/cezen-nginx/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Reload nginx
+  systemd:
+    name: nginx
+    state: reloaded

ansible/roles/cezen-nginx/tasks/main.yml

@@ -0,0 +1,59 @@
+---
+# cezen-nginx role: installs Nginx, deploys portal static files and site config
+
+- name: Install Nginx
+  apt:
+    name: nginx
+    state: present
+    update_cache: yes
+
+- name: Create portal directory
+  file:
+    path: /opt/cezen/portal
+    state: directory
+    owner: "{{ cezen_user }}"
+    group: www-data
+    mode: "0755"
+
+- name: Sync portal static files
+  synchronize:
+    src: "{{ playbook_dir }}/../../../cezen-portal/"
+    dest: /opt/cezen/portal/
+    delete: yes
+    recursive: yes
+    rsync_opts:
+      - "--exclude=.DS_Store"
+      - "--exclude=*.sh"
+  notify: Reload nginx
+
+- name: Deploy Nginx site config
+  copy:
+    src: cezen.conf
+    dest: /etc/nginx/sites-available/cezen
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload nginx
+
+- name: Enable Cezen site
+  file:
+    src: /etc/nginx/sites-available/cezen
+    dest: /etc/nginx/sites-enabled/cezen
+    state: link
+  notify: Reload nginx
+
+- name: Disable default Nginx site
+  file:
+    path: /etc/nginx/sites-enabled/default
+    state: absent
+  notify: Reload nginx
+
+- name: Validate Nginx config
+  command: nginx -t
+  changed_when: false
+
+- name: Ensure Nginx is enabled and running
+  systemd:
+    name: nginx
+    enabled: yes
+    state: started

ansible/roles/cezen-ttyd/files/cezen-ttyd.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Cezen Web Terminal (ttyd)
+After=network.target
+
+[Service]
+# Bind to localhost only — Nginx proxies /console/ to this port
+ExecStart=/usr/bin/ttyd \
+    --port 7681 \
+    --interface 127.0.0.1 \
+    --writable \
+    login -f cezen-console
+Restart=always
+RestartSec=5
+User=root
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/cezen-ttyd/handlers/main.yml

@@ -0,0 +1,9 @@
+---
+- name: Reload systemd
+  systemd:
+    daemon_reload: yes
+
+- name: Restart cezen-ttyd
+  systemd:
+    name: cezen-ttyd
+    state: restarted

ansible/roles/cezen-ttyd/tasks/main.yml

@@ -0,0 +1,72 @@
+---
+# cezen-ttyd role: browser-based terminal via ttyd, bound to localhost
+
+- name: Install ttyd
+  apt:
+    name: ttyd
+    state: present
+    update_cache: yes
+
+- name: Create cezen-console restricted user
+  user:
+    name: cezen-console
+    shell: /bin/bash
+    comment: "Cezen Web Console User"
+    groups: "{{ cezen_user }}"
+    append: yes
+    state: present
+    create_home: yes
+
+- name: Set cezen-console password
+  # Change this password after first login or use PAM/SSO integration
+  shell: echo "cezen-console:CezenConsole2024!" | chpasswd
+  changed_when: false
+  no_log: true
+
+- name: Restrict cezen-console home directory
+  file:
+    path: /home/cezen-console
+    owner: cezen-console
+    group: cezen-console
+    mode: "0750"
+
+- name: Add useful aliases for console user
+  copy:
+    dest: /home/cezen-console/.bashrc
+    owner: cezen-console
+    group: cezen-console
+    mode: "0644"
+    content: |
+      # Cezen Web Console — restricted shell environment
+      PS1='\[\033[01;32m\]cezen-console\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
+
+      # Useful shortcuts
+      alias ll='ls -lah --color=auto'
+      alias logs='journalctl -u cezen-api -f'
+      alias api-status='systemctl status cezen-api'
+      alias ollama-ps='ollama ps'
+      alias gpu='nvidia-smi'
+      alias ports='ss -tlnp'
+
+      # Prevent accidental system damage
+      alias rm='rm -i'
+      alias mv='mv -i'
+      alias cp='cp -i'
+
+- name: Install ttyd systemd service
+  copy:
+    src: cezen-ttyd.service
+    dest: /etc/systemd/system/cezen-ttyd.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify:
+    - Reload systemd
+    - Restart cezen-ttyd
+
+- name: Enable and start ttyd service
+  systemd:
+    name: cezen-ttyd
+    enabled: yes
+    state: started
+    daemon_reload: yes

ansible/roles/jupyterlab/tasks/main.yml

@@ -51,7 +51,7 @@
   copy:
     dest: /opt/cezen/notebooks/README.md
     content: |
-      # Cezen AI Suite — JupyterLab
+      # Nexus One AI — JupyterLab
 
       Default token: `cezen2024`
 

ansible/roles/mlflow/tasks/main.yml

@@ -53,5 +53,5 @@
   wait_for:
     host: localhost
     port: 5000
-    timeout: 30
+    timeout: 120
   ignore_errors: true

ansible/roles/monitoring/tasks/main.yml

@@ -25,6 +25,7 @@
   register: dcgm_result
   failed_when: dcgm_result.rc != 0 and 'already in use' not in dcgm_result.stderr
   ignore_errors: true
+  when: gpu_available | default(false) | bool
 
 # ── Prometheus ──────────────────────────────────────────
 - name: Write Prometheus config
@@ -99,7 +100,8 @@
   wait_for:
     host: localhost
     port: 3000
-    timeout: 60
+    timeout: 120
+  register: grafana_wait
   ignore_errors: true
 
 - name: Add Prometheus datasource to Grafana
@@ -118,6 +120,7 @@
       isDefault: true
     status_code: [200, 409]  # 409 = already exists, that's fine
   ignore_errors: true
+  when: not (grafana_wait is failed)
 
 - name: Import NVIDIA GPU dashboard (ID 12239)
   uri:
@@ -143,3 +146,6 @@
         uid: "nvidia-gpu"
     status_code: [200, 412]
   ignore_errors: true
+  when:
+    - not (grafana_wait is failed)
+    - gpu_available | default(false) | bool

ansible/roles/nvidia/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
 # NVIDIA role: Drivers + CUDA + cuDNN
-# NOTE: Tested on L40S (Entry) and A40 (lab). Requires reboot after this role.
+# NOTE: Tested on RTX Pro 6000 (Entry) and A40 (lab). Requires reboot after this role.
 #       If no GPU is present, this role will install drivers but nvidia-smi won't show GPUs.
 
 - name: Add NVIDIA package repository key

ansible/roles/ollama/tasks/main.yml

@@ -28,7 +28,7 @@
       Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
       Environment="OLLAMA_HOST=0.0.0.0:11434"
       Environment="OLLAMA_MODELS=/opt/cezen/models/ollama"
-      Environment="CUDA_VISIBLE_DEVICES=0,1,2"
+      Environment="CUDA_VISIBLE_DEVICES=0"
 
       [Install]
       WantedBy=multi-user.target
@@ -54,50 +54,49 @@
     port: 11434
     timeout: 60
 
-- name: Pull default models (Llama 3.1 8B + Mistral 7B)
+- name: Select tier model set
+  set_fact:
+    ollama_models: >-
+      {{
+        {
+          'starter': ['phi3:mini', 'nomic-embed-text'],
+          'basic': ['llama3.1:8b', 'mistral:7b', 'nomic-embed-text', 'codellama:13b'],
+          'entry': ['llama3.1:8b', 'mistral:7b', 'nomic-embed-text', 'codellama:13b'],
+          'pro': ['llama3.1:8b', 'mistral:7b', 'nomic-embed-text', 'codellama:13b', 'llama3.1:70b', 'mixtral:8x7b', 'deepseek-coder-v2:16b'],
+          'max': ['llama3.1:8b', 'mistral:7b', 'nomic-embed-text', 'codellama:13b', 'llama3.1:70b', 'mixtral:8x7b', 'deepseek-coder-v2:16b', 'llama3.1:405b', 'mixtral:8x22b']
+        }.get(tier | default('basic'), ['llama3.1:8b', 'mistral:7b', 'nomic-embed-text'])
+      }}
+
+- name: Pull tier Ollama models
   become_user: cezen
   command: ollama pull {{ item }}
-  loop:
-    - llama3.1:8b
-    - mistral:7b
+  loop: "{{ ollama_models }}"
   environment:
     OLLAMA_HOST: "http://localhost:11434"
   retries: 3
   delay: 15
-  # NOTE: Models are large (~5GB each). This step takes time on first run.
-  # Skip by setting: ansible-playbook ... -e "skip_model_pull=true"
+  # NOTE: Pro/Max models are very large. Skip with --skip-model-pull for
+  # bandwidth-constrained installs, then run models/pull-models.sh later.
   when: not (skip_model_pull | default(false))
 
 # Open WebUI (chat interface on top of Ollama)
-- name: Deploy Open WebUI via Docker
-  community.docker.docker_container:
-    name: open-webui
-    image: ghcr.io/open-webui/open-webui:main
-    state: started
-    restart_policy: always
-    ports:
-      - "3001:8080"
-    volumes:
-      - open-webui:/app/backend/data
-    env:
-      OLLAMA_BASE_URL: "http://host-gateway:11434"
-    etc_hosts:
-      host-gateway: "172.17.0.1"
-  # Note: Requires docker community collection. Install with:
-  # ansible-galaxy collection install community.docker
-  ignore_errors: true  # Falls back gracefully if docker collection not available
-
-- name: Alternative Open WebUI start (if community.docker not available)
+- name: Start Open WebUI via Docker CLI
   shell: |
-    docker run -d \
-      --name open-webui \
-      --restart always \
-      -p 3001:8080 \
-      --add-host=host-gateway:172.17.0.1 \
-      -v open-webui:/app/backend/data \
-      -e OLLAMA_BASE_URL=http://host-gateway:11434 \
-      ghcr.io/open-webui/open-webui:main
+    if docker ps -a --format '{{ "{{" }}.Names{{ "}}" }}' | grep -qx open-webui; then
+      docker start open-webui
+    else
+      docker run -d \
+        --name open-webui \
+        --restart always \
+        -p 3001:8080 \
+        --add-host=host-gateway:172.17.0.1 \
+        -v open-webui:/app/backend/data \
+        -e OLLAMA_BASE_URL=http://host-gateway:11434 \
+        ghcr.io/open-webui/open-webui:main
+    fi
   args:
     executable: /bin/bash
   register: webui_result
-  failed_when: webui_result.rc != 0 and 'already in use' not in webui_result.stderr
+  changed_when: webui_result.rc == 0
+  failed_when: webui_result.rc != 0 and 'already in use' not in (webui_result.stderr | default(''))
+  ignore_errors: true

ansible/roles/vllm/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+# vLLM role defaults — overridden per-tier in the tier playbook vars block
+vllm_model: "meta-llama/Meta-Llama-3.1-8B-Instruct"
+vllm_tensor_parallel: 1
+vllm_gpu_memory_util: "0.70"
+vllm_max_model_len: 8192
+vllm_quantization: ""              # blank = full precision; set to "awq" for 4-bit

ansible/roles/vllm/tasks/main.yml

@@ -1,6 +1,12 @@
 ---
 # vLLM — high-performance LLM inference with OpenAI-compatible API
 # Skipped automatically if no GPU is present.
+# Variables (set defaults in defaults/main.yml, override per-tier in the playbook):
+#   vllm_model              HuggingFace model ID to load on start
+#   vllm_tensor_parallel    Number of GPUs for tensor parallelism (1 for Starter/Basic)
+#   vllm_gpu_memory_util    Fraction of VRAM to reserve for vLLM (0.0–1.0)
+#   vllm_max_model_len      Maximum context length in tokens
+#   vllm_quantization       Quantization method: "" (none) | "awq" | "gptq" | "fp8"
 
 - name: Check for NVIDIA GPU
   shell: nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1
@@ -10,7 +16,9 @@
 
 - name: Skip vLLM if no GPU detected
   debug:
-    msg: "No GPU detected — skipping vLLM image pull. Run manually when GPU is available: docker pull vllm/vllm-openai:latest"
+    msg: >
+      No GPU detected — skipping vLLM image pull.
+      Run manually when GPU is available: docker pull vllm/vllm-openai:latest
   when: gpu_check.stdout == "" or gpu_check.rc != 0
 
 - name: Pull vLLM Docker image
@@ -19,18 +27,23 @@
   delay: 15
   when: gpu_check.stdout != "" and gpu_check.rc == 0
 
+- name: Build vLLM quantization flag
+  set_fact:
+    vllm_quant_flag: "{{ '--quantization ' + vllm_quantization if vllm_quantization != '' else '' }}"
+
 - name: Create vLLM systemd service
   copy:
     dest: /etc/systemd/system/vllm.service
+    mode: "0644"
     content: |
       [Unit]
-      Description=vLLM OpenAI-Compatible Inference Server
-      After=docker.service ollama.service
+      Description=vLLM OpenAI-Compatible Inference Server ({{ vllm_model }})
+      After=docker.service
       Requires=docker.service
 
       [Service]
       Restart=always
-      RestartSec=5
+      RestartSec=10
       ExecStartPre=-/usr/bin/docker stop vllm
       ExecStartPre=-/usr/bin/docker rm vllm
       ExecStart=/usr/bin/docker run \
@@ -41,15 +54,16 @@
         -v /opt/cezen/models:/root/.cache/huggingface \
         -e HF_HOME=/root/.cache/huggingface \
         vllm/vllm-openai:latest \
-        --model meta-llama/Meta-Llama-3.1-8B-Instruct \
-        --gpu-memory-utilization 0.7 \
-        --max-model-len 8192 \
-        --tensor-parallel-size 1
+        --model {{ vllm_model }} \
+        --gpu-memory-utilization {{ vllm_gpu_memory_util }} \
+        --max-model-len {{ vllm_max_model_len }} \
+        --tensor-parallel-size {{ vllm_tensor_parallel }} \
+        {{ vllm_quant_flag }}
       ExecStop=/usr/bin/docker stop vllm
+      TimeoutStartSec=300
 
       [Install]
       WantedBy=multi-user.target
-    mode: "0644"
 
 - name: Create vLLM model directory
   file:
@@ -57,3 +71,26 @@
     state: directory
     owner: cezen
     group: cezen
+    mode: "0755"
+
+- name: Write vLLM tier config file (for portal reference)
+  copy:
+    dest: /opt/cezen/vllm-config.json
+    owner: cezen
+    group: cezen
+    mode: "0644"
+    content: |
+      {
+        "model": "{{ vllm_model }}",
+        "tensor_parallel_size": {{ vllm_tensor_parallel }},
+        "gpu_memory_utilization": {{ vllm_gpu_memory_util }},
+        "max_model_len": {{ vllm_max_model_len }},
+        "quantization": "{{ vllm_quantization }}"
+      }
+
+- name: Enable and start vLLM service
+  systemd:
+    name: vllm
+    enabled: true
+    daemon_reload: true
+  when: gpu_check.stdout != "" and gpu_check.rc == 0

ansible/starter.yml

@@ -0,0 +1,76 @@
+---
+# Nexus One AI — Starter Tier Stack
+# Hardware: 1× NVIDIA RTX 5090 (32 GB GDDR7), 64 GB DDR5, 2 TB NVMe, 2.5 GbE
+# Capacity: 1–5 concurrent users
+# Runs after NVIDIA driver reboot (phase1_nvidia.yml)
+#
+# Differences from Basic tier:
+#   - No k3s (Kubernetes overhead not justified for 64 GB RAM / 1-5 users)
+#   - No MLflow (fine-tuning tracking overkill for Starter)
+#   - No MinIO (local model cache is sufficient)
+#   - vLLM uses Phi-3 Mini / Llama 3.2 3B with 4-bit quant by default
+#   - JupyterLab is optional (off by default, wizard can enable)
+
+- name: Nexus One AI — Starter Tier Stack
+  hosts: localhost
+  connection: local
+  become: true
+  vars:
+    cezen_user: "cezen"
+    cezen_home: "/opt/cezen"
+    cezen_login_home: "/home/cezen"
+    python_version: "3.11"
+    cuda_version: "12.6"          # RTX 5090 requires CUDA 12.6+
+    skip_roles: ""                # comma-separated list of roles to skip
+    gpu_available: false
+    tier: "starter"
+
+    # ── vLLM — Starter defaults ──────────────────
+    # Small 4-bit quantised model fits comfortably in 32 GB GDDR7.
+    # Users can swap to Llama-3.2-3B-Instruct via the portal Model Manager.
+    vllm_model: "microsoft/Phi-3-mini-4k-instruct"
+    vllm_tensor_parallel: 1
+    vllm_gpu_memory_util: "0.85"
+    vllm_max_model_len: 4096
+    vllm_quantization: "awq"
+
+    # ── Ollama — lightweight models ───────────────
+    ollama_default_model: "phi3:mini"
+
+  roles:
+    - role: base
+      when: "'base' not in skip_roles.split(',')"
+
+    - role: docker
+      when: "'docker' not in skip_roles.split(',')"
+
+    # k3s intentionally omitted for Starter — insufficient RAM headroom
+
+    - role: ollama
+      when: "'ollama' not in skip_roles.split(',')"
+
+    - role: vllm
+      when: "'vllm' not in skip_roles.split(',')"
+
+    - role: chromadb
+      when: "'chromadb' not in skip_roles.split(',')"
+
+    # mlflow / minio omitted for Starter
+
+    - role: monitoring
+      when: "'monitoring' not in skip_roles.split(',')"
+
+    - role: cezen-backend
+      when: "'cezen-backend' not in skip_roles.split(',')"
+
+    - role: cezen-ttyd
+      when: "'cezen-ttyd' not in skip_roles.split(',')"
+
+    - role: cezen-nginx
+      when: "'cezen-nginx' not in skip_roles.split(',')"
+
+    # JupyterLab — optional, install only when explicitly requested
+    - role: jupyterlab
+      when: >
+        'jupyterlab' not in skip_roles.split(',') and
+        install_jupyterlab | default(false) | bool