diff --git a/ansible/roles/cezen-backend/files/main.py b/ansible/roles/cezen-backend/files/main.py
index f1a22ad..c238201 100644
--- a/ansible/roles/cezen-backend/files/main.py
+++ b/ansible/roles/cezen-backend/files/main.py
@@ -2630,6 +2630,32 @@ async def guardrail_log(limit: int = 100, admin: dict = Depends(admin_only)):
db.close()
return [dict(r) for r in rows]
+@app.get("/api/guardrails/flags")
+async def guardrail_flags(limit: int = 20, admin: dict = Depends(admin_only)):
+ db = get_db()
+ rows = db.execute(
+ """SELECT gl.*, gr.label
+ FROM guardrail_log gl
+ LEFT JOIN guardrail_rules gr ON gr.id=gl.rule_id
+ ORDER BY gl.logged_at DESC LIMIT ?""",
+ (limit,)
+ ).fetchall()
+ db.close()
+ flags = []
+ for r in rows:
+ action = r["action"] or ""
+ severity = "high" if action == "block" else "medium" if action == "redact" else "low"
+ flags.append({
+ "id": r["id"],
+ "severity": severity,
+ "excerpt": r["pattern"],
+ "user": r["username"],
+ "ts": r["logged_at"],
+ "rule": r["label"] or r["pattern"] or r["rule_type"],
+ "action": action,
+ })
+ return {"flags": flags}
+
# ── Model Benchmarking ────────────────────────────────────────────────────────
class BenchmarkRun(BaseModel):
@@ -3294,6 +3320,80 @@ async def delete_doc_job(job_id: int, user: dict = Depends(current_user)):
# ── Multimodal Chat ────────────────────────────────────────────────────────────
+@app.post("/api/chat")
+async def chat_completion(body: dict, user: dict = Depends(current_user)):
+ import urllib.request as _ur
+ model = body.get("model") or "llama3"
+ message = (body.get("message") or body.get("prompt") or "").strip()
+ system_prompt = (body.get("system_prompt") or "").strip()
+ if not message:
+ raise HTTPException(400, "message required")
+
+ messages = []
+ if system_prompt:
+ messages.append({"role": "system", "content": system_prompt})
+ messages.append({"role": "user", "content": message})
+
+ routed = _select_router_model(message, model, {"content_type": "text", "user_role": user.get("role", "")})
+ options = {}
+ if body.get("temperature") is not None:
+ options["temperature"] = float(body.get("temperature"))
+ if body.get("max_tokens") is not None:
+ options["num_predict"] = int(body.get("max_tokens"))
+ elif routed.get("max_tokens"):
+ options["num_predict"] = int(routed["max_tokens"])
+
+ payload = {"model": routed["model"], "messages": messages, "stream": False}
+ if options:
+ payload["options"] = options
+
+ started = datetime.now(timezone.utc)
+ try:
+ req = _ur.Request(
+ f"{OLLAMA_URL}/api/chat",
+ data=json.dumps(payload).encode(),
+ headers={"Content-Type": "application/json"},
+ method="POST"
+ )
+ with _ur.urlopen(req, timeout=300) as resp:
+ data = json.loads(resp.read())
+ response_text = data.get("message", {}).get("content", "")
+ except Exception as e:
+ db = get_db()
+ try:
+ db.execute(
+ "INSERT INTO query_log (user_id,username,model,prompt_tokens,completion_tokens,duration_ms,status,logged_at) VALUES (?,?,?,?,?,?,?,?)",
+ (user["sub"], user["username"], routed["model"], _estimate_tokens(message), 0, 0, "error", utcnow())
+ )
+ db.commit()
+ finally:
+ db.close()
+ raise HTTPException(502, f"Ollama error: {e}")
+
+ duration_ms = int((datetime.now(timezone.utc) - started).total_seconds() * 1000)
+ prompt_tokens = _estimate_tokens(system_prompt + " " + message)
+ completion_tokens = _estimate_tokens(response_text)
+ db = get_db()
+ try:
+ db.execute(
+ "INSERT INTO query_log (user_id,username,model,prompt_tokens,completion_tokens,duration_ms,status,logged_at) VALUES (?,?,?,?,?,?,?,?)",
+ (user["sub"], user["username"], routed["model"], prompt_tokens, completion_tokens, duration_ms, "ok", utcnow())
+ )
+ db.commit()
+ finally:
+ db.close()
+ return {
+ "response": response_text,
+ "model": routed["model"],
+ "route": routed.get("route", "default"),
+ "usage": {
+ "prompt_tokens": prompt_tokens,
+ "completion_tokens": completion_tokens,
+ "total_tokens": prompt_tokens + completion_tokens,
+ },
+ "duration_ms": duration_ms,
+ }
+
@app.post("/api/multimodal/chat")
async def multimodal_chat(body: dict, user: dict = Depends(current_user)):
"""
@@ -4191,6 +4291,73 @@ async def list_schedule_runs(job_id: int, user=Depends(current_user)):
db.close()
return [dict(r) for r in rows]
+@app.get("/api/jobs/failed")
+async def failed_jobs(limit: int = 10, admin: dict = Depends(admin_only)):
+ db = get_db()
+ items = []
+ for r in db.execute(
+ "SELECT id,orig_name,error_msg,created_at,finished_at FROM doc_jobs WHERE status='error' ORDER BY COALESCE(finished_at,created_at) DESC LIMIT ?",
+ (limit,)
+ ).fetchall():
+ items.append({"id": f"doc:{r['id']}", "type": "doc", "name": r["orig_name"], "error": r["error_msg"] or "Document job failed", "ts": r["finished_at"] or r["created_at"]})
+ for r in db.execute(
+ "SELECT id,agent_name,error_msg,created_at,finished_at FROM agent_runs WHERE status='error' ORDER BY COALESCE(finished_at,created_at) DESC LIMIT ?",
+ (limit,)
+ ).fetchall():
+ items.append({"id": f"agent:{r['id']}", "type": "agent", "name": r["agent_name"], "error": r["error_msg"] or "Agent run failed", "ts": r["finished_at"] or r["created_at"]})
+ for r in db.execute(
+ "SELECT id,job_id,job_name,error_msg,started_at,finished_at FROM scheduled_job_runs WHERE status='error' ORDER BY COALESCE(finished_at,started_at) DESC LIMIT ?",
+ (limit,)
+ ).fetchall():
+ items.append({"id": f"sched:{r['job_id']}", "type": "sched", "name": r["job_name"], "error": r["error_msg"] or "Scheduled job failed", "ts": r["finished_at"] or r["started_at"]})
+ for r in db.execute(
+ "SELECT id,name,status,created_at,finished_at FROM training_jobs WHERE status='error' ORDER BY COALESCE(finished_at,created_at) DESC LIMIT ?",
+ (limit,)
+ ).fetchall():
+ items.append({"id": f"training:{r['id']}", "type": "train", "name": r["name"], "error": "Training job failed", "ts": r["finished_at"] or r["created_at"]})
+ db.close()
+ items.sort(key=lambda x: x.get("ts") or "", reverse=True)
+ return {"jobs": items[:limit]}
+
+@app.post("/api/jobs/{job_key:path}/retry")
+async def retry_failed_job(job_key: str, admin: dict = Depends(admin_only)):
+ if ":" not in job_key:
+ raise HTTPException(400, "job id must include type prefix")
+ kind, raw_id = job_key.split(":", 1)
+ try:
+ jid = int(raw_id)
+ except ValueError:
+ raise HTTPException(400, "invalid job id")
+
+ db = get_db()
+ try:
+ if kind == "doc":
+ row = db.execute("SELECT id FROM doc_jobs WHERE id=?", (jid,)).fetchone()
+ if not row:
+ raise HTTPException(404, "Document job not found")
+ db.execute("UPDATE doc_jobs SET status='pending', error_msg=NULL, finished_at=NULL WHERE id=?", (jid,))
+ db.commit()
+ asyncio.create_task(_run_doc_job(jid))
+ return {"ok": True, "message": "Document job queued"}
+ if kind == "agent":
+ row = db.execute("SELECT id FROM agent_runs WHERE id=?", (jid,)).fetchone()
+ if not row:
+ raise HTTPException(404, "Agent run not found")
+ db.execute("UPDATE agent_runs SET status='pending', error_msg=NULL, finished_at=NULL WHERE id=?", (jid,))
+ db.commit()
+ asyncio.create_task(_execute_agent(jid))
+ return {"ok": True, "message": "Agent run queued"}
+ if kind == "sched":
+ row = db.execute("SELECT id FROM scheduled_jobs WHERE id=?", (jid,)).fetchone()
+ if not row:
+ raise HTTPException(404, "Scheduled job not found")
+ loop = asyncio.get_event_loop()
+ loop.run_in_executor(None, _run_scheduled_job_sync, jid)
+ return {"ok": True, "message": "Scheduled job triggered"}
+ finally:
+ db.close()
+ raise HTTPException(400, f"Retry not supported for job type: {kind}")
+
# ── Notifications ─────────────────────────────────────────────────────────────
diff --git a/ansible/roles/cezen-nginx/tasks/main.yml b/ansible/roles/cezen-nginx/tasks/main.yml
index be59f70..1fca52d 100644
--- a/ansible/roles/cezen-nginx/tasks/main.yml
+++ b/ansible/roles/cezen-nginx/tasks/main.yml
@@ -3,7 +3,9 @@
- name: Install Nginx
apt:
- name: nginx
+ name:
+ - nginx
+ - rsync
state: present
update_cache: yes
@@ -17,7 +19,7 @@
- name: Sync portal static files
synchronize:
- src: "{{ playbook_dir }}/../../../cezen-portal/"
+ src: "{{ playbook_dir }}/../cezen-portal/"
dest: /opt/cezen/portal/
delete: yes
recursive: yes
diff --git a/cezen-portal/admin.html b/cezen-portal/admin.html
new file mode 100644
index 0000000..4b3d235
--- /dev/null
+++ b/cezen-portal/admin.html
@@ -0,0 +1,621 @@
+
+
+
+
+
+ Admin Guide — Nexus One AI Portal
+
+
+
+
+
+ Nexus One AI
+
+ 🔔
+ Basic Tier
+
+
+
+
+
Admin Guide
+
System Administration Reference
+
For the person managing the Nexus One AI server. Commands to restart services, manage users, add models, check health, and pull logs — all in one place.
+
+
+
+
+
+ 🔐 This page is for system administrators only. All commands below require SSH access to the server. If you're an end user looking for help, visit Troubleshooting or FAQ instead.
+
Notebook server — restrict to developer users only
+
+
+
FastAPI
+
8080
+
http://ai.local:8080
+
Custom API layer — if deployed
+
+
+
+
+
+
+
Model management
+
+
+
+
+
List installed models
+
Shows all models currently available on the system with their sizes.
+
+
+
ollama list
+
+
+
+
+
+
+
Pull (install) a new model
+
Downloads and installs a model. Examples: llama3.1:8b, mistral:7b, gemma2:9b. Requires the model files to be available on your network or internet (if connected).
+
+
+
ollama pull MODEL_NAME
+
+
+
+
+
+
+
Remove a model
+
Deletes a model from disk to free up space. Make sure no users are actively using it.
+
+
+
ollama rm MODEL_NAME
+
+
+
+
+
+
+
Show model info
+
Displays the model's parameters, context length, and configuration.
+
+
+
ollama show MODEL_NAME
+
+
+
+
+
+
+
Check what's currently loaded in GPU
+
Shows running Ollama processes and their VRAM usage.
+
+
+
ollama ps
+
+
+
+
+
+
+
Model storage location
+
All model files are stored here. Monitor this directory for disk usage.
+
+
+
ls -lh /usr/share/ollama/.ollama/models/
+
+
+
+
+
+
+
+
User management (Open WebUI)
+
User accounts are managed through the Open WebUI admin panel at http://ai.local:3001/admin/users. Log in with your admin account and use the panel for most user tasks. The commands below are for bulk operations or when the UI is unavailable.
+
+
+
+
+
+
Open user admin panel
+
The primary way to create, disable, and manage user accounts. Open this URL in a browser while connected to the internal network.
+
+
+
http://ai.local:3001/admin/users
+
+
+
+
+
+
+
Open WebUI database location
+
User data is stored in a SQLite database at this path. Back this up regularly.
+
+
+
/opt/open-webui/data/webui.db
+
+
+
+
+
+
+
Back up Nexus One AI data
+
Creates a local archive of Cezen data, settings, audit logs, and uploaded working files.
Models are added via the Ollama CLI. On an air-gapped system, copy the model file to the server first.
+
+
Check available disk space: df -h / — ensure at least 2× the model size is free
+
Pull the model (if internet-connected): ollama pull llama3.1:8b
+
For air-gapped install, copy the GGUF file then: ollama create mymodel -f /path/to/Modelfile
+
Verify it appears: ollama list
+
Test it loads: ollama run MODEL_NAME "Hello"
+
The model will appear automatically in Open WebUI's model selector — no restart needed
+
+
+
+
+
+
+ 🗑️
+ Freeing up disk space
+
+
+
Disk space is most commonly consumed by model files, ChromaDB data, and Jupyter notebooks. Check each before deleting anything.
+
+
Find what's using the most space: sudo du -sh /opt/* | sort -rh | head -10
+
List models with sizes: ollama list
+
Remove unused models (confirm with users first): ollama rm MODEL_NAME
+
Clear old journal logs (keeps last 7 days): sudo journalctl --vacuum-time=7d
+
Verify space recovered: df -h /
+
+
+
+
+
+
+ 🔑
+ Resetting an admin password (Open WebUI)
+
+
+
If the admin password is lost, reset it via the Open WebUI CLI while the service is stopped.
+
+
Stop Open WebUI: sudo systemctl stop open-webui
+
Run the reset command: sudo open-webui reset-admin-password --email admin@example.com --password newpassword
+
Restart Open WebUI: sudo systemctl start open-webui
+
Log in with the new password and immediately change it to something secure
+
+
+
+
+
+
+
📞
+
Need Cezen support?
+
For hardware issues, system-level failures, or anything beyond this guide, contact Cezen. Include: the service name, the error from journalctl, and the output of nvidia-smi and df -h.
+ Keys can also be verified at /api/apikeys/verify — useful for custom middleware or integrations that need to validate a key before proxying requests to Ollama or Open WebUI.
+
+
+
+
+
+
+
+
+
+
+
Generate API Key
+
+
+
+
+
+
+
+
+
+
+
+
🔑 Your New API Key
+
This key will not be shown again. Copy it now and store it securely.