Complete backend production integrations

This commit is contained in:
Jino Jose committed 2026-06-30 09:09:43 +05:30
1 parent 56668f7bdc
commit 492a4b191a
4 files changed
+667 -29

No files matched your search

+376 -29
View File
@@ -798,6 +798,19 @@ def audit(db, user_id, username, action, detail="", ip="", result="success"):
(utcnow(), user_id, username, action, detail, ip, result)
)
def _audit(db, username, action, detail="", ip="", result="success", user_id=None):
"""Audit helper for API sections that do not already hold a DB connection."""
own_db = db is None
if own_db:
db = get_db()
try:
audit(db, user_id, username, action, detail, ip, result)
if own_db:
db.commit()
finally:
if own_db:
db.close()
def _setting_value(key: str, default: str = "") -> str:
db = get_db()
try:
@@ -1023,8 +1036,8 @@ async def change_password(body: ChangePasswordRequest, request: Request, user: d
# ── User management (admin) ───────────────────────────────────────────────────
@app.get("/api/users/sessions")
async def active_sessions(admin: dict = Depends(admin_only)):
@app.get("/api/users/recent-logins")
async def recent_logins(admin: dict = Depends(admin_only)):
"""Return list of users who have logged in within the last 8 hours (token lifetime)."""
db = get_db()
# Use the same isoformat() style that utcnow() stores (includes +00:00 suffix)
@@ -1960,7 +1973,7 @@ async def upload_document(cid: int, file: UploadFile = File(...), admin: dict =
# Launch ingest subprocess
runner = Path(__file__).parent / "rag_ingest.py"
subprocess.Popen(
["python3", str(runner),
[sys.executable, str(runner),
"--doc-id", str(doc_id),
"--db-path", str(DB_PATH),
"--file", str(dest),
@@ -2800,7 +2813,8 @@ AGENT_STEP_TYPES = {"prompt", "summarise", "extract", "classify", "rag", "format
def _call_ollama_sync(model: str, prompt: str) -> str:
import urllib.request as _ur
payload = json.dumps({"model": model, "prompt": prompt, "stream": False}).encode()
routed = _select_router_model(prompt, model, {"content_type": "text"})
payload = json.dumps({"model": routed["model"], "prompt": prompt, "stream": False}).encode()
req = _ur.Request(
f"{OLLAMA_URL}/api/generate",
data=payload,
@@ -2839,12 +2853,13 @@ async def _execute_agent(run_id: int):
try:
run_row = db.execute("SELECT * FROM agent_runs WHERE id=?", (run_id,)).fetchone()
if not run_row:
return
return {"status": "error", "output": "", "error": "Agent run not found"}
agent_row = db.execute("SELECT * FROM agents WHERE id=?", (run_row["agent_id"],)).fetchone()
if not agent_row:
db.execute("UPDATE agent_runs SET status='error',error_msg='Agent not found',finished_at=? WHERE id=?",
(utcnow(), run_id))
db.commit(); db.close(); return
db.commit()
return {"status": "error", "output": "", "error": "Agent not found"}
steps = json.loads(agent_row["steps"])
pipeline_input = run_row["input"]
@@ -2903,18 +2918,21 @@ async def _execute_agent(run_id: int):
"UPDATE agent_runs SET status='error',error_msg=?,steps_log=?,output=?,finished_at=? WHERE id=?",
(str(e), json.dumps(steps_log), prev_output, utcnow(), run_id)
)
db.commit(); db.close(); return
db.commit()
return {"status": "error", "output": prev_output, "error": str(e)}
db.execute(
"UPDATE agent_runs SET status='done',output=?,steps_log=?,finished_at=? WHERE id=?",
(prev_output, json.dumps(steps_log), utcnow(), run_id)
)
db.commit()
return {"status": "done", "output": prev_output, "error": None}
except Exception as e:
db.execute("UPDATE agent_runs SET status='error',error_msg=?,finished_at=? WHERE id=?",
(str(e), utcnow(), run_id))
db.commit()
return {"status": "error", "output": "", "error": str(e)}
finally:
db.close()
@@ -3669,8 +3687,9 @@ Respond with ONLY valid JSON, no markdown, no explanation:
def _call_model_sync(model: str, prompt_text: str) -> str:
"""Call an Ollama model and return its text response."""
import urllib.request, json as _json
routed = _select_router_model(prompt_text, model, {"content_type": "text"})
payload = {
"model": model,
"model": routed["model"],
"messages": [{"role": "user", "content": prompt_text}],
"stream": False,
}
@@ -3933,7 +3952,8 @@ def _run_scheduled_job_sync(job_id: int):
try:
if job["job_type"] == "prompt":
# Build messages and call Ollama
model = job["model"] or "llama3"
routed = _select_router_model(job["prompt_text"], job["model"] or "llama3", {"content_type": "text"})
model = routed["model"]
payload = {
"model": model,
"messages": [{"role": "user", "content": job["prompt_text"]}],
@@ -3954,15 +3974,20 @@ def _run_scheduled_job_sync(job_id: int):
agent = db.execute("SELECT * FROM agents WHERE id=?", (job["agent_id"],)).fetchone()
if not agent:
raise ValueError("Agent not found")
steps = json.loads(agent["steps"])
input_text = job["prompt_text"] or f"Scheduled run of agent: {agent['name']}"
# Reuse the existing agent execution helper
agent_run_id = db.execute(
"INSERT INTO agent_runs (agent_id,agent_name,user_id,username,input,status,created_at) VALUES (?,?,?,?,?,'pending',?)",
(agent["id"], agent["name"], job["user_id"], job["username"], input_text, utcnow())
).lastrowid
db.commit()
loop = _asyncio.new_event_loop()
try:
result = loop.run_until_complete(_execute_agent(steps, input_text, agent))
result = loop.run_until_complete(_execute_agent(agent_run_id))
finally:
loop.close()
output = result.get("output", "")
if result.get("status") == "error":
raise ValueError(result.get("error") or "Agent run failed")
except Exception as e:
error = str(e)
@@ -4475,12 +4500,66 @@ def _connector_from_row(row) -> dict:
"stats": _json_load(d["stats_json"], {"files": 0, "rowsRead": 0, "lastSync": None, "errors": 0}),
}
def _estimate_tokens(text: str) -> int:
return max(1, round(len((text or "").split()) * 1.3))
def _select_router_model(prompt: str, default_model: str, context: Optional[dict] = None) -> dict:
"""Apply enabled model-router rules to backend model calls."""
ctx = context or {}
token_est = _estimate_tokens(prompt)
content_type = ctx.get("content_type") or "text"
user_role = ctx.get("user_role") or ""
db = get_db()
try:
rows = db.execute(
"SELECT * FROM router_rules WHERE enabled=1 ORDER BY priority, id"
).fetchall()
finally:
db.close()
for row in rows:
conditions = _json_load(row["conditions_json"], [])
matched = True
for cond in conditions:
ctype = (cond.get("type") or "").strip()
value = str(cond.get("value") or "").strip()
if ctype == "keyword":
keywords = [k.strip().lower() for k in value.split(",") if k.strip()]
matched = any(k in (prompt or "").lower() for k in keywords)
elif ctype == "content_type":
matched = value == content_type
elif ctype == "token_gt":
try:
matched = token_est > int(value or 0)
except ValueError:
matched = False
elif ctype == "token_lt":
try:
matched = token_est < int(value or 0)
except ValueError:
matched = False
elif ctype == "user_role":
matched = value == user_role
else:
matched = False
if not matched:
break
if matched and row["model"]:
return {
"model": row["model"],
"route": row["name"],
"max_tokens": row["max_tokens"],
"temperature": row["temperature"],
}
return {"model": default_model or "llama3", "route": "default", "max_tokens": None, "temperature": None}
def _call_ollama_generate(model: str, prompt: str, max_tokens: int = 512) -> str:
routed = _select_router_model(prompt, model, {"content_type": "text"})
payload = json.dumps({
"model": model or "llama3",
"model": routed["model"],
"prompt": prompt,
"stream": False,
"options": {"num_predict": max_tokens}
"options": {"num_predict": int(routed.get("max_tokens") or max_tokens)}
}).encode()
req = urllib.request.Request(f"{OLLAMA_URL}/api/generate", data=payload, method="POST")
req.add_header("Content-Type", "application/json")
@@ -4500,6 +4579,185 @@ def _summarise_locally(text: str) -> str:
preview = " ".join(words[:80])
return f"Summary generated from {len(words)} words. Key discussion: {preview}{'...' if len(words) > 80 else ''}"
def _transcribe_meeting_audio(raw: bytes, suffix: str) -> str:
try:
from faster_whisper import WhisperModel
except ImportError as exc:
raise HTTPException(
status_code=501,
detail="Audio transcription requires faster-whisper. Install backend requirements and ensure ffmpeg is available.",
) from exc
model_size = os.environ.get("CEZEN_WHISPER_MODEL", "base")
device = os.environ.get("CEZEN_WHISPER_DEVICE", "cpu")
compute_type = os.environ.get("CEZEN_WHISPER_COMPUTE_TYPE", "int8")
cache_key = (model_size, device, compute_type)
if getattr(_transcribe_meeting_audio, "_cache_key", None) != cache_key:
_transcribe_meeting_audio._model = WhisperModel(model_size, device=device, compute_type=compute_type)
_transcribe_meeting_audio._cache_key = cache_key
with tempfile.NamedTemporaryFile(suffix=suffix or ".audio", delete=False) as tmp:
tmp.write(raw)
tmp_path = tmp.name
try:
segments, _info = _transcribe_meeting_audio._model.transcribe(tmp_path, vad_filter=True)
transcript = "\n".join(seg.text.strip() for seg in segments if seg.text.strip())
finally:
try:
Path(tmp_path).unlink(missing_ok=True)
except Exception:
pass
if not transcript:
raise HTTPException(status_code=422, detail="No speech could be transcribed from this file")
return transcript
class WorkflowStop(Exception):
def __init__(self, message: str):
super().__init__(message)
self.message = message
def _workflow_condition_true(condition: str, values: dict) -> bool:
expr = _template(condition or "", values).strip()
if not expr:
return True
if "==" in expr:
left, right = expr.split("==", 1)
return left.strip().strip("'\"") == right.strip().strip("'\"")
if "!=" in expr:
left, right = expr.split("!=", 1)
return left.strip().strip("'\"") != right.strip().strip("'\"")
for op in (">=", "<=", ">", "<"):
if op in expr:
left, right = expr.split(op, 1)
try:
a = float(left.strip())
b = float(right.strip())
except ValueError:
return False
return {">=": a >= b, "<=": a <= b, ">": a > b, "<": a < b}[op]
return expr.lower() in ("true", "yes", "1", "continue")
def _workflow_rag_search(cfg: dict, values: dict) -> str:
query = _template(cfg.get("query") or values.get("input") or "", values).strip()
collection = cfg.get("collection") or cfg.get("collection_id")
if not query:
raise ValueError("RAG search requires input or query")
db = get_db()
try:
if str(collection).isdigit():
col = db.execute("SELECT * FROM kb_collections WHERE id=?", (int(collection),)).fetchone()
else:
col = db.execute("SELECT * FROM kb_collections WHERE name=? OR chroma_name=?", (str(collection), str(collection))).fetchone()
finally:
db.close()
if not col:
raise ValueError(f"Knowledge collection not found: {collection or '(blank)'}")
embed = _ollama_embed([query], model=col["embed_model"])
result = _chroma_req("POST", f"/api/v1/collections/{col['chroma_name']}/query", body={
"query_embeddings": embed,
"n_results": int(cfg.get("n_results") or 5),
"include": ["documents", "metadatas", "distances"],
})
docs = result.get("documents", [[]])[0]
metas = result.get("metadatas", [[]])[0]
items = []
for text, meta in zip(docs, metas):
source = meta.get("source") or "knowledge base"
page = f" p.{meta.get('page')}" if meta.get("page") else ""
items.append(f"[{source}{page}] {text}")
return "\n\n".join(items) if items else "No matching knowledge passages found."
def _ensure_kb_collection(name: str) -> sqlite3.Row:
db = get_db()
try:
row = db.execute("SELECT * FROM kb_collections WHERE name=? OR chroma_name=?", (name, name)).fetchone()
if row:
return row
import re as _re
chroma_name = "cezen-" + _re.sub(r"[^a-z0-9-]", "-", name.lower())[:55]
try:
_chroma_req("POST", "/api/v1/collections", body={"name": chroma_name, "metadata": {"hnsw:space": "cosine"}})
except HTTPException as e:
if e.status_code not in (409,):
raise
cur = db.execute(
"INSERT INTO kb_collections (name, description, chroma_name, embed_model, created_at) VALUES (?,?,?,?,?)",
(name, "Created by workflow automation", chroma_name, OLLAMA_EMBED, utcnow())
)
db.commit()
return db.execute("SELECT * FROM kb_collections WHERE id=?", (cur.lastrowid,)).fetchone()
finally:
db.close()
def _workflow_save_kb(cfg: dict, values: dict) -> str:
collection_name = cfg.get("collection") or "Workflow Outputs"
text = _template(cfg.get("text") or cfg.get("body") or values.get("input") or values.get("step1.output") or "", values).strip()
if not text:
raise ValueError("Save to KB requires text/body/input")
col = _ensure_kb_collection(collection_name)
col_dir = KB_DOCS_DIR / str(col["id"])
col_dir.mkdir(exist_ok=True)
dest = col_dir / f"workflow-{uuid.uuid4().hex}.txt"
dest.write_text(text, encoding="utf-8")
db = get_db()
try:
doc_id = db.execute(
"INSERT INTO kb_documents (collection_id, orig_name, file_path, size_bytes, status, uploaded_at) VALUES (?,?,?,?,?,?)",
(col["id"], dest.name, str(dest), dest.stat().st_size, "pending", utcnow())
).lastrowid
db.commit()
finally:
db.close()
runner = Path(__file__).parent / "rag_ingest.py"
subprocess.Popen(
[sys.executable, str(runner), "--doc-id", str(doc_id), "--db-path", str(DB_PATH), "--file", str(dest),
"--collection", col["chroma_name"], "--embed-model", col["embed_model"], "--chroma-url", CHROMA_URL,
"--ollama-url", OLLAMA_URL],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True,
)
return f"Saved document {doc_id} to knowledge collection {col['name']} and started ingestion."
def _workflow_send_email(cfg: dict, values: dict) -> str:
import smtplib
from email.message import EmailMessage
host = os.environ.get("CEZEN_SMTP_HOST")
if not host:
raise RuntimeError("SMTP is not configured. Set CEZEN_SMTP_HOST and related CEZEN_SMTP_* variables.")
port = int(os.environ.get("CEZEN_SMTP_PORT", "587"))
sender = os.environ.get("CEZEN_SMTP_FROM") or os.environ.get("CEZEN_SMTP_USER")
recipient = _template(cfg.get("to") or "", values)
if not sender or not recipient:
raise RuntimeError("Email step requires sender and recipient")
msg = EmailMessage()
msg["From"] = sender
msg["To"] = recipient
msg["Subject"] = _template(cfg.get("subject") or "Nexus One AI workflow output", values)
msg.set_content(_template(cfg.get("body") or cfg.get("bodyTemplate") or values.get("input") or "", values))
with smtplib.SMTP(host, port, timeout=30) as smtp:
if os.environ.get("CEZEN_SMTP_TLS", "true").lower() in ("1", "true", "yes"):
smtp.starttls()
if os.environ.get("CEZEN_SMTP_USER"):
smtp.login(os.environ["CEZEN_SMTP_USER"], os.environ.get("CEZEN_SMTP_PASSWORD", ""))
smtp.send_message(msg)
return f"Email sent to {recipient}."
def _workflow_http_request(cfg: dict, values: dict) -> str:
import urllib.parse
url = _template(cfg.get("url") or "", values).replace("{{_API}}", "http://localhost:8080")
parsed = urllib.parse.urlparse(url)
if parsed.scheme not in ("http", "https"):
raise ValueError("HTTP step requires an http:// or https:// URL")
method = (cfg.get("method") or "POST").upper()
headers = cfg.get("headers") or {}
if isinstance(headers, str):
headers = _json_load(headers, {})
body_text = _template(cfg.get("bodyTemplate") or cfg.get("body") or "", values)
data = body_text.encode() if method not in ("GET", "HEAD") and body_text else None
req = urllib.request.Request(url, data=data, headers=headers, method=method)
if data and "Content-Type" not in headers:
req.add_header("Content-Type", "application/json")
with urllib.request.urlopen(req, timeout=int(cfg.get("timeout") or 60)) as resp:
payload = resp.read(512 * 1024).decode(errors="replace")
return f"HTTP {method} {url} -> {resp.status}\n{payload}"
def _meeting_result_from_text(transcript: str, meta: dict, opts: dict) -> dict:
prompt = (
"Analyse this meeting transcript and return concise meeting output with sections: "
@@ -4568,15 +4826,19 @@ def _workflow_step_output(step: dict, values: dict) -> str:
if stype == "summarise":
return _summarise_locally(values.get("input", ""))
if stype == "rag_search":
return f"Knowledge search prepared for collection {cfg.get('collection') or 'default'}."
return _workflow_rag_search(cfg, values)
if stype == "save_kb":
return f"Queued save into knowledge collection {cfg.get('collection') or 'default'}."
return _workflow_save_kb(cfg, values)
if stype == "filter":
return "Condition evaluated as true."
matched = _workflow_condition_true(cfg.get("condition") or "", values)
action = cfg.get("trueAction" if matched else "falseAction") or "continue"
if action == "stop":
raise WorkflowStop(f"Condition evaluated as {str(matched).lower()}; workflow stopped.")
return f"Condition evaluated as {str(matched).lower()}; continuing workflow."
if stype == "email":
return f"Email queued for {cfg.get('to') or 'recipient'}."
return _workflow_send_email(cfg, values)
if stype == "http":
return f"{cfg.get('method', 'POST')} request prepared for {cfg.get('url') or 'endpoint'}."
return _workflow_http_request(cfg, values)
return "Step completed."
@app.get("/api/models/list")
@@ -4650,7 +4912,11 @@ async def run_workflow_api(workflow_id: str, body: dict, user: dict = Depends(cu
outputs = {}
try:
for idx, step in enumerate(wf.get("steps") or [], start=1):
out = _workflow_step_output(step, values)
try:
out = _workflow_step_output(step, values)
except WorkflowStop as stop:
log.append({"step": step.get("name") or step.get("type") or f"Step {idx}", "status": "stopped", "detail": stop.message})
break
key = (step.get("config") or {}).get("outputVar") or f"step{idx}.output"
values[key] = out
values[f"step{idx}.output"] = out
@@ -4711,6 +4977,86 @@ async def save_connectors(body: list[dict], user: dict = Depends(current_user)):
db.commit(); db.close()
return {"ok": True}
def _connector_db_stats(cfg: dict) -> tuple[int, int]:
db_type = (cfg.get("dbType") or cfg.get("engine") or cfg.get("type") or "sqlite").lower()
tables = [t.strip() for t in (cfg.get("tables") or "").split(",") if t.strip()]
if db_type == "sqlite":
path = cfg.get("path") or cfg.get("database")
if not path:
raise ValueError("SQLite connector requires path or database")
conn = sqlite3.connect(path)
try:
if not tables:
tables = [r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").fetchall()]
rows = 0
for table in tables:
if not table.replace("_", "").replace("-", "").isalnum():
raise ValueError(f"Unsafe table name: {table}")
rows += int(conn.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0])
return len(tables), rows
finally:
conn.close()
if db_type in ("postgres", "postgresql"):
try:
import psycopg2
except ImportError as exc:
raise RuntimeError("PostgreSQL connector requires psycopg2-binary") from exc
conn = psycopg2.connect(
host=cfg.get("host") or "localhost",
port=int(cfg.get("port") or 5432),
dbname=cfg.get("database"),
user=cfg.get("username") or cfg.get("user"),
password=cfg.get("password") or "",
connect_timeout=int(cfg.get("timeout") or 10),
)
try:
with conn.cursor() as cur:
if not tables:
cur.execute("SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE'")
tables = [r[0] for r in cur.fetchall()]
rows = 0
for table in tables:
cur.execute("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='public' AND table_name=%s", (table,))
if cur.fetchone()[0] != 1:
raise ValueError(f"Table not found: {table}")
safe_table = table.replace('"', '""')
cur.execute(f'SELECT COUNT(*) FROM "{safe_table}"')
rows += int(cur.fetchone()[0])
return len(tables), rows
finally:
conn.close()
if db_type in ("mysql", "mariadb"):
try:
import pymysql
except ImportError as exc:
raise RuntimeError("MySQL connector requires pymysql") from exc
conn = pymysql.connect(
host=cfg.get("host") or "localhost",
port=int(cfg.get("port") or 3306),
database=cfg.get("database"),
user=cfg.get("username") or cfg.get("user"),
password=cfg.get("password") or "",
connect_timeout=int(cfg.get("timeout") or 10),
read_timeout=int(cfg.get("timeout") or 10),
)
try:
with conn.cursor() as cur:
if not tables:
cur.execute("SELECT table_name FROM information_schema.tables WHERE table_schema=%s AND table_type='BASE TABLE'", (cfg.get("database"),))
tables = [r[0] for r in cur.fetchall()]
rows = 0
for table in tables:
cur.execute("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=%s AND table_name=%s", (cfg.get("database"), table))
if cur.fetchone()[0] != 1:
raise ValueError(f"Table not found: {table}")
safe_table = table.replace("`", "``")
cur.execute(f"SELECT COUNT(*) FROM `{safe_table}`")
rows += int(cur.fetchone()[0])
return len(tables), rows
finally:
conn.close()
raise ValueError(f"Unsupported database connector type: {db_type}")
@app.post("/api/connectors/{connector_id}/sync")
async def sync_connector(connector_id: str, user: dict = Depends(current_user)):
db = get_db()
@@ -4738,11 +5084,15 @@ async def sync_connector(connector_id: str, user: dict = Depends(current_user)):
level = "warn"; status = "error"; errors = int(stats.get("errors") or 0) + 1
stats.update({"files": files, "lastSync": utcnow(), "errors": errors})
else:
tables = [t.strip() for t in (cfg.get("tables") or "").split(",") if t.strip()]
rows_read = max(int(stats.get("rowsRead") or 0), len(tables) * 100)
stats.update({"rowsRead": rows_read, "lastSync": utcnow(), "errors": 0})
msg = f"[{c['name']}] Read-only schema check complete — {len(tables)} tables configured"
level = "ok"; status = "ok"
try:
table_count, rows_read = _connector_db_stats(cfg)
stats.update({"rowsRead": rows_read, "tables": table_count, "lastSync": utcnow(), "errors": 0})
msg = f"[{c['name']}] Read-only sync complete — {table_count} tables, {rows_read} rows visible"
level = "ok"; status = "ok"
except Exception as e:
stats.update({"lastSync": utcnow(), "errors": int(stats.get("errors") or 0) + 1})
msg = f"[{c['name']}] Database sync failed: {e}"
level = "error"; status = "error"
db.execute("UPDATE connectors SET status=?, stats_json=?, updated_at=? WHERE id=?",
(status, json.dumps(stats), utcnow(), connector_id))
db.execute("INSERT INTO connector_log (connector_id,level,msg,ts) VALUES (?,?,?,?)",
@@ -4903,10 +5253,7 @@ async def process_meeting(file: UploadFile = File(...), meta: str = Form("{}"),
if suffix in (".txt", ".md", ".vtt", ".srt"):
text = raw.decode(errors="replace")
if not text:
text = (
f"Uploaded meeting file: {file.filename or 'audio'} ({round(len(raw)/1024/1024, 2)} MB). "
"Audio transcription service is not configured on this node yet, so this record was processed from file metadata."
)
text = _transcribe_meeting_audio(raw, suffix)
result = _meeting_result_from_text(text, meta_obj, meta_obj.get("opts") or {})
db = get_db()
db.execute(