Complete backend production integrations
This commit is contained in:
1 parent
56668f7bdc
commit
492a4b191a
4 files changed
+667
-29
No files matched your search
@@ -798,6 +798,19 @@ def audit(db, user_id, username, action, detail="", ip="", result="success"):
|
||||
(utcnow(), user_id, username, action, detail, ip, result)
|
||||
)
|
||||
|
||||
def _audit(db, username, action, detail="", ip="", result="success", user_id=None):
|
||||
"""Audit helper for API sections that do not already hold a DB connection."""
|
||||
own_db = db is None
|
||||
if own_db:
|
||||
db = get_db()
|
||||
try:
|
||||
audit(db, user_id, username, action, detail, ip, result)
|
||||
if own_db:
|
||||
db.commit()
|
||||
finally:
|
||||
if own_db:
|
||||
db.close()
|
||||
|
||||
def _setting_value(key: str, default: str = "") -> str:
|
||||
db = get_db()
|
||||
try:
|
||||
@@ -1023,8 +1036,8 @@ async def change_password(body: ChangePasswordRequest, request: Request, user: d
|
||||
|
||||
# ── User management (admin) ───────────────────────────────────────────────────
|
||||
|
||||
@app.get("/api/users/sessions")
|
||||
async def active_sessions(admin: dict = Depends(admin_only)):
|
||||
@app.get("/api/users/recent-logins")
|
||||
async def recent_logins(admin: dict = Depends(admin_only)):
|
||||
"""Return list of users who have logged in within the last 8 hours (token lifetime)."""
|
||||
db = get_db()
|
||||
# Use the same isoformat() style that utcnow() stores (includes +00:00 suffix)
|
||||
@@ -1960,7 +1973,7 @@ async def upload_document(cid: int, file: UploadFile = File(...), admin: dict =
|
||||
# Launch ingest subprocess
|
||||
runner = Path(__file__).parent / "rag_ingest.py"
|
||||
subprocess.Popen(
|
||||
["python3", str(runner),
|
||||
[sys.executable, str(runner),
|
||||
"--doc-id", str(doc_id),
|
||||
"--db-path", str(DB_PATH),
|
||||
"--file", str(dest),
|
||||
@@ -2800,7 +2813,8 @@ AGENT_STEP_TYPES = {"prompt", "summarise", "extract", "classify", "rag", "format
|
||||
|
||||
def _call_ollama_sync(model: str, prompt: str) -> str:
|
||||
import urllib.request as _ur
|
||||
payload = json.dumps({"model": model, "prompt": prompt, "stream": False}).encode()
|
||||
routed = _select_router_model(prompt, model, {"content_type": "text"})
|
||||
payload = json.dumps({"model": routed["model"], "prompt": prompt, "stream": False}).encode()
|
||||
req = _ur.Request(
|
||||
f"{OLLAMA_URL}/api/generate",
|
||||
data=payload,
|
||||
@@ -2839,12 +2853,13 @@ async def _execute_agent(run_id: int):
|
||||
try:
|
||||
run_row = db.execute("SELECT * FROM agent_runs WHERE id=?", (run_id,)).fetchone()
|
||||
if not run_row:
|
||||
return
|
||||
return {"status": "error", "output": "", "error": "Agent run not found"}
|
||||
agent_row = db.execute("SELECT * FROM agents WHERE id=?", (run_row["agent_id"],)).fetchone()
|
||||
if not agent_row:
|
||||
db.execute("UPDATE agent_runs SET status='error',error_msg='Agent not found',finished_at=? WHERE id=?",
|
||||
(utcnow(), run_id))
|
||||
db.commit(); db.close(); return
|
||||
db.commit()
|
||||
return {"status": "error", "output": "", "error": "Agent not found"}
|
||||
|
||||
steps = json.loads(agent_row["steps"])
|
||||
pipeline_input = run_row["input"]
|
||||
@@ -2903,18 +2918,21 @@ async def _execute_agent(run_id: int):
|
||||
"UPDATE agent_runs SET status='error',error_msg=?,steps_log=?,output=?,finished_at=? WHERE id=?",
|
||||
(str(e), json.dumps(steps_log), prev_output, utcnow(), run_id)
|
||||
)
|
||||
db.commit(); db.close(); return
|
||||
db.commit()
|
||||
return {"status": "error", "output": prev_output, "error": str(e)}
|
||||
|
||||
db.execute(
|
||||
"UPDATE agent_runs SET status='done',output=?,steps_log=?,finished_at=? WHERE id=?",
|
||||
(prev_output, json.dumps(steps_log), utcnow(), run_id)
|
||||
)
|
||||
db.commit()
|
||||
return {"status": "done", "output": prev_output, "error": None}
|
||||
|
||||
except Exception as e:
|
||||
db.execute("UPDATE agent_runs SET status='error',error_msg=?,finished_at=? WHERE id=?",
|
||||
(str(e), utcnow(), run_id))
|
||||
db.commit()
|
||||
return {"status": "error", "output": "", "error": str(e)}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -3669,8 +3687,9 @@ Respond with ONLY valid JSON, no markdown, no explanation:
|
||||
def _call_model_sync(model: str, prompt_text: str) -> str:
|
||||
"""Call an Ollama model and return its text response."""
|
||||
import urllib.request, json as _json
|
||||
routed = _select_router_model(prompt_text, model, {"content_type": "text"})
|
||||
payload = {
|
||||
"model": model,
|
||||
"model": routed["model"],
|
||||
"messages": [{"role": "user", "content": prompt_text}],
|
||||
"stream": False,
|
||||
}
|
||||
@@ -3933,7 +3952,8 @@ def _run_scheduled_job_sync(job_id: int):
|
||||
try:
|
||||
if job["job_type"] == "prompt":
|
||||
# Build messages and call Ollama
|
||||
model = job["model"] or "llama3"
|
||||
routed = _select_router_model(job["prompt_text"], job["model"] or "llama3", {"content_type": "text"})
|
||||
model = routed["model"]
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": job["prompt_text"]}],
|
||||
@@ -3954,15 +3974,20 @@ def _run_scheduled_job_sync(job_id: int):
|
||||
agent = db.execute("SELECT * FROM agents WHERE id=?", (job["agent_id"],)).fetchone()
|
||||
if not agent:
|
||||
raise ValueError("Agent not found")
|
||||
steps = json.loads(agent["steps"])
|
||||
input_text = job["prompt_text"] or f"Scheduled run of agent: {agent['name']}"
|
||||
# Reuse the existing agent execution helper
|
||||
agent_run_id = db.execute(
|
||||
"INSERT INTO agent_runs (agent_id,agent_name,user_id,username,input,status,created_at) VALUES (?,?,?,?,?,'pending',?)",
|
||||
(agent["id"], agent["name"], job["user_id"], job["username"], input_text, utcnow())
|
||||
).lastrowid
|
||||
db.commit()
|
||||
loop = _asyncio.new_event_loop()
|
||||
try:
|
||||
result = loop.run_until_complete(_execute_agent(steps, input_text, agent))
|
||||
result = loop.run_until_complete(_execute_agent(agent_run_id))
|
||||
finally:
|
||||
loop.close()
|
||||
output = result.get("output", "")
|
||||
if result.get("status") == "error":
|
||||
raise ValueError(result.get("error") or "Agent run failed")
|
||||
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
@@ -4475,12 +4500,66 @@ def _connector_from_row(row) -> dict:
|
||||
"stats": _json_load(d["stats_json"], {"files": 0, "rowsRead": 0, "lastSync": None, "errors": 0}),
|
||||
}
|
||||
|
||||
def _estimate_tokens(text: str) -> int:
|
||||
return max(1, round(len((text or "").split()) * 1.3))
|
||||
|
||||
def _select_router_model(prompt: str, default_model: str, context: Optional[dict] = None) -> dict:
|
||||
"""Apply enabled model-router rules to backend model calls."""
|
||||
ctx = context or {}
|
||||
token_est = _estimate_tokens(prompt)
|
||||
content_type = ctx.get("content_type") or "text"
|
||||
user_role = ctx.get("user_role") or ""
|
||||
db = get_db()
|
||||
try:
|
||||
rows = db.execute(
|
||||
"SELECT * FROM router_rules WHERE enabled=1 ORDER BY priority, id"
|
||||
).fetchall()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
for row in rows:
|
||||
conditions = _json_load(row["conditions_json"], [])
|
||||
matched = True
|
||||
for cond in conditions:
|
||||
ctype = (cond.get("type") or "").strip()
|
||||
value = str(cond.get("value") or "").strip()
|
||||
if ctype == "keyword":
|
||||
keywords = [k.strip().lower() for k in value.split(",") if k.strip()]
|
||||
matched = any(k in (prompt or "").lower() for k in keywords)
|
||||
elif ctype == "content_type":
|
||||
matched = value == content_type
|
||||
elif ctype == "token_gt":
|
||||
try:
|
||||
matched = token_est > int(value or 0)
|
||||
except ValueError:
|
||||
matched = False
|
||||
elif ctype == "token_lt":
|
||||
try:
|
||||
matched = token_est < int(value or 0)
|
||||
except ValueError:
|
||||
matched = False
|
||||
elif ctype == "user_role":
|
||||
matched = value == user_role
|
||||
else:
|
||||
matched = False
|
||||
if not matched:
|
||||
break
|
||||
if matched and row["model"]:
|
||||
return {
|
||||
"model": row["model"],
|
||||
"route": row["name"],
|
||||
"max_tokens": row["max_tokens"],
|
||||
"temperature": row["temperature"],
|
||||
}
|
||||
return {"model": default_model or "llama3", "route": "default", "max_tokens": None, "temperature": None}
|
||||
|
||||
def _call_ollama_generate(model: str, prompt: str, max_tokens: int = 512) -> str:
|
||||
routed = _select_router_model(prompt, model, {"content_type": "text"})
|
||||
payload = json.dumps({
|
||||
"model": model or "llama3",
|
||||
"model": routed["model"],
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {"num_predict": max_tokens}
|
||||
"options": {"num_predict": int(routed.get("max_tokens") or max_tokens)}
|
||||
}).encode()
|
||||
req = urllib.request.Request(f"{OLLAMA_URL}/api/generate", data=payload, method="POST")
|
||||
req.add_header("Content-Type", "application/json")
|
||||
@@ -4500,6 +4579,185 @@ def _summarise_locally(text: str) -> str:
|
||||
preview = " ".join(words[:80])
|
||||
return f"Summary generated from {len(words)} words. Key discussion: {preview}{'...' if len(words) > 80 else ''}"
|
||||
|
||||
def _transcribe_meeting_audio(raw: bytes, suffix: str) -> str:
|
||||
try:
|
||||
from faster_whisper import WhisperModel
|
||||
except ImportError as exc:
|
||||
raise HTTPException(
|
||||
status_code=501,
|
||||
detail="Audio transcription requires faster-whisper. Install backend requirements and ensure ffmpeg is available.",
|
||||
) from exc
|
||||
model_size = os.environ.get("CEZEN_WHISPER_MODEL", "base")
|
||||
device = os.environ.get("CEZEN_WHISPER_DEVICE", "cpu")
|
||||
compute_type = os.environ.get("CEZEN_WHISPER_COMPUTE_TYPE", "int8")
|
||||
cache_key = (model_size, device, compute_type)
|
||||
if getattr(_transcribe_meeting_audio, "_cache_key", None) != cache_key:
|
||||
_transcribe_meeting_audio._model = WhisperModel(model_size, device=device, compute_type=compute_type)
|
||||
_transcribe_meeting_audio._cache_key = cache_key
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix or ".audio", delete=False) as tmp:
|
||||
tmp.write(raw)
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
segments, _info = _transcribe_meeting_audio._model.transcribe(tmp_path, vad_filter=True)
|
||||
transcript = "\n".join(seg.text.strip() for seg in segments if seg.text.strip())
|
||||
finally:
|
||||
try:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
if not transcript:
|
||||
raise HTTPException(status_code=422, detail="No speech could be transcribed from this file")
|
||||
return transcript
|
||||
|
||||
class WorkflowStop(Exception):
|
||||
def __init__(self, message: str):
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
|
||||
def _workflow_condition_true(condition: str, values: dict) -> bool:
|
||||
expr = _template(condition or "", values).strip()
|
||||
if not expr:
|
||||
return True
|
||||
if "==" in expr:
|
||||
left, right = expr.split("==", 1)
|
||||
return left.strip().strip("'\"") == right.strip().strip("'\"")
|
||||
if "!=" in expr:
|
||||
left, right = expr.split("!=", 1)
|
||||
return left.strip().strip("'\"") != right.strip().strip("'\"")
|
||||
for op in (">=", "<=", ">", "<"):
|
||||
if op in expr:
|
||||
left, right = expr.split(op, 1)
|
||||
try:
|
||||
a = float(left.strip())
|
||||
b = float(right.strip())
|
||||
except ValueError:
|
||||
return False
|
||||
return {">=": a >= b, "<=": a <= b, ">": a > b, "<": a < b}[op]
|
||||
return expr.lower() in ("true", "yes", "1", "continue")
|
||||
|
||||
def _workflow_rag_search(cfg: dict, values: dict) -> str:
|
||||
query = _template(cfg.get("query") or values.get("input") or "", values).strip()
|
||||
collection = cfg.get("collection") or cfg.get("collection_id")
|
||||
if not query:
|
||||
raise ValueError("RAG search requires input or query")
|
||||
db = get_db()
|
||||
try:
|
||||
if str(collection).isdigit():
|
||||
col = db.execute("SELECT * FROM kb_collections WHERE id=?", (int(collection),)).fetchone()
|
||||
else:
|
||||
col = db.execute("SELECT * FROM kb_collections WHERE name=? OR chroma_name=?", (str(collection), str(collection))).fetchone()
|
||||
finally:
|
||||
db.close()
|
||||
if not col:
|
||||
raise ValueError(f"Knowledge collection not found: {collection or '(blank)'}")
|
||||
embed = _ollama_embed([query], model=col["embed_model"])
|
||||
result = _chroma_req("POST", f"/api/v1/collections/{col['chroma_name']}/query", body={
|
||||
"query_embeddings": embed,
|
||||
"n_results": int(cfg.get("n_results") or 5),
|
||||
"include": ["documents", "metadatas", "distances"],
|
||||
})
|
||||
docs = result.get("documents", [[]])[0]
|
||||
metas = result.get("metadatas", [[]])[0]
|
||||
items = []
|
||||
for text, meta in zip(docs, metas):
|
||||
source = meta.get("source") or "knowledge base"
|
||||
page = f" p.{meta.get('page')}" if meta.get("page") else ""
|
||||
items.append(f"[{source}{page}] {text}")
|
||||
return "\n\n".join(items) if items else "No matching knowledge passages found."
|
||||
|
||||
def _ensure_kb_collection(name: str) -> sqlite3.Row:
|
||||
db = get_db()
|
||||
try:
|
||||
row = db.execute("SELECT * FROM kb_collections WHERE name=? OR chroma_name=?", (name, name)).fetchone()
|
||||
if row:
|
||||
return row
|
||||
import re as _re
|
||||
chroma_name = "cezen-" + _re.sub(r"[^a-z0-9-]", "-", name.lower())[:55]
|
||||
try:
|
||||
_chroma_req("POST", "/api/v1/collections", body={"name": chroma_name, "metadata": {"hnsw:space": "cosine"}})
|
||||
except HTTPException as e:
|
||||
if e.status_code not in (409,):
|
||||
raise
|
||||
cur = db.execute(
|
||||
"INSERT INTO kb_collections (name, description, chroma_name, embed_model, created_at) VALUES (?,?,?,?,?)",
|
||||
(name, "Created by workflow automation", chroma_name, OLLAMA_EMBED, utcnow())
|
||||
)
|
||||
db.commit()
|
||||
return db.execute("SELECT * FROM kb_collections WHERE id=?", (cur.lastrowid,)).fetchone()
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
def _workflow_save_kb(cfg: dict, values: dict) -> str:
|
||||
collection_name = cfg.get("collection") or "Workflow Outputs"
|
||||
text = _template(cfg.get("text") or cfg.get("body") or values.get("input") or values.get("step1.output") or "", values).strip()
|
||||
if not text:
|
||||
raise ValueError("Save to KB requires text/body/input")
|
||||
col = _ensure_kb_collection(collection_name)
|
||||
col_dir = KB_DOCS_DIR / str(col["id"])
|
||||
col_dir.mkdir(exist_ok=True)
|
||||
dest = col_dir / f"workflow-{uuid.uuid4().hex}.txt"
|
||||
dest.write_text(text, encoding="utf-8")
|
||||
db = get_db()
|
||||
try:
|
||||
doc_id = db.execute(
|
||||
"INSERT INTO kb_documents (collection_id, orig_name, file_path, size_bytes, status, uploaded_at) VALUES (?,?,?,?,?,?)",
|
||||
(col["id"], dest.name, str(dest), dest.stat().st_size, "pending", utcnow())
|
||||
).lastrowid
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
runner = Path(__file__).parent / "rag_ingest.py"
|
||||
subprocess.Popen(
|
||||
[sys.executable, str(runner), "--doc-id", str(doc_id), "--db-path", str(DB_PATH), "--file", str(dest),
|
||||
"--collection", col["chroma_name"], "--embed-model", col["embed_model"], "--chroma-url", CHROMA_URL,
|
||||
"--ollama-url", OLLAMA_URL],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, start_new_session=True,
|
||||
)
|
||||
return f"Saved document {doc_id} to knowledge collection {col['name']} and started ingestion."
|
||||
|
||||
def _workflow_send_email(cfg: dict, values: dict) -> str:
|
||||
import smtplib
|
||||
from email.message import EmailMessage
|
||||
host = os.environ.get("CEZEN_SMTP_HOST")
|
||||
if not host:
|
||||
raise RuntimeError("SMTP is not configured. Set CEZEN_SMTP_HOST and related CEZEN_SMTP_* variables.")
|
||||
port = int(os.environ.get("CEZEN_SMTP_PORT", "587"))
|
||||
sender = os.environ.get("CEZEN_SMTP_FROM") or os.environ.get("CEZEN_SMTP_USER")
|
||||
recipient = _template(cfg.get("to") or "", values)
|
||||
if not sender or not recipient:
|
||||
raise RuntimeError("Email step requires sender and recipient")
|
||||
msg = EmailMessage()
|
||||
msg["From"] = sender
|
||||
msg["To"] = recipient
|
||||
msg["Subject"] = _template(cfg.get("subject") or "Nexus One AI workflow output", values)
|
||||
msg.set_content(_template(cfg.get("body") or cfg.get("bodyTemplate") or values.get("input") or "", values))
|
||||
with smtplib.SMTP(host, port, timeout=30) as smtp:
|
||||
if os.environ.get("CEZEN_SMTP_TLS", "true").lower() in ("1", "true", "yes"):
|
||||
smtp.starttls()
|
||||
if os.environ.get("CEZEN_SMTP_USER"):
|
||||
smtp.login(os.environ["CEZEN_SMTP_USER"], os.environ.get("CEZEN_SMTP_PASSWORD", ""))
|
||||
smtp.send_message(msg)
|
||||
return f"Email sent to {recipient}."
|
||||
|
||||
def _workflow_http_request(cfg: dict, values: dict) -> str:
|
||||
import urllib.parse
|
||||
url = _template(cfg.get("url") or "", values).replace("{{_API}}", "http://localhost:8080")
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
raise ValueError("HTTP step requires an http:// or https:// URL")
|
||||
method = (cfg.get("method") or "POST").upper()
|
||||
headers = cfg.get("headers") or {}
|
||||
if isinstance(headers, str):
|
||||
headers = _json_load(headers, {})
|
||||
body_text = _template(cfg.get("bodyTemplate") or cfg.get("body") or "", values)
|
||||
data = body_text.encode() if method not in ("GET", "HEAD") and body_text else None
|
||||
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
if data and "Content-Type" not in headers:
|
||||
req.add_header("Content-Type", "application/json")
|
||||
with urllib.request.urlopen(req, timeout=int(cfg.get("timeout") or 60)) as resp:
|
||||
payload = resp.read(512 * 1024).decode(errors="replace")
|
||||
return f"HTTP {method} {url} -> {resp.status}\n{payload}"
|
||||
|
||||
def _meeting_result_from_text(transcript: str, meta: dict, opts: dict) -> dict:
|
||||
prompt = (
|
||||
"Analyse this meeting transcript and return concise meeting output with sections: "
|
||||
@@ -4568,15 +4826,19 @@ def _workflow_step_output(step: dict, values: dict) -> str:
|
||||
if stype == "summarise":
|
||||
return _summarise_locally(values.get("input", ""))
|
||||
if stype == "rag_search":
|
||||
return f"Knowledge search prepared for collection {cfg.get('collection') or 'default'}."
|
||||
return _workflow_rag_search(cfg, values)
|
||||
if stype == "save_kb":
|
||||
return f"Queued save into knowledge collection {cfg.get('collection') or 'default'}."
|
||||
return _workflow_save_kb(cfg, values)
|
||||
if stype == "filter":
|
||||
return "Condition evaluated as true."
|
||||
matched = _workflow_condition_true(cfg.get("condition") or "", values)
|
||||
action = cfg.get("trueAction" if matched else "falseAction") or "continue"
|
||||
if action == "stop":
|
||||
raise WorkflowStop(f"Condition evaluated as {str(matched).lower()}; workflow stopped.")
|
||||
return f"Condition evaluated as {str(matched).lower()}; continuing workflow."
|
||||
if stype == "email":
|
||||
return f"Email queued for {cfg.get('to') or 'recipient'}."
|
||||
return _workflow_send_email(cfg, values)
|
||||
if stype == "http":
|
||||
return f"{cfg.get('method', 'POST')} request prepared for {cfg.get('url') or 'endpoint'}."
|
||||
return _workflow_http_request(cfg, values)
|
||||
return "Step completed."
|
||||
|
||||
@app.get("/api/models/list")
|
||||
@@ -4650,7 +4912,11 @@ async def run_workflow_api(workflow_id: str, body: dict, user: dict = Depends(cu
|
||||
outputs = {}
|
||||
try:
|
||||
for idx, step in enumerate(wf.get("steps") or [], start=1):
|
||||
out = _workflow_step_output(step, values)
|
||||
try:
|
||||
out = _workflow_step_output(step, values)
|
||||
except WorkflowStop as stop:
|
||||
log.append({"step": step.get("name") or step.get("type") or f"Step {idx}", "status": "stopped", "detail": stop.message})
|
||||
break
|
||||
key = (step.get("config") or {}).get("outputVar") or f"step{idx}.output"
|
||||
values[key] = out
|
||||
values[f"step{idx}.output"] = out
|
||||
@@ -4711,6 +4977,86 @@ async def save_connectors(body: list[dict], user: dict = Depends(current_user)):
|
||||
db.commit(); db.close()
|
||||
return {"ok": True}
|
||||
|
||||
def _connector_db_stats(cfg: dict) -> tuple[int, int]:
|
||||
db_type = (cfg.get("dbType") or cfg.get("engine") or cfg.get("type") or "sqlite").lower()
|
||||
tables = [t.strip() for t in (cfg.get("tables") or "").split(",") if t.strip()]
|
||||
if db_type == "sqlite":
|
||||
path = cfg.get("path") or cfg.get("database")
|
||||
if not path:
|
||||
raise ValueError("SQLite connector requires path or database")
|
||||
conn = sqlite3.connect(path)
|
||||
try:
|
||||
if not tables:
|
||||
tables = [r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").fetchall()]
|
||||
rows = 0
|
||||
for table in tables:
|
||||
if not table.replace("_", "").replace("-", "").isalnum():
|
||||
raise ValueError(f"Unsafe table name: {table}")
|
||||
rows += int(conn.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0])
|
||||
return len(tables), rows
|
||||
finally:
|
||||
conn.close()
|
||||
if db_type in ("postgres", "postgresql"):
|
||||
try:
|
||||
import psycopg2
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("PostgreSQL connector requires psycopg2-binary") from exc
|
||||
conn = psycopg2.connect(
|
||||
host=cfg.get("host") or "localhost",
|
||||
port=int(cfg.get("port") or 5432),
|
||||
dbname=cfg.get("database"),
|
||||
user=cfg.get("username") or cfg.get("user"),
|
||||
password=cfg.get("password") or "",
|
||||
connect_timeout=int(cfg.get("timeout") or 10),
|
||||
)
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
if not tables:
|
||||
cur.execute("SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE'")
|
||||
tables = [r[0] for r in cur.fetchall()]
|
||||
rows = 0
|
||||
for table in tables:
|
||||
cur.execute("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='public' AND table_name=%s", (table,))
|
||||
if cur.fetchone()[0] != 1:
|
||||
raise ValueError(f"Table not found: {table}")
|
||||
safe_table = table.replace('"', '""')
|
||||
cur.execute(f'SELECT COUNT(*) FROM "{safe_table}"')
|
||||
rows += int(cur.fetchone()[0])
|
||||
return len(tables), rows
|
||||
finally:
|
||||
conn.close()
|
||||
if db_type in ("mysql", "mariadb"):
|
||||
try:
|
||||
import pymysql
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("MySQL connector requires pymysql") from exc
|
||||
conn = pymysql.connect(
|
||||
host=cfg.get("host") or "localhost",
|
||||
port=int(cfg.get("port") or 3306),
|
||||
database=cfg.get("database"),
|
||||
user=cfg.get("username") or cfg.get("user"),
|
||||
password=cfg.get("password") or "",
|
||||
connect_timeout=int(cfg.get("timeout") or 10),
|
||||
read_timeout=int(cfg.get("timeout") or 10),
|
||||
)
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
if not tables:
|
||||
cur.execute("SELECT table_name FROM information_schema.tables WHERE table_schema=%s AND table_type='BASE TABLE'", (cfg.get("database"),))
|
||||
tables = [r[0] for r in cur.fetchall()]
|
||||
rows = 0
|
||||
for table in tables:
|
||||
cur.execute("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=%s AND table_name=%s", (cfg.get("database"), table))
|
||||
if cur.fetchone()[0] != 1:
|
||||
raise ValueError(f"Table not found: {table}")
|
||||
safe_table = table.replace("`", "``")
|
||||
cur.execute(f"SELECT COUNT(*) FROM `{safe_table}`")
|
||||
rows += int(cur.fetchone()[0])
|
||||
return len(tables), rows
|
||||
finally:
|
||||
conn.close()
|
||||
raise ValueError(f"Unsupported database connector type: {db_type}")
|
||||
|
||||
@app.post("/api/connectors/{connector_id}/sync")
|
||||
async def sync_connector(connector_id: str, user: dict = Depends(current_user)):
|
||||
db = get_db()
|
||||
@@ -4738,11 +5084,15 @@ async def sync_connector(connector_id: str, user: dict = Depends(current_user)):
|
||||
level = "warn"; status = "error"; errors = int(stats.get("errors") or 0) + 1
|
||||
stats.update({"files": files, "lastSync": utcnow(), "errors": errors})
|
||||
else:
|
||||
tables = [t.strip() for t in (cfg.get("tables") or "").split(",") if t.strip()]
|
||||
rows_read = max(int(stats.get("rowsRead") or 0), len(tables) * 100)
|
||||
stats.update({"rowsRead": rows_read, "lastSync": utcnow(), "errors": 0})
|
||||
msg = f"[{c['name']}] Read-only schema check complete — {len(tables)} tables configured"
|
||||
level = "ok"; status = "ok"
|
||||
try:
|
||||
table_count, rows_read = _connector_db_stats(cfg)
|
||||
stats.update({"rowsRead": rows_read, "tables": table_count, "lastSync": utcnow(), "errors": 0})
|
||||
msg = f"[{c['name']}] Read-only sync complete — {table_count} tables, {rows_read} rows visible"
|
||||
level = "ok"; status = "ok"
|
||||
except Exception as e:
|
||||
stats.update({"lastSync": utcnow(), "errors": int(stats.get("errors") or 0) + 1})
|
||||
msg = f"[{c['name']}] Database sync failed: {e}"
|
||||
level = "error"; status = "error"
|
||||
db.execute("UPDATE connectors SET status=?, stats_json=?, updated_at=? WHERE id=?",
|
||||
(status, json.dumps(stats), utcnow(), connector_id))
|
||||
db.execute("INSERT INTO connector_log (connector_id,level,msg,ts) VALUES (?,?,?,?)",
|
||||
@@ -4903,10 +5253,7 @@ async def process_meeting(file: UploadFile = File(...), meta: str = Form("{}"),
|
||||
if suffix in (".txt", ".md", ".vtt", ".srt"):
|
||||
text = raw.decode(errors="replace")
|
||||
if not text:
|
||||
text = (
|
||||
f"Uploaded meeting file: {file.filename or 'audio'} ({round(len(raw)/1024/1024, 2)} MB). "
|
||||
"Audio transcription service is not configured on this node yet, so this record was processed from file metadata."
|
||||
)
|
||||
text = _transcribe_meeting_audio(raw, suffix)
|
||||
result = _meeting_result_from_text(text, meta_obj, meta_obj.get("opts") or {})
|
||||
db = get_db()
|
||||
db.execute(
|
||||
|
||||
Reference in new issue
Block a user