jinojose/aipackage
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

release: freeze Nexus One AI 1.0.0-rc1

Browse Source
This commit is contained in:
Jino Jose committed 2026-08-17 11:05:19 +05:30
1 parent 63acefd890
commit 4f1419ca93
152 files changed
+9445 -828
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

ansible/roles/cezen-backend/files/main.py
+304 -19
View File
@@ -67,6 +67,7 @@ JWT_ALGORITHM = "HS256"
JWT_EXPIRE_HRS = 8
COOKIE_NAME = "cezen_token"
OLLAMA_URL = os.environ.get("OLLAMA_URL", "http://localhost:11434")
PROTOTYPE_LICENSE_BYPASS = os.environ.get("CEZEN_PROTOTYPE_BYPASS_LICENSE", "").strip().lower() in {"1", "true", "yes"}
SCHEMA_VERSION = 1
CUSTOMER_STATES = {"loading", "empty", "ready", "degraded", "failed", "restricted", "unavailable", "stale"}
@@ -379,6 +380,33 @@ def init_db():
finished_at TEXT
);
CREATE TABLE IF NOT EXISTS dataset_generation_jobs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
source_name TEXT NOT NULL,
source_path TEXT NOT NULL,
model TEXT NOT NULL,
format TEXT NOT NULL DEFAULT 'qa',
target_count INTEGER NOT NULL DEFAULT 20,
status TEXT NOT NULL DEFAULT 'pending',
generated_count INTEGER NOT NULL DEFAULT 0,
error_msg TEXT,
created_by TEXT NOT NULL,
created_at TEXT NOT NULL,
started_at TEXT,
finished_at TEXT
);
CREATE TABLE IF NOT EXISTS dataset_generation_samples (
id INTEGER PRIMARY KEY AUTOINCREMENT,
job_id INTEGER NOT NULL,
instruction TEXT NOT NULL,
output TEXT NOT NULL,
source_text TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT 'pending',
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS prompts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT NOT NULL,
@@ -1171,6 +1199,8 @@ def _feature_enabled(feature_key: str, feasibility: Optional[dict] = None) -> bo
def require_feature(feature_key: str):
def dependency(user: dict = Depends(current_user)):
if PROTOTYPE_LICENSE_BYPASS:
return user
if not _feature_enabled(feature_key):
raise HTTPException(status_code=403, detail=f"Feature '{feature_key}' is not licensed or provisioned")
return user
@@ -1571,12 +1601,14 @@ SERVICES = [
{"name": "jupyter", "port": 8888, "label": "JupyterLab"},
{"name": "mlflow", "port": 5000, "label": "MLflow"},
{"name": "grafana", "port": 3000, "label": "Grafana"},
{"name": "k3s", "host": "192.168.122.1", "port": 6443, "label": "K3s"},
{"name": "harbor", "host": "192.168.122.1", "port": 30443, "label": "Harbor Registry"},
]
def service_status(svc: dict) -> dict:
import socket
try:
s = socket.create_connection(("127.0.0.1", svc["port"]), timeout=1)
s = socket.create_connection((svc.get("host", "127.0.0.1"), svc["port"]), timeout=1)
s.close()
running = True
except OSError:
@@ -2634,25 +2666,261 @@ async def rag_query(body: dict, user: dict = Depends(current_user)):
DATASETS_DIR = DATA_DIR / "datasets"
MODELS_DIR = DATA_DIR / "finetuned"
JOBS_LOG_DIR = DATA_DIR / "job_logs"
DATASET_SOURCES_DIR = DATA_DIR / "dataset_sources"
DATASETS_DIR.mkdir(parents=True, exist_ok=True)
MODELS_DIR.mkdir(parents=True, exist_ok=True)
JOBS_LOG_DIR.mkdir(parents=True, exist_ok=True)
DATASET_SOURCES_DIR.mkdir(parents=True, exist_ok=True)
def _count_rows(path: Path, orig_name: str) -> int:
"""Count data rows in JSONL or CSV file."""
try:
if orig_name.lower().endswith(".csv"):
import csv
with open(path, newline="", encoding="utf-8", errors="replace") as f:
return max(0, sum(1 for _ in csv.reader(f)) - 1) # exclude header
_dataset_generation_tasks: set[asyncio.Task] = set()
def _extract_dataset_source(path: Path, orig_name: str) -> str:
"""Extract plain text from supported dataset source documents."""
ext = Path(orig_name).suffix.lower()
if ext == ".pdf":
import fitz
with fitz.open(path) as doc:
return "\n\n".join(page.get_text("text") for page in doc)
if ext == ".docx":
from docx import Document
doc = Document(path)
return "\n".join(p.text for p in doc.paragraphs)
if ext == ".csv":
import csv
with open(path, newline="", encoding="utf-8", errors="replace") as f:
return "\n".join(" | ".join(str(v) for v in row) for row in csv.reader(f))
return path.read_text(encoding="utf-8", errors="replace")
def _chunk_dataset_source(text: str, max_chars: int = 6000) -> list[str]:
paragraphs = [" ".join(p.split()) for p in text.splitlines() if p.strip()]
chunks, current = [], ""
for paragraph in paragraphs:
if current and len(current) + len(paragraph) + 1 > max_chars:
chunks.append(current)
current = ""
if len(paragraph) > max_chars:
if current:
chunks.append(current)
current = ""
chunks.extend(paragraph[i:i + max_chars] for i in range(0, len(paragraph), max_chars))
else:
return sum(1 for line in open(path, encoding="utf-8", errors="replace") if line.strip())
except Exception:
return 0
current = f"{current}\n{paragraph}".strip()
if current:
chunks.append(current)
return chunks
def _parse_generated_samples(raw: str) -> list[dict]:
"""Accept a JSON array or an object containing a samples array."""
raw = raw.strip()
if raw.startswith("```"):
raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
try:
data = json.loads(raw)
except json.JSONDecodeError:
start, end = raw.find("["), raw.rfind("]")
if start < 0 or end <= start:
return []
data = json.loads(raw[start:end + 1])
if isinstance(data, dict):
data = data.get("samples", [])
if not isinstance(data, list):
return []
valid = []
for row in data:
if not isinstance(row, dict):
continue
instruction = str(row.get("instruction") or row.get("question") or row.get("prompt") or "").strip()
output = str(row.get("output") or row.get("answer") or row.get("completion") or "").strip()
if instruction and output:
valid.append({"instruction": instruction[:8000], "output": output[:16000]})
return valid
def _generate_samples_for_chunk(model: str, source: str, count: int, output_format: str) -> list[dict]:
format_guidance = {
"qa": "Create grounded question and answer pairs.",
"instruction": "Create practical instruction and response pairs.",
"summary": "Create summarisation instructions and faithful summaries.",
}.get(output_format, "Create grounded question and answer pairs.")
prompt = f"""You create high-quality supervised fine-tuning data. {format_guidance}
Use only facts present in SOURCE. Do not invent facts. Make each example distinct and useful.
Return only a JSON array with exactly {count} objects. Each object must have string fields
\"instruction\" and \"output\". Do not use markdown fences.
SOURCE:
{source}
"""
raw = _call_ollama_sync(model, prompt)
return _parse_generated_samples(raw)
async def _run_dataset_generation(job_id: int):
db = get_db()
job = db.execute("SELECT * FROM dataset_generation_jobs WHERE id=?", (job_id,)).fetchone()
if not job:
db.close()
return
db.execute("UPDATE dataset_generation_jobs SET status='running', started_at=? WHERE id=?", (utcnow(), job_id))
db.commit()
db.close()
try:
text = await asyncio.to_thread(_extract_dataset_source, Path(job["source_path"]), job["source_name"])
chunks = _chunk_dataset_source(text)
if not chunks:
raise ValueError("No readable text was found in the source document")
target = int(job["target_count"])
generated, seen = 0, set()
for index, chunk in enumerate(chunks):
remaining = target - generated
if remaining <= 0:
break
remaining_chunks = max(1, len(chunks) - index)
count = min(5, max(1, (remaining + remaining_chunks - 1) // remaining_chunks))
rows = await asyncio.to_thread(_generate_samples_for_chunk, job["model"], chunk, count, job["format"])
db = get_db()
for row in rows:
key = row["instruction"].casefold()
if key in seen or generated >= target:
continue
seen.add(key)
db.execute(
"INSERT INTO dataset_generation_samples (job_id,instruction,output,source_text,status,created_at) VALUES (?,?,?,?,?,?)",
(job_id, row["instruction"], row["output"], chunk[:6000], "pending", utcnow())
)
generated += 1
db.execute("UPDATE dataset_generation_jobs SET generated_count=? WHERE id=?", (generated, job_id))
db.commit()
db.close()
db = get_db()
db.execute("UPDATE dataset_generation_jobs SET status='completed', generated_count=?, finished_at=? WHERE id=?", (generated, utcnow(), job_id))
db.commit()
db.close()
except Exception as exc:
db = get_db()
db.execute("UPDATE dataset_generation_jobs SET status='failed', error_msg=?, finished_at=? WHERE id=?", (str(exc)[:1000], utcnow(), job_id))
db.commit()
db.close()
@app.post("/api/training/dataset-generation")
async def create_dataset_generation(
file: UploadFile = File(...), name: str = Form(...), model: str = Form(...),
format: str = Form("qa"), target_count: int = Form(20),
admin: dict = Depends(admin_only)
):
allowed_ext = {".pdf", ".docx", ".txt", ".md", ".csv"}
orig = Path(file.filename or "source.txt").name
ext = Path(orig).suffix.lower()
if ext not in allowed_ext:
raise HTTPException(status_code=400, detail="Use PDF, DOCX, TXT, Markdown, or CSV source files")
if format not in {"qa", "instruction", "summary"}:
raise HTTPException(status_code=400, detail="Unsupported dataset format")
if not model.strip() or not name.strip():
raise HTTPException(status_code=400, detail="Dataset name and local model are required")
target_count = max(1, min(int(target_count), 200))
data = await file.read()
if not data or len(data) > 20 * 1024 * 1024:
raise HTTPException(status_code=400, detail="Source file must be between 1 byte and 20 MB")
source_path = DATASET_SOURCES_DIR / f"{uuid.uuid4().hex}{ext}"
source_path.write_bytes(data)
db = get_db()
cur = db.execute(
"INSERT INTO dataset_generation_jobs (name,source_name,source_path,model,format,target_count,status,created_by,created_at) VALUES (?,?,?,?,?,?,?,?,?)",
(name.strip()[:160], orig, str(source_path), model.strip()[:160], format, target_count, "pending", admin["username"], utcnow())
)
db.commit(); job_id = cur.lastrowid; db.close()
task = asyncio.create_task(_run_dataset_generation(job_id))
_dataset_generation_tasks.add(task)
task.add_done_callback(_dataset_generation_tasks.discard)
_audit(None, admin["username"], "dataset_generation_start", f"Started dataset generation: {name} using {model}")
return {"job_id": job_id, "status": "pending"}
@app.get("/api/training/dataset-generation")
async def list_dataset_generations(admin: dict = Depends(admin_only)):
db = get_db()
rows = db.execute("SELECT id,name,source_name,model,format,target_count,generated_count,status,error_msg,created_at,finished_at FROM dataset_generation_jobs ORDER BY id DESC").fetchall()
db.close()
return {"jobs": [dict(row) for row in rows]}
@app.get("/api/training/dataset-generation/{job_id}/samples")
async def list_generated_samples(job_id: int, admin: dict = Depends(admin_only)):
db = get_db()
rows = db.execute("SELECT id,instruction,output,source_text,status FROM dataset_generation_samples WHERE job_id=? ORDER BY id", (job_id,)).fetchall()
db.close()
return {"samples": [dict(row) for row in rows]}
@app.patch("/api/training/dataset-generation/samples/{sample_id}")
async def update_generated_sample(sample_id: int, body: dict, admin: dict = Depends(admin_only)):
status_value = body.get("status", "pending")
if status_value not in {"pending", "approved", "rejected"}:
raise HTTPException(status_code=400, detail="Invalid review status")
instruction, output = str(body.get("instruction", "")).strip(), str(body.get("output", "")).strip()
if not instruction or not output:
raise HTTPException(status_code=400, detail="Instruction and output cannot be empty")
db = get_db()
cur = db.execute("UPDATE dataset_generation_samples SET instruction=?,output=?,status=? WHERE id=?", (instruction[:8000], output[:16000], status_value, sample_id))
db.commit(); db.close()
if not cur.rowcount:
raise HTTPException(status_code=404, detail="Sample not found")
return {"ok": True}
@app.post("/api/training/dataset-generation/{job_id}/publish")
async def publish_generated_dataset(job_id: int, admin: dict = Depends(admin_only)):
db = get_db()
job = db.execute("SELECT * FROM dataset_generation_jobs WHERE id=?", (job_id,)).fetchone()
if not job:
db.close(); raise HTTPException(status_code=404, detail="Generation job not found")
rows = db.execute("SELECT instruction,output FROM dataset_generation_samples WHERE job_id=? AND status='approved' ORDER BY id", (job_id,)).fetchall()
if not rows:
db.close(); raise HTTPException(status_code=400, detail="Approve at least one sample before publishing")
filename = f"{uuid.uuid4().hex}.jsonl"
dest = DATASETS_DIR / filename
content = "".join(json.dumps({"instruction": row["instruction"], "output": row["output"]}, ensure_ascii=False) + "\n" for row in rows)
dest.write_text(content, encoding="utf-8")
orig_name = "".join(c if c.isalnum() or c in "-_" else "_" for c in job["name"]).strip("_") or "generated_dataset"
orig_name += ".jsonl"
cur = db.execute("INSERT INTO training_datasets (filename,orig_name,file_path,size_bytes,row_count,uploaded_at) VALUES (?,?,?,?,?,?)", (filename, orig_name, str(dest), len(content.encode()), len(rows), utcnow()))
db.commit(); dataset_id = cur.lastrowid; db.close()
_audit(None, admin["username"], "dataset_generation_publish", f"Published {orig_name} ({len(rows)} approved rows)")
return {"dataset_id": dataset_id, "filename": orig_name, "row_count": len(rows)}
def _validate_training_dataset(path: Path, orig_name: str) -> int:
"""Validate a training dataset and return its semantic record count."""
import csv
ext = Path(orig_name).suffix.lower()
if ext == ".csv":
with open(path, newline="", encoding="utf-8", errors="replace") as f:
rows = list(csv.DictReader(f))
elif ext == ".json":
with open(path, encoding="utf-8", errors="replace") as f:
payload = json.load(f)
if isinstance(payload, list):
rows = payload
elif isinstance(payload, dict):
rows = next((payload[key] for key in ("samples", "data", "records")
if isinstance(payload.get(key), list)), None)
if rows is None:
raise ValueError("JSON must be an array of training records or contain a samples, data, or records array")
else:
raise ValueError("JSON must contain an array of training records")
else:
rows = []
with open(path, encoding="utf-8", errors="replace") as f:
for line_number, line in enumerate(f, 1):
if not line.strip():
continue
try:
rows.append(json.loads(line))
except json.JSONDecodeError as exc:
raise ValueError(f"Invalid JSON on line {line_number}") from exc
if not rows:
raise ValueError("Dataset has no training records")
if not all(isinstance(row, dict) and row for row in rows):
raise ValueError("Every training record must be a non-empty object or CSV row")
return len(rows)
@app.post("/api/training/datasets")
async def upload_dataset(file: UploadFile = File(...), _licensed: dict = Depends(require_feature("fine_tuning")), admin: dict = Depends(admin_only)):
async def upload_dataset(file: UploadFile = File(...), admin: dict = Depends(admin_only)):
"""Upload a JSONL or CSV dataset for fine-tuning."""
allowed_ext = {".jsonl", ".json", ".csv"}
orig = file.filename or "dataset"
@@ -2668,7 +2936,11 @@ async def upload_dataset(file: UploadFile = File(...), _licensed: dict = Depends
dest = DATASETS_DIR / filename
dest.write_bytes(data)
row_count = _count_rows(dest, orig)
try:
row_count = _validate_training_dataset(dest, orig)
except (ValueError, json.JSONDecodeError) as exc:
dest.unlink(missing_ok=True)
raise HTTPException(status_code=400, detail=f"Invalid training dataset: {exc}") from exc
db = get_db()
cur = db.execute(
@@ -2683,7 +2955,7 @@ async def upload_dataset(file: UploadFile = File(...), _licensed: dict = Depends
return {"id": row_id, "filename": orig, "size_bytes": len(data), "row_count": row_count}
@app.get("/api/training/datasets")
async def list_datasets(_licensed: dict = Depends(require_feature("fine_tuning")), admin: dict = Depends(admin_only)):
async def list_datasets(admin: dict = Depends(admin_only)):
db = get_db()
rows = db.execute(
"SELECT id, orig_name, size_bytes, row_count, uploaded_at FROM training_datasets ORDER BY uploaded_at DESC"
@@ -2692,7 +2964,7 @@ async def list_datasets(_licensed: dict = Depends(require_feature("fine_tuning")
return {"datasets": [dict(r) for r in rows]}
@app.delete("/api/training/datasets/{dataset_id}")
async def delete_dataset(dataset_id: int, _licensed: dict = Depends(require_feature("fine_tuning")), admin: dict = Depends(admin_only)):
async def delete_dataset(dataset_id: int, admin: dict = Depends(admin_only)):
db = get_db()
row = db.execute("SELECT * FROM training_datasets WHERE id=?", (dataset_id,)).fetchone()
if not row:
@@ -2726,6 +2998,11 @@ async def launch_job(body: dict, _licensed: dict = Depends(require_feature("fine
if not ds:
db.close()
raise HTTPException(status_code=404, detail="Dataset not found")
try:
_validate_training_dataset(Path(ds["file_path"]), ds["orig_name"])
except (OSError, ValueError, json.JSONDecodeError) as exc:
db.close()
raise HTTPException(status_code=400, detail=f"Invalid training dataset: {exc}") from exc
config = {
"epochs": int(body.get("epochs", 3)),
@@ -3402,6 +3679,10 @@ async def my_analytics(days: int = 30, user: dict = Depends(current_user)):
AGENT_STEP_TYPES = {"prompt", "summarise", "extract", "classify", "rag", "format"}
# Keep strong references to in-flight tasks. asyncio only maintains weak
# references, so fire-and-forget tasks can otherwise disappear mid-run.
_agent_run_tasks: set = set()
def _call_ollama_sync(model: str, prompt: str) -> str:
import urllib.request as _ur
routed = _select_router_model(prompt, model, {"content_type": "text"})
@@ -3626,7 +3907,9 @@ async def run_agent(agent_id: int, body: dict, user: dict = Depends(current_user
)
run_id = cur.lastrowid
db.commit(); db.close()
asyncio.create_task(_execute_agent(run_id))
task = asyncio.create_task(_execute_agent(run_id))
_agent_run_tasks.add(task)
task.add_done_callback(_agent_run_tasks.discard)
return {"run_id": run_id, "status": "pending"}
@@ -4317,6 +4600,7 @@ Respond with ONLY valid JSON, no markdown, no explanation:
payload = {
"model": judge_model,
"messages": [{"role": "user", "content": judge_prompt}],
"format": "json",
"stream": False,
}
try:
@@ -4595,13 +4879,13 @@ async def get_eval_results(run_id: int, user=Depends(current_user)):
# ── Scheduled Jobs ────────────────────────────────────────────────────────────
def _run_scheduled_job_sync(job_id: int):
def _run_scheduled_job_sync(job_id: int, force: bool = False):
"""Execute a scheduled job (runs in executor thread)."""
import asyncio as _asyncio
db = get_db()
try:
job = db.execute("SELECT * FROM scheduled_jobs WHERE id=?", (job_id,)).fetchone()
if not job or not job["is_active"]:
if not job or (not job["is_active"] and not force):
db.close()
return
@@ -4842,7 +5126,8 @@ async def trigger_schedule(job_id: int, user=Depends(current_user)):
if not row:
raise HTTPException(404, "Job not found")
loop = asyncio.get_event_loop()
loop.run_in_executor(None, _run_scheduled_job_sync, job_id)
# A manual run is explicit user intent and must also work for paused jobs.
loop.run_in_executor(None, _run_scheduled_job_sync, job_id, True)
return {"ok": True, "message": "Job triggered"}
ansible/roles/cezen-backend/files/train_qlora.py
+16 -2
View File
@@ -73,7 +73,7 @@ def set_status(status: str):
# ── Dataset loading ───────────────────────────────────────────────────────────
def load_dataset_from_file(path: str):
"""Load JSONL or CSV dataset into a list of dicts with 'text' or 'prompt'/'completion' keys."""
"""Load JSON, JSONL, or CSV training records as a list of objects."""
p = Path(path)
rows = []
if p.suffix.lower() == ".csv":
@@ -82,6 +82,16 @@ def load_dataset_from_file(path: str):
reader = csv.DictReader(f)
for row in reader:
rows.append(dict(row))
elif p.suffix.lower() == ".json":
with open(path, encoding="utf-8", errors="replace") as f:
payload = json.load(f)
if isinstance(payload, list):
rows = payload
elif isinstance(payload, dict):
for key in ("samples", "data", "records"):
if isinstance(payload.get(key), list):
rows = payload[key]
break
else:
with open(path, encoding="utf-8", errors="replace") as f:
for line in f:
@@ -90,7 +100,11 @@ def load_dataset_from_file(path: str):
try:
rows.append(json.loads(line))
except Exception:
pass
raise ValueError("Dataset contains an invalid JSONL row")
if not rows:
raise ValueError("Dataset has no training records")
if not all(isinstance(row, dict) for row in rows):
raise ValueError("Every training record must be a JSON object or CSV row")
return rows
def format_row(row: dict) -> str: