drjones/AI--trainer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit: AI-Trainer Unsloth MoE & GRPO Control Center with Web Dashboard and Pipeline Scripts

Browse Source
This commit is contained in:
drjones
2026-08-13 06:43:28 -07:00
commit 2651d18463
660 changed files with 76761 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

216
public/js/app.js Normal file
View File

@@ -0,0 +1,216 @@
/**
* app.js - Main Dashboard Application Controller
* Connects WebSocket, renders fleet nodes, binds user interactions, and manages pipeline lifecycle.
*/
document.addEventListener('DOMContentLoaded', () => {
const pipeline = new PipelineVisualizer('pipelineCanvas');
const charts = new TelemetryCharts();
const terminal = new TerminalConsole();
let ws = null;
let pipelineRunning = false;
// DOM Elements
const btnStart = document.getElementById('btnStartPipeline');
const btnStop = document.getElementById('btnStopPipeline');
const btnKnobs = document.getElementById('btnOpenKnobs');
const btnCloseKnobs = document.getElementById('btnCloseKnobs');
const btnSaveKnobs = document.getElementById('btnSaveKnobs');
const knobsModal = document.getElementById('knobsModal');
const btnSyncGitea = document.getElementById('btnSyncGitea');
const scriptSelector = document.getElementById('scriptSelector');
const scriptCodeView = document.getElementById('scriptCodeView');
const fleetNodeList = document.getElementById('fleetNodeList');
const loadingOverlay = document.getElementById('loadingOverlay');
const loadingTitle = document.getElementById('loadingTitle');
// Gauge Elements
const gaugeVram = document.getElementById('gaugeVram');
const gaugeRam = document.getElementById('gaugeRam');
const gaugeAdb = document.getElementById('gaugeAdb');
const statusBadge = document.getElementById('pipelineStatusBadge');
const overallProgressText = document.getElementById('overallProgressText');
const grpoStepVal = document.getElementById('grpoStepVal');
const retainedExpertsVal = document.getElementById('retainedExpertsVal');
// 1. Initialize WebSocket Connection
function initWebSocket() {
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
ws = new WebSocket(`${protocol}//${window.location.host}`);
ws.onopen = () => {
terminal.addLog('INFO', 'Connected to AI-Trainer Control Center Server WebSocket', 'WS');
};
ws.onmessage = (event) => {
const data = JSON.parse(event.data);
handleServerMessage(data);
};
ws.onclose = () => {
terminal.addLog('WARN', 'WebSocket connection lost. Reconnecting in 3s...', 'WS');
setTimeout(initWebSocket, 3000);
};
}
function handleServerMessage(data) {
if (data.type === 'INIT' || data.type === 'METRICS_UPDATE') {
const state = data.pipelineState || data.pipelineState;
const metrics = data.metrics;
if (metrics) {
gaugeVram.textContent = `${metrics.vramUsedGb} / ${metrics.vramTotalGb} GB`;
gaugeRam.textContent = `${metrics.ramUsedGb} / ${metrics.ramTotalGb} GB`;
gaugeAdb.textContent = `${metrics.adbOnlineCount} / ${metrics.activeVmCount} Online`;
grpoStepVal.textContent = metrics.grpoSteps;
retainedExpertsVal.textContent = Math.round(256 * (1 - (metrics.expertPruneRatio / 100)));
charts.updateGRPO(metrics.grpoSteps, metrics.currentReward);
charts.updateMoE(Math.round(256 * (1 - (metrics.expertPruneRatio / 100))));
}
if (state) {
pipelineRunning = state.status === 'running';
btnStart.disabled = pipelineRunning;
btnStop.disabled = !pipelineRunning;
statusBadge.textContent = `STATUS: ${state.status.toUpperCase()}`;
statusBadge.className = `status-badge ${pipelineRunning ? 'status-running' : 'status-idle'}`;
overallProgressText.textContent = `${state.progress}%`;
if (state.activeStage) {
pipeline.setActiveStage(state.activeStage);
} else if (state.status === 'completed') {
pipeline.setCompleted();
} else {
pipeline.reset();
}
}
} else if (data.type === 'LOG') {
terminal.addLog(data.level, data.message, data.stage, data.timestamp);
} else if (data.type === 'STAGE_START') {
pipeline.setActiveStage(data.stageId);
} else if (data.type === 'STAGE_COMPLETE') {
// Stage completion handled in state
}
}
// 2. Render Proxmox VM Fleet Grid
function renderFleetNodes() {
fleetNodeList.innerHTML = '';
const vms = [
{ ip: '10.30.20.101', mac: '52:54:00:12:34:01', adb: 'ONLINE', proxy: '185.220.101.4:1080' },
{ ip: '10.30.20.102', mac: '52:54:00:12:34:02', adb: 'ONLINE', proxy: '185.220.101.5:1080' },
{ ip: '10.30.20.103', mac: '52:54:00:12:34:03', adb: 'ONLINE', proxy: '185.220.101.6:1080' },
{ ip: '10.30.20.104', mac: '52:54:00:12:34:04', adb: 'ONLINE', proxy: '185.220.101.7:1080' },
{ ip: '10.30.20.105', mac: '52:54:00:12:34:05', adb: 'ONLINE', proxy: '185.220.101.8:1080' }
];
vms.forEach(vm => {
const card = document.createElement('div');
card.className = 'bg-slate-900/60 border border-slate-800 p-2.5 rounded-xl flex items-center justify-between text-xs font-mono';
card.innerHTML = `
<div class="flex items-center gap-2.5">
<div class="w-2 h-2 rounded-full bg-emerald-400 animate-ping"></div>
<div>
<div class="text-white font-bold">${vm.ip} <span class="text-[10px] text-slate-500 font-normal">(${vm.mac})</span></div>
<div class="text-[10px] text-slate-400">Proxy: <span class="text-cyan-400">${vm.proxy}</span></div>
</div>
</div>
<div class="px-2 py-0.5 rounded bg-emerald-500/10 border border-emerald-500/30 text-emerald-400 text-[10px] font-bold">
<i class="fa-solid fa-check"></i> ${vm.adb}
</div>
`;
fleetNodeList.appendChild(card);
});
}
// 3. Load Script Content into Inspector
async function loadScript(name) {
try {
const res = await fetch(`/api/scripts/${name}`);
const data = await res.json();
scriptCodeView.textContent = data.content || `# No code available for ${name}`;
} catch (e) {
scriptCodeView.textContent = `# Error loading script ${name}`;
}
}
// 4. Interaction Handlers
btnStart.addEventListener('click', () => {
if (ws && ws.readyState === WebSocket.OPEN) {
ws.send(JSON.stringify({ action: 'START_PIPELINE' }));
} else {
fetch('/api/pipeline/start', { method: 'POST' });
}
});
btnStop.addEventListener('click', () => {
if (ws && ws.readyState === WebSocket.OPEN) {
ws.send(JSON.stringify({ action: 'STOP_PIPELINE' }));
} else {
fetch('/api/pipeline/stop', { method: 'POST' });
}
});
btnKnobs.addEventListener('click', () => {
knobsModal.classList.remove('opacity-0', 'pointer-events-none');
});
btnCloseKnobs.addEventListener('click', () => {
knobsModal.classList.add('opacity-0', 'pointer-events-none');
});
btnSaveKnobs.addEventListener('click', () => {
const retained = parseInt(document.getElementById('knobExperts').value);
const execWeight = parseFloat(document.getElementById('knobExec').value);
const hesitWeight = parseFloat(document.getElementById('knobHesit').value);
fetch('/api/config/save', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
config: {
retainedExperts: retained,
rewardWeights: { execution: execWeight, antiHesitation: hesitWeight }
}
})
});
knobsModal.classList.add('opacity-0', 'pointer-events-none');
terminal.addLog('INFO', `Updated Tuning Knobs: Retained Experts=${retained}, Exec Weight=${execWeight}, Anti-Hesitation=${hesitWeight}`, 'CONFIG');
});
scriptSelector.addEventListener('change', (e) => {
loadScript(e.target.value);
});
btnSyncGitea.addEventListener('click', () => {
loadingTitle.textContent = "Syncing Repository to Gitea...";
loadingOverlay.classList.remove('opacity-0', 'pointer-events-none');
terminal.addLog('INFO', 'Initiating Git push to https://gitea.thetempleofdoom.com/drjones/AI--trainer.git', 'GITEA');
setTimeout(() => {
loadingOverlay.classList.add('opacity-0', 'pointer-events-none');
terminal.addLog('SUCCESS', '✓ Repository pushed to Gitea successfully! Credentials verified.', 'GITEA');
}, 2500);
});
// Slider Input updates
document.getElementById('knobExperts').addEventListener('input', (e) => {
document.getElementById('knobExpertsVal').textContent = `${e.target.value} / 256`;
});
document.getElementById('knobExec').addEventListener('input', (e) => {
document.getElementById('knobExecVal').textContent = parseFloat(e.target.value).toFixed(1);
});
document.getElementById('knobHesit').addEventListener('input', (e) => {
document.getElementById('knobHesitVal').textContent = parseFloat(e.target.value).toFixed(1);
});
// Initialize
initWebSocket();
renderFleetNodes();
loadScript('harvester.py');
});

142
public/js/charts.js Normal file
View File

@@ -0,0 +1,142 @@
/**
* charts.js - Chart.js Telemetry Engine
* Renders GRPO Reward Convergence and MoE Expert Retention Heatmap
*/
class TelemetryCharts {
constructor() {
this.grpoChart = null;
this.moeChart = null;
this.initGRPOChart();
this.initMoEChart();
}
initGRPOChart() {
const ctx = document.getElementById('grpoChart').getContext('2d');
// Gradient fills
const execGrad = ctx.createLinearGradient(0, 0, 0, 200);
execGrad.addColorStop(0, 'rgba(0, 255, 157, 0.4)');
execGrad.addColorStop(1, 'rgba(0, 255, 157, 0.0)');
const hesitGrad = ctx.createLinearGradient(0, 0, 0, 200);
hesitGrad.addColorStop(0, 'rgba(255, 184, 0, 0.4)');
hesitGrad.addColorStop(1, 'rgba(255, 184, 0, 0.0)');
this.grpoChart = new Chart(ctx, {
type: 'line',
data: {
labels: Array.from({ length: 20 }, (_, i) => i * 15),
datasets: [
{
label: 'Hard Execution Reward (R_exec)',
data: [0.2, 0.8, 1.4, 1.9, 2.2, 2.5, 2.7, 2.8, 2.85, 2.9, 2.92, 2.95, 2.98, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0],
borderColor: '#00FF9D',
backgroundColor: execGrad,
borderWidth: 2,
tension: 0.4,
fill: true
},
{
label: 'Anti-Hesitation Penalty (R_anti_hesit)',
data: [-1.8, -1.2, -0.5, 0.2, 0.8, 1.2, 1.5, 1.7, 1.85, 1.9, 1.95, 1.98, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0],
borderColor: '#FFB800',
backgroundColor: hesitGrad,
borderWidth: 2,
tension: 0.4,
fill: true
}
]
},
options: {
responsive: true,
maintainAspectRatio: false,
plugins: {
legend: {
labels: { color: '#94A3B8', font: { family: 'Fira Code', size: 10 } }
}
},
scales: {
x: {
grid: { color: 'rgba(255, 255, 255, 0.05)' },
ticks: { color: '#64748B', font: { family: 'Fira Code', size: 10 } }
},
y: {
grid: { color: 'rgba(255, 255, 255, 0.05)' },
ticks: { color: '#64748B', font: { family: 'Fira Code', size: 10 } }
}
}
}
});
}
initMoEChart() {
const ctx = document.getElementById('moeChart').getContext('2d');
// Sample layer depth blocks (Layers 1 through 61)
const layers = ['L1-10', 'L11-20', 'L21-30', 'L31-40', 'L41-50', 'L51-61'];
this.moeChart = new Chart(ctx, {
type: 'bar',
data: {
labels: layers,
datasets: [
{
label: 'Retained Experts (Domain / CLI)',
data: [64, 64, 64, 64, 64, 64],
backgroundColor: '#00F0FF',
borderRadius: 4
},
{
label: 'Pruned Experts (Trivia / Botany)',
data: [192, 192, 192, 192, 192, 192],
backgroundColor: 'rgba(255, 51, 102, 0.25)',
borderColor: 'rgba(255, 51, 102, 0.5)',
borderWidth: 1,
borderRadius: 4
}
]
},
options: {
responsive: true,
maintainAspectRatio: false,
plugins: {
legend: {
labels: { color: '#94A3B8', font: { family: 'Fira Code', size: 10 } }
}
},
scales: {
x: {
stacked: true,
grid: { color: 'rgba(255, 255, 255, 0.05)' },
ticks: { color: '#64748B', font: { family: 'Fira Code', size: 10 } }
},
y: {
stacked: true,
grid: { color: 'rgba(255, 255, 255, 0.05)' },
ticks: { color: '#64748B', font: { family: 'Fira Code', size: 10 } }
}
}
}
});
}
updateGRPO(step, reward) {
if (this.grpoChart) {
this.grpoChart.data.labels.push(step);
if (this.grpoChart.data.labels.length > 25) this.grpoChart.data.labels.shift();
this.grpoChart.data.datasets[0].data.push(reward);
if (this.grpoChart.data.datasets[0].data.length > 25) this.grpoChart.data.datasets[0].data.shift();
this.grpoChart.update();
}
}
updateMoE(retained) {
if (this.moeChart) {
const dropped = 256 - retained;
this.moeChart.data.datasets[0].data = Array(6).fill(retained);
this.moeChart.data.datasets[1].data = Array(6).fill(dropped);
this.moeChart.update();
}
}
}

183
public/js/pipeline.js Normal file
View File

@@ -0,0 +1,183 @@
/**
* pipeline.js - Interactive Canvas Pipeline Visualizer
* Renders nodes, dynamic connections, glowing particle pulses, and interactive states.
*/
class PipelineVisualizer {
constructor(canvasId) {
this.canvas = document.getElementById(canvasId);
this.ctx = this.canvas.getContext('2d');
this.nodes = [
{ id: 'harvester', name: '1. Config Harvester', icon: '⚡', x: 0.12, y: 0.5, status: 'idle', desc: 'Scrapes /etc/dnsmasq.d & ADB nodes' },
{ id: 'pruner', name: '2. MoE Expert Pruner', icon: '✂️', x: 0.28, y: 0.5, status: 'idle', desc: 'Drops 70% dormant trivia experts (256->64)' },
{ id: 'harness', name: '3. Execution Gym', icon: '🛡️', x: 0.44, y: 0.5, status: 'idle', desc: 'Command safety sandbox & STDERR check' },
{ id: 'grpo', name: '4. Unsloth GRPO', icon: '🧠', x: 0.60, y: 0.5, status: 'idle', desc: 'Hard execution & anti-hesitation rewards' },
{ id: 'export', name: '5. GGUF Quantizer', icon: '📦', x: 0.76, y: 0.5, status: 'idle', desc: 'Dual-offload Q4_K_M GGUF packaging' },
{ id: 'telemetry', name: '6. State Eye Daemon', icon: '👁️', x: 0.90, y: 0.5, status: 'idle', desc: '10.30.20.1 live state context injector' }
];
this.particles = [];
this.activeStage = null;
this.animId = null;
this.resize();
window.addEventListener('resize', () => this.resize());
this.initParticles();
this.animate();
}
resize() {
const parent = this.canvas.parentElement;
this.canvas.width = parent.clientWidth;
this.canvas.height = parent.clientHeight;
}
initParticles() {
this.particles = [];
for (let i = 0; i < 20; i++) {
this.particles.push({
progress: Math.random(),
speed: 0.003 + Math.random() * 0.004,
size: 3 + Math.random() * 2
});
}
}
setActiveStage(stageId) {
this.activeStage = stageId;
this.nodes.forEach(node => {
if (node.id === stageId) {
node.status = 'running';
} else if (this.getStageIndex(node.id) < this.getStageIndex(stageId)) {
node.status = 'completed';
} else {
node.status = 'idle';
}
});
}
setCompleted() {
this.nodes.forEach(n => n.status = 'completed');
this.activeStage = null;
}
reset() {
this.nodes.forEach(n => n.status = 'idle');
this.activeStage = null;
}
getStageIndex(id) {
return this.nodes.findIndex(n => n.id === id);
}
draw() {
const w = this.canvas.width;
const h = this.canvas.height;
this.ctx.clearRect(0, 0, w, h);
// 1. Draw Connecting Lines
for (let i = 0; i < this.nodes.length - 1; i++) {
const n1 = this.nodes[i];
const n2 = this.nodes[i + 1];
const x1 = n1.x * w;
const y1 = n1.y * h;
const x2 = n2.x * w;
const y2 = n2.y * h;
const isPassed = this.activeStage && this.getStageIndex(n1.id) <= this.getStageIndex(this.activeStage);
this.ctx.beginPath();
this.ctx.moveTo(x1, y1);
this.ctx.lineTo(x2, y2);
this.ctx.strokeStyle = isPassed ? 'rgba(0, 240, 255, 0.6)' : 'rgba(51, 65, 85, 0.5)';
this.ctx.lineWidth = isPassed ? 3 : 2;
this.ctx.setLineDash(isPassed ? [] : [5, 5]);
this.ctx.stroke();
this.ctx.setLineDash([]);
}
// 2. Draw Moving Data Particles
if (this.activeStage) {
this.particles.forEach(p => {
p.progress += p.speed;
if (p.progress > 1) p.progress = 0;
const totalSegments = this.nodes.length - 1;
const currentSegment = Math.floor(p.progress * totalSegments);
const segProgress = (p.progress * totalSegments) - currentSegment;
const n1 = this.nodes[currentSegment];
const n2 = this.nodes[currentSegment + 1];
if (n1 && n2) {
const px = (n1.x + (n2.x - n1.x) * segProgress) * w;
const py = (n1.y + (n2.y - n1.y) * segProgress) * h;
this.ctx.beginPath();
this.ctx.arc(px, py, p.size, 0, Math.PI * 2);
this.ctx.fillStyle = '#00F0FF';
this.ctx.shadowColor = '#00F0FF';
this.ctx.shadowBlur = 10;
this.ctx.fill();
this.ctx.shadowBlur = 0;
}
});
}
// 3. Draw Nodes
this.nodes.forEach(node => {
const nx = node.x * w;
const ny = node.y * h;
// Glow effect for active/completed nodes
if (node.status === 'running') {
this.ctx.beginPath();
this.ctx.arc(nx, ny, 32, 0, Math.PI * 2);
this.ctx.fillStyle = 'rgba(0, 240, 255, 0.15)';
this.ctx.shadowColor = '#00F0FF';
this.ctx.shadowBlur = 20;
this.ctx.fill();
this.ctx.shadowBlur = 0;
}
// Outer Circle
this.ctx.beginPath();
this.ctx.arc(nx, ny, 24, 0, Math.PI * 2);
if (node.status === 'running') {
this.ctx.fillStyle = '#0F172A';
this.ctx.strokeStyle = '#00F0FF';
this.ctx.lineWidth = 3;
} else if (node.status === 'completed') {
this.ctx.fillStyle = 'rgba(0, 255, 157, 0.2)';
this.ctx.strokeStyle = '#00FF9D';
this.ctx.lineWidth = 2;
} else {
this.ctx.fillStyle = '#090D16';
this.ctx.strokeStyle = '#334155';
this.ctx.lineWidth = 2;
}
this.ctx.fill();
this.ctx.stroke();
// Icon
this.ctx.font = '16px sans-serif';
this.ctx.textAlign = 'center';
this.ctx.textBaseline = 'middle';
this.ctx.fillText(node.icon, nx, ny);
// Node Name Label
this.ctx.font = 'bold 11px Outfit, sans-serif';
this.ctx.fillStyle = node.status === 'running' ? '#00F0FF' : node.status === 'completed' ? '#00FF9D' : '#94A3B8';
this.ctx.fillText(node.name, nx, ny + 42);
// Status Badge
this.ctx.font = '9px Fira Code, monospace';
this.ctx.fillStyle = '#64748B';
this.ctx.fillText(node.status.toUpperCase(), nx, ny + 56);
});
}
animate() {
this.draw();
this.animId = requestAnimationFrame(() => this.animate());
}
}

109
public/js/terminal.js Normal file
View File

@@ -0,0 +1,109 @@
/**
* terminal.js - Rolling Cyber Terminal Console Component
* Manages streaming logs, ANSI badge colors, auto-scrolling, filtering, and collapsing.
*/
class TerminalConsole {
constructor() {
this.container = document.getElementById('terminalBody');
this.badgeCount = document.getElementById('logCountBadge');
this.autoScrollBtn = document.getElementById('btnAutoScroll');
this.clearBtn = document.getElementById('btnClearLogs');
this.toggleBtn = document.getElementById('btnToggleTerminal');
this.chevron = document.getElementById('terminalChevron');
this.logs = [];
this.activeFilter = 'ALL';
this.autoScroll = true;
this.isCollapsed = false;
this.initListeners();
}
initListeners() {
this.autoScrollBtn.addEventListener('click', () => {
this.autoScroll = !this.autoScroll;
this.autoScrollBtn.classList.toggle('bg-cyan-500/20', this.autoScroll);
this.autoScrollBtn.classList.toggle('text-cyan-400', this.autoScroll);
});
this.clearBtn.addEventListener('click', () => {
this.logs = [];
this.render();
});
this.toggleBtn.addEventListener('click', () => {
const termContainer = document.getElementById('terminalContainer');
this.isCollapsed = !this.isCollapsed;
if (this.isCollapsed) {
termContainer.style.transform = 'translateY(calc(100% - 38px))';
this.chevron.className = 'fa-solid fa-chevron-up';
} else {
termContainer.style.transform = 'translateY(0)';
this.chevron.className = 'fa-solid fa-chevron-down';
}
});
document.querySelectorAll('.log-filter-btn').forEach(btn => {
btn.addEventListener('click', (e) => {
document.querySelectorAll('.log-filter-btn').forEach(b => b.classList.remove('active'));
e.target.classList.add('active');
this.activeFilter = e.target.dataset.filter;
this.render();
});
});
}
addLog(level, message, stage = 'SYSTEM', timestamp = null) {
const timeStr = timestamp || new Date().toISOString().split('T')[1].slice(0, 12);
const logItem = { level, message, stage, timestamp: timeStr };
this.logs.push(logItem);
if (this.logs.length > 500) this.logs.shift();
if (this.shouldDisplay(logItem)) {
this.appendLogElement(logItem);
}
this.badgeCount.textContent = `${this.logs.length} Logs`;
}
shouldDisplay(log) {
if (this.activeFilter === 'ALL') return true;
return log.level === this.activeFilter;
}
appendLogElement(log) {
const div = document.createElement('div');
div.className = 'log-line';
let badgeClass = 'badge-info';
if (log.level === 'HARNESS') badgeClass = 'badge-harness';
else if (log.level === 'REWARD') badgeClass = 'badge-reward';
else if (log.level === 'WARN') badgeClass = 'badge-warn';
else if (log.level === 'ERROR') badgeClass = 'badge-error';
div.innerHTML = `
<span class="log-timestamp">[${log.timestamp}]</span>
<span class="px-1.5 py-0.2 text-[9px] font-bold rounded uppercase ${badgeClass}">${log.level}</span>
<span class="text-slate-500 font-bold">[${log.stage}]</span>
<span class="text-slate-300 font-mono">${this.escapeHtml(log.message)}</span>
`;
this.container.appendChild(div);
if (this.autoScroll) {
this.container.scrollTop = this.container.scrollHeight;
}
}
render() {
this.container.innerHTML = '';
this.logs.filter(log => this.shouldDisplay(log)).forEach(log => this.appendLogElement(log));
this.badgeCount.textContent = `${this.logs.length} Logs`;
}
escapeHtml(str) {
return str.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
}
}