Eval Generator Code App

CREATE TABLE IF NOT EXISTS features (
  id TEXT PRIMARY KEY,
  title TEXT,
  description TEXT,
  acceptance_criteria TEXT,
  priority TEXT DEFAULT 'medium',
  connectors TEXT,
  layer_presence INTEGER DEFAULT 1,
  presence_status TEXT DEFAULT 'pending',
  mapped_files TEXT
);

{
  "generated": "<ISO timestamp>",
  "project": "<project root basename>",
  "generatedFrom": "brd | code-review",
  "features": [
    {
      "id": "user-search",
      "title": "User Search",
      "description": "Allow users to search for people using the Office365Users connector",
      "acceptance_criteria": [
        "Search input triggers Office365UsersService call",
        "Results display name, email, and job title",
        "Empty query shows placeholder, not error"
      ],
      "priority": "high",
      "connectors": ["office365users"],
      "layers": { "presence": true, "unit": true }
    }
  ]
}

grep: "PowerProvider|initialize\(\)" in src/App.tsx or src/main.tsx

# Generated service imports in developer code (src/ excluding src/generated/)
pattern: "from '.*generated/services/" → connector is wired up and used

# Dataverse operations via generated service static methods
pattern: "<ApiName>Service\.\w+\(" → connector operation call
pattern: "result\.value" → Dataverse result access (IOperationResult.value)

# Standard connector result access
pattern: "result\.data" → connector result (IOperationResult.data)
pattern: "result\.success" → success check — should be in every connector call
pattern: "result\.errorMessage" → error handling

# TanStack Query
pattern: "useQuery|useMutation" → async data fetching

# Office365Users (match actual generated service file methods)
pattern: "Office365UsersService\." → any Office365Users service call

// evals/presence/<feature-id>.check.ts
// AUTO-GENERATED by eval-generator-code-app
// Feature: <feature title>
// Description: <feature description>

import { readFileSync, existsSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { globSync } from 'glob';

// Anchored to this file's own location — works correctly whether invoked via runner or directly
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const PROJECT_ROOT = join(__dirname, '..', '..');  // evals/presence → evals → project root
const SRC = join(PROJECT_ROOT, 'src');
const GENERATED = join(PROJECT_ROOT, 'src', 'generated');

interface PresenceResult {
  featureId: string;
  checks: Array<{ name: string; passed: boolean; detail?: string }>;
}

export function check(): PresenceResult {
  const checks: PresenceResult['checks'] = [];

  // ── Check: Generated service file exists ──
  const serviceFile = join(GENERATED, 'services', '<ApiName>Service.ts');
  const serviceExists = existsSync(serviceFile);
  checks.push({
    name: 'generated service file exists',
    passed: serviceExists,
    detail: serviceExists ? undefined : 'Expected: src/generated/services/<ApiName>Service.ts — run npx power-apps add-dataverse-api',
  });

  // ── Check: Service is imported in developer code (outside src/generated/) ──
  const srcFiles = globSync(`${SRC}/**/*.ts?(x)`, { ignore: `${GENERATED}/**` });
  const allSource = srcFiles.map(f => readFileSync(f, 'utf-8')).join('\n');

  const importFound = /from ['"].*generated\/services\/<ApiName>Service['"]/.test(allSource);
  checks.push({
    name: 'generated service imported in src/',
    passed: importFound,
    detail: importFound ? undefined : 'No import of <ApiName>Service found outside src/generated/',
  });

  // ── Check: result.success is checked (error handling) ──
  const successCheck = /result\.success/.test(allSource);
  checks.push({
    name: 'result.success checked after service call',
    passed: successCheck,
    detail: successCheck ? undefined : 'result.success is never checked — connector errors may be silently swallowed',
  });

  return { featureId: '<feature-id>', checks };
}

// evals/unit/helpers/setup.ts
// Vitest global setup — must exist even if empty

// evals/unit/helpers/mocks.ts
// Power Apps Code App service mocks for Vitest
import { vi } from 'vitest';
import type { AppData } from '../../../src/lib/AppData';

/** Mock IOperationResult<T[]> for service.getAll() */
export function mockGetAll<T>(data: T[] = []) {
  return vi.fn().mockResolvedValue({ success: true, data, error: null });
}

/** Mock IOperationResult<T> for service.get() / .create() / .update() */
export function mockOperation<T>(value: T | null = null, success = true) {
  return vi.fn().mockResolvedValue({
    success,
    value,
    error: success ? null : { message: 'Mock error' },
  });
}

/** Mock AppData context value for renderWithAppData()
 *
 * ⛔ GROUNDING REQUIRED: Before writing this function, read src/lib/AppData.ts
 * and copy the REAL field names from the AppData interface.
 * The field list below is a TEMPLATE PLACEHOLDER — using invented field names
 * compiles but causes TypeScript errors in actual test files that import AppData.
 * Replace every field with the real ones from the project.
 */
export function createMockAppData(overrides?: Partial<AppData>): AppData {
  // Replace this object with REAL fields from the AppData interface in src/lib/AppData.ts
  return {
    loading: false,
    error: undefined,
    // <real_field_1>: [],
    // <real_field_2>: [],
    // <real_reload_method>: vi.fn().mockResolvedValue(undefined),
    ...overrides,
  } as unknown as AppData;
}

// evals/unit/helpers/factories.ts
// GROUNDED: field names derived from src/generated/models/ and src/lib/types.ts
// One factory per Dataverse entity used in the project

// Replace <EntityName>, <EntityInterface>, <entity_primary_key>, and field names
// with REAL values read from src/generated/models/<EntityName>Model.ts
export function make<EntityName>(overrides?: Partial<<EntityInterface>>): <EntityInterface> {
  return {
    // Primary key — from model file (e.g. demo1_employeeid)
    '<entity_primary_key>': `mock-${Math.random().toString(36).slice(2)}`,
    // Fields — from interface in src/lib/types.ts or src/generated/models/
    // '<demo1_fieldname>': 'Test Value',
    ...overrides,
  } as <EntityInterface>;
}

// evals/vitest.config.ts
import { defineConfig } from 'vitest/config';
import react from '@vitejs/plugin-react';
import { resolve, dirname } from 'path';
import { fileURLToPath } from 'url';

const __dirname = dirname(fileURLToPath(import.meta.url));
const PROJECT_ROOT = resolve(__dirname, '..');  // evals/ → project root

export default defineConfig({
  plugins: [react()],
  test: {
    globals: true,
    environment: 'jsdom',
    root: PROJECT_ROOT,
    setupFiles: [resolve(__dirname, 'unit/helpers/setup.ts')],
    include: ['evals/unit/**/*.test.ts?(x)'],
    reporters: [['verbose'], ['json', { outputFile: 'evals/results/vitest-results.json' }]],
    pool: 'threads',
    passWithNoTests: true,
  },
});

// evals/unit/<FeatureName>.test.tsx
// AUTO-GENERATED by eval-generator-code-app
// Feature: <feature title>

import { describe, it, expect, beforeEach, vi } from 'vitest';

// Mock the service class BEFORE importing anything that uses it
// Replace path and class name with REAL values from src/generated/services/
vi.mock('../../src/generated/services/<ServiceClass>', () => ({
  <ServiceClass>: {
    getAll: vi.fn().mockResolvedValue({ success: true, data: [] }),
    create: vi.fn().mockResolvedValue({ success: true, value: {} }),
    update: vi.fn().mockResolvedValue({ success: true, value: {} }),
  }
}));

import { <ServiceClass> } from '../../src/generated/services/<ServiceClass>';
import { make<EntityName> } from './helpers/factories';

describe('<feature-title> — <ServiceClass>', () => {
  beforeEach(() => { vi.clearAllMocks(); });

  // AC: <criterion> OR Derived from: <ServiceClass>.getAll() returns data array
  it('<feature-id>: getAll() returns data on success', async () => {
    const entity = make<EntityName>({ /* real field from types */ });
    (<any><ServiceClass>).getAll.mockResolvedValueOnce({ success: true, data: [entity] });

    const result = await <ServiceClass>.getAll();
    expect(result.success).toBe(true);
    expect(result.data).toHaveLength(1);
    expect(result.data[0].<real_primary_key_field>).toBe(entity.<real_primary_key_field>);
  });

  // AC: <criterion> OR Derived from: <ServiceClass>.create() creates record
  it('<feature-id>: create() is called with correct payload shape', async () => {
    const payload = make<EntityName>();
    await <ServiceClass>.create(payload as any);
    expect(<ServiceClass>.create).toHaveBeenCalledWith(expect.objectContaining(payload));
  });

  // Add it.todo for unimplemented criteria:
  // it.todo('<feature-id>: <unimplemented criterion description>');
});

// evals/unit/AppData.test.tsx
import { describe, it, expect, vi } from 'vitest';
import { renderHook, waitFor } from '@testing-library/react';
import { AppDataProvider, useAppData } from '../../src/lib/AppData';

// Mock all generated services — use REAL service class names from src/generated/services/
// Replace the 6 service imports with REAL names found in the project
vi.mock('../../src/generated/services/Demo1_employeesService', () => ({
  Demo1_employeesService: { getAll: vi.fn().mockResolvedValue({ success: true, data: [] }) }
}));
// ... repeat vi.mock() for each of the other 5 services

describe('AppData context', () => {
  it('app-data-bootstrap: loads all entities on mount', async () => {
    const wrapper = ({ children }: any) => <AppDataProvider>{children}</AppDataProvider>;
    const { result } = renderHook(() => useAppData(), { wrapper });
    await waitFor(() => expect(result.current.loading).toBe(false));
    // Assert each service was called — use REAL service class names
    const { Demo1_employeesService } = await import('../../src/generated/services/Demo1_employeesService');
    expect(Demo1_employeesService.getAll).toHaveBeenCalledTimes(1);
  });

  // AC: useAppData hook throws if used outside provider
  it('app-data-bootstrap: useAppData throws when used outside AppDataProvider', () => {
    expect(() => renderHook(() => useAppData())).toThrow();
  });
});

#!/usr/bin/env tsx
// evals/runner/security-runner.ts
// Static security analysis — outputs SecurityResult JSON to stdout

import { readFileSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { globSync } from 'glob';

const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const ROOT = join(__dirname, '..', '..');  // evals/runner → evals → project
const SRC = join(ROOT, 'src');

type Severity = 'critical' | 'high' | 'medium' | 'low';

interface SecurityFinding {
  id: string; category: string; severity: Severity;
  file: string; line: number; snippet: string; detail: string;
}

interface SecurityResult {
  layer: 'security';
  passed: number; failed: number; skipped: number;
  findings: SecurityFinding[];
}

const findings: SecurityFinding[] = [];
const srcFiles = globSync(`${SRC}/**/*.ts?(x)`);

function scan(id: string, category: string, severity: Severity, pattern: RegExp, detail: string, exclude?: RegExp) {
  for (const file of srcFiles) {
    let content: string;
    try { content = readFileSync(file, 'utf-8'); } catch { continue; }
    const lines = content.split('\n');
    lines.forEach((line, i) => {
      if (pattern.test(line) && !(exclude && exclude.test(line))) {
        findings.push({ id, category, severity,
          file: file.replace(ROOT + '\\', '').replace(ROOT + '/', ''),
          line: i + 1, snippet: line.trim().slice(0, 120), detail });
      }
    });
  }
}

// SEC-001: OData Injection
scan('SEC-001', 'OData Injection', 'critical',
  /(\$filter.*['"`]\s*\+|\$filter.*\$\{)/,
  'OData $filter built by string concatenation or template literal. Sanitize inputs or use fixed filter strings.');

// SEC-002: XSS
scan('SEC-002', 'XSS', 'critical',
  /dangerouslySetInnerHTML|\.innerHTML\s*=/,
  'Direct HTML injection. Sanitize with DOMPurify or avoid raw HTML insertion entirely.');

// SEC-003: Hardcoded secrets
// Exclude: (1) comment lines, (2) values that are purely alphabetical/kebab-case — real credentials
// always contain digits or special chars. This prevents false positives on TypeScript object maps
// where property keys like `token` hold UI label strings like 'skipToken' or 'badge-token'.
scan('SEC-003', 'Hardcoded Secret', 'critical',
  /(?:apikey|api_key|password|secret|token|client_secret)\s*[:=]\s*['"`][^'"`\s]{6,}/i,
  'Possible hardcoded credential. Move to environment variables or Azure Key Vault.',
  /^\s*\/\/|(?:apikey|api_key|password|secret|token|client_secret)\s*[:=]\s*['"`][a-zA-Z$_][a-zA-Z$_\-]*['"`]/i);

// SEC-004: PII in console logs
// Exclude: console.log("...") calls where the PII keyword appears ONLY inside a single string literal
// (e.g. "Email field is required") — those are UI messages, not variable dumps.
scan('SEC-004', 'PII in Logs', 'high',
  /console\.(log|warn|error|debug)\b.*\b(email|mail|userId|phone|address|password|token|DisplayName)\b/i,
  'Sensitive data written to console. Remove or mask PII before logging.',
  /console\.(log|warn|error|debug)\s*\(\s*(?:'[^']*'|"[^"]*"|`[^`]*`)\s*\)/i);

// SEC-005: Raw error surfaced to UI state
scan('SEC-005', 'Error Disclosure', 'high',
  /set\w+\(\s*(e\.message|String\(e\)|e\.toString\(\))/,
  'Raw error surfaced to UI state. Show a generic user-facing message instead.');

// SEC-006: fetch() to non-Microsoft external URLs
scan('SEC-006', 'Unsafe External Call', 'high',
  /fetch\s*\(\s*['"`]https?:\/\//,
  'fetch() to an external URL. Confirm it is an approved Microsoft/Power Platform endpoint.',
  /\.dynamics\.com|\.microsoft\.com|\.microsoftonline\.com/);

// SEC-007: Client-side auth bypass
scan('SEC-007', 'Client-Side Auth Bypass', 'high',
  /\b(isAdmin|userRole|hasPermission|canEdit|canDelete|isOwner)\b.*&&.*Service\.(delete|update|create|write)/i,
  'Role check in UI but no server-side guard before service write. Authenticated users may bypass the UI.');

// SEC-008: PII in browser storage
// Removed bare 'user' — too generic (matches user-preferences, user-theme etc.)
// Kept specific PII/auth key names only.
scan('SEC-008', 'PII in Browser Storage', 'medium',
  /(localStorage|sessionStorage)\.setItem\s*\([^,]*(?:email|userId|userEmail|userName|token|auth|authToken|mail|profile|password|DisplayName)/i,
  'PII or auth data in localStorage/sessionStorage. Vulnerable to XSS exfiltration.');

// SEC-009: Unvalidated user input to generated service calls
// Removed 'value' and 'text' — too generic (match every service call with common param names).
// Kept unambiguously user-input-specific terms only.
scan('SEC-009', 'Unvalidated Input to Generated Service', 'medium',
  /Service\.\w+\s*\([^)]*\b(input|userInput|searchTerm|searchQuery|filterValue|rawInput)\b/i,
  'User input variable in generated service call. Validate and sanitize before passing to connector.');

// SEC-010: Sensitive variable names in useState
scan('SEC-010', 'Sensitive Data in State', 'low',
  /useState[^)]*\b(password|token|secret|apiKey|api_key)\b/i,
  'Sensitive data may be stored in React component state. Consider secure alternatives.');

// SEC-011: import.meta.env values bundled in public JS
scan('SEC-011', 'Sensitive Data in Published Bundle', 'high',
  /import\.meta\.env\.VITE_\w*(KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL|API)/i,
  'import.meta.env.VITE_* values are bundled into public JS at build time. Never store secrets in VITE_ env vars.');

// SEC-014: Dynamic code execution
scan('SEC-014', 'Dynamic Code Execution', 'critical',
  /\beval\s*\(/,
  'eval() executes a string as code. If the string can contain user-controlled content, this is a code injection vulnerability. Replace with a safer alternative.');

scan('SEC-014', 'Dynamic Code Execution', 'critical',
  /new\s+Function\s*\(/,
  'new Function() constructs executable code from a string — equivalent to eval(). Avoid or ensure the source string is never user-controlled.');

scan('SEC-014', 'Dynamic Code Execution', 'high',
  /(setTimeout|setInterval)\s*\(\s*['"`]/,
  'setTimeout/setInterval with a string argument evaluates it like eval(). Pass a function reference instead: setTimeout(() => doSomething(), delay).');

// SEC-015: Broader external HTTP clients
scan('SEC-015', 'Unsafe External HTTP Client', 'high',
  /axios\s*\.\s*(get|post|put|patch|delete|request)\s*\(\s*['"`]https?:\/\//i,
  'axios call to an external URL. Confirm it is an approved Microsoft/Power Platform endpoint.',
  /\.dynamics\.com|\.microsoft\.com|\.microsoftonline\.com/);

scan('SEC-015', 'Unsafe External HTTP Client', 'high',
  /new\s+XMLHttpRequest\s*\(\s*\)/,
  'XMLHttpRequest instantiation detected. Verify the request target is an approved Microsoft/Power Platform endpoint — use the generated service classes instead.',
  /\.dynamics\.com|\.microsoft\.com|\.microsoftonline\.com/);

scan('SEC-015', 'Unsafe External HTTP Client', 'high',
  /\$\s*\.\s*(ajax|get|post)\s*\(/,
  'jQuery HTTP call detected. jQuery is unexpected in a Code App — verify the target URL and consider replacing with generated service classes.');

// SEC-016: Sensitive data in URL parameters
scan('SEC-016', 'Sensitive URL Parameters', 'high',
  /useSearchParams[^;]*\b(token|auth|apikey|secret|password|credential)\b/i,
  'Sensitive parameter name read from URL query string. Tokens and credentials must never appear in URLs — they are logged by browsers, proxies, and servers.');

scan('SEC-016', 'Sensitive URL Parameters', 'high',
  /[?&](token|auth|apikey|api_key|secret|password|userId|user_id|credential)=/i,
  'Sensitive parameter embedded in a URL string. Credentials and user identifiers must not be passed as URL query parameters.');

const ALL_CATEGORIES = ['SEC-001','SEC-002','SEC-003','SEC-004','SEC-005','SEC-006','SEC-007','SEC-008','SEC-009','SEC-010','SEC-011','SEC-014','SEC-015','SEC-016'];

// SEC-012: power.config.json secret scan
try {
  const configContent = readFileSync(join(ROOT, 'power.config.json'), 'utf-8');
  const secretPattern = /(?:password|secret|token|apikey|api_key|client_secret|connectionstring)\s*[:=]\s*['"`][^'"`\s]{6,}/i;
  const labelPattern  = /(?:password|secret|token|apikey|api_key|client_secret|connectionstring)\s*[:=]\s*['"`][a-zA-Z$_][a-zA-Z$_\-]*['"`]/i;
  if (secretPattern.test(configContent) && !labelPattern.test(configContent)) {
    findings.push({
      id: 'SEC-012', category: 'power.config.json Secret', severity: 'critical',
      file: 'power.config.json', line: 0,
      snippet: 'Credential pattern detected in power.config.json',
      detail: 'power.config.json is committed to source control. Never store secrets here.',
    });
  }
} catch { /* power.config.json not found — skip */ }

// SEC-013: npm audit — known CVEs in installed dependencies
// npm audit exits with code 1 when vulnerabilities are found — catch is expected
try {
  const { execSync } = await import('child_process');
  const auditOut = execSync('npm audit --json', {
    cwd: ROOT, encoding: 'utf-8', stdio: 'pipe', timeout: 60_000,
  });
  const audit = JSON.parse(auditOut);
  const vulns = audit.vulnerabilities ?? {};
  for (const [pkg, v] of Object.entries(vulns as Record<string, any>)) {
    const sev: Severity = (v.severity === 'critical' || v.severity === 'high') ? v.severity : v.severity === 'moderate' ? 'medium' : 'low';
    if (sev === 'critical' || sev === 'high') {
      const via = (v.via ?? []).filter((x: any) => typeof x === 'object');
      const cve = via.map((x: any) => x.title ?? x.url ?? '').filter(Boolean).join(', ') || 'see npm audit for details';
      findings.push({
        id: 'SEC-013', category: 'Vulnerable Dependency', severity: sev,
        file: 'package.json', line: 0,
        snippet: `${pkg}@${v.range ?? 'unknown'} — ${sev}`,
        detail: `Known vulnerability in dependency "${pkg}". ${cve}. Run npm audit fix or update to a patched version.`,
      });
    }
  }
} catch (auditErr: any) {
  // npm audit exits 1 when vulns found — re-parse from stderr/stdout if available
  try {
    const out = auditErr.stdout ?? auditErr.output?.[1] ?? '';
    if (out) {
      const audit = JSON.parse(out);
      const vulns = audit.vulnerabilities ?? {};
      for (const [pkg, v] of Object.entries(vulns as Record<string, any>)) {
        const sev: Severity = v.severity === 'critical' ? 'critical' : v.severity === 'high' ? 'high' : v.severity === 'moderate' ? 'medium' : 'low';
        if (sev === 'critical' || sev === 'high') {
          const via = (v.via ?? []).filter((x: any) => typeof x === 'object');
          const cve = via.map((x: any) => x.title ?? x.url ?? '').filter(Boolean).join(', ') || 'see npm audit for details';
          findings.push({
            id: 'SEC-013', category: 'Vulnerable Dependency', severity: sev,
            file: 'package.json', line: 0,
            snippet: `${pkg}@${v.range ?? 'unknown'} — ${sev}`,
            detail: `Known vulnerability in dependency "${pkg}". ${cve}. Run npm audit fix or update to a patched version.`,
          });
        }
      }
    }
  } catch { /* npm audit not available or no package.json — skip */ }
}

const hitIds = new Set(findings.map(f => f.id));
const result: SecurityResult = {
  layer: 'security',
  passed: ALL_CATEGORIES.filter(c => !hitIds.has(c)).length,
  failed: hitIds.size,
  skipped: 0,
  findings,
};
process.stdout.write(JSON.stringify(result));

#!/usr/bin/env tsx
// evals/runner/run-evals.ts
// Orchestrates all eval layers and writes evals/results/latest.json

import { execSync } from 'child_process';
import { writeFileSync, mkdirSync, readFileSync, existsSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';

// ESM-safe __dirname
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

const ROOT = join(__dirname, '..');         // evals/runner → evals
const PROJECT_ROOT = join(ROOT, '..');      // evals → project root
const RESULTS_DIR = join(ROOT, 'results');

interface LayerResult {
  layer: 'presence' | 'unit' | 'security';
  passed: number; failed: number; skipped: number;
  details?: Array<{ featureId: string; name: string; status: 'pass' | 'fail' | 'skip'; detail?: string }>;
  findings?: any[];
}

interface EvalResults {
  timestamp: string;
  layers: LayerResult[];
  summary: {
    totalFeatures: number;
    allPresent: number;
    allBehavioral: number;
    openBRDGaps: string[];
    blockers: string[];
    securityFindings: Array<{ id: string; severity: string; category: string; file: string; line: number; detail: string }>;
  };
}

mkdirSync(RESULTS_DIR, { recursive: true });

const manifest = JSON.parse(readFileSync(join(ROOT, 'manifest.json'), 'utf-8'));

const results: EvalResults = {
  timestamp: new Date().toISOString(),
  layers: [],
  summary: {
    totalFeatures: manifest.features.length,
    allPresent: 0,
    allBehavioral: 0,
    openBRDGaps: manifest.openBRDGaps ?? [],
    blockers: [],
    securityFindings: [],
  },
};

// ── Layer 1: Presence Checks ──────────────────────────────────────────────────
console.log('\n🔍 Running presence checks...');
try {
  const presenceOut = execSync('npx tsx runner/presence-runner.ts', { cwd: ROOT, encoding: 'utf-8', timeout: 60_000 });
  const presenceResult: LayerResult = JSON.parse(presenceOut);
  results.layers.push(presenceResult);
  console.log(`  ✅ ${presenceResult.passed} passed  ❌ ${presenceResult.failed} failed`);
} catch (e) {
  console.error('  Presence checks failed to run:', e);
  results.layers.push({ layer: 'presence', passed: 0, failed: 1, skipped: 0, details: [] });
}

// ── Layer 2: Vitest Unit Tests ─────────────────────────────────────────────────
// IMPORTANT: execSync and JSON reading MUST be in separate try-catches.
// Vitest exits with code 1 when any tests fail or are todo — that must NOT prevent
// reading the JSON output file which is always written before vitest exits.
console.log('\n🧪 Running Vitest unit tests...');
try {
  execSync('npx vitest run --config evals/vitest.config.ts --passWithNoTests', {
    cwd: PROJECT_ROOT, encoding: 'utf-8', stdio: 'pipe', timeout: 120_000,
  });
} catch { /* vitest exits 1 on failures/todos — expected, JSON is still written */ }

const vitestPath = join(RESULTS_DIR, 'vitest-results.json');
if (existsSync(vitestPath)) {
  try {
    const vitestData = JSON.parse(readFileSync(vitestPath, 'utf-8'));
    const unitDetails: LayerResult['details'] = [];
    for (const suite of vitestData.testResults ?? []) {
      for (const t of suite.assertionResults ?? []) {
        const match = t.fullName.match(/([\w-]+):\s*(.+)$/);
        const featureId = match?.[1] ?? 'unknown';
        const name = match?.[2] ?? t.fullName;
        unitDetails!.push({
          featureId,
          name,
          status: t.status === 'passed' ? 'pass' : (t.status === 'pending' || t.status === 'todo') ? 'skip' : 'fail',
          detail: t.failureMessages?.join('\n') || undefined,
        });
      }
    }
    results.layers.push({
      layer: 'unit',
      passed: vitestData.numPassedTests ?? 0,
      failed: vitestData.numFailedTests ?? 0,
      skipped: vitestData.numTodoTests ?? 0,
      details: unitDetails,
    });
    console.log(`  🧪 ${vitestData.numPassedTests ?? 0} passed  ❌ ${vitestData.numFailedTests ?? 0} failed  ⏭ ${vitestData.numTodoTests ?? 0} todo`);
  } catch {
    results.layers.push({ layer: 'unit', passed: 0, failed: 1, skipped: 0, details: [] });
  }
} else {
  console.log('  ⚠️  No vitest-results.json found — evals/unit/ may be empty or missing');
  results.layers.push({ layer: 'unit', passed: 0, failed: 0, skipped: 0, details: [] });
}

// ── Layer 3: Security Analysis ────────────────────────────────────────────────
console.log('\n🔒 Running security analysis...');
try {
  const secOut = execSync('npx tsx runner/security-runner.ts', { cwd: ROOT, encoding: 'utf-8', stdio: 'pipe', timeout: 60_000 });
  const secResult = JSON.parse(secOut);
  results.layers.push(secResult);
  const critHigh = (secResult.findings ?? []).filter((f: any) => f.severity === 'critical' || f.severity === 'high').length;
  console.log(`  🔒 ${(secResult.findings ?? []).length} finding(s) — ${critHigh} critical/high`);
} catch (e) {
  console.error('  Security analysis failed to run:', e);
  results.layers.push({ layer: 'security', passed: 0, failed: 1, skipped: 0, findings: [] } as any);
}

// ── Compute summary ───────────────────────────────────────────────────────────
const allFailed = (results.layers as any[])
  .filter(l => l.layer === 'presence')
  .flatMap(l => (l.details ?? []).filter((d: any) => d.status === 'fail'));
results.summary.blockers = allFailed.map((d: any) => `${d.featureId}: ${d.name}`);
results.summary.securityFindings = ((results.layers as any[]).find(l => l.layer === 'security')?.findings ?? []);
results.summary.allPresent = (results.layers as any[]).find(l => l.layer === 'presence')?.passed ?? 0;
results.summary.allBehavioral = (results.layers as any[]).find(l => l.layer === 'unit')?.passed ?? 0;

const latestPath = join(RESULTS_DIR, 'latest.json');
writeFileSync(latestPath, JSON.stringify(results, null, 2));
console.log('\n📊 Results written to evals/results/latest.json');

// ── Bake results into dashboard HTML ─────────────────────────────────────────
const dashboardPath = join(ROOT, 'dashboard', 'index.html');
if (existsSync(dashboardPath)) {
  let html = readFileSync(dashboardPath, 'utf-8');
  const baked = JSON.stringify(results, null, 2);
  html = html.replace(/\/\* BAKED DATA[\s\S]*?END BAKED DATA \*\//, `/* BAKED DATA — replaced by npm run eval — do not edit manually */\nvar DATA = ${baked};\n/* END BAKED DATA */`);
  writeFileSync(dashboardPath, html);
  console.log('📊 Dashboard updated with baked results.');
}

// ── Exit code ─────────────────────────────────────────────────────────────────
const totalFailed = results.layers.reduce((sum, l) => sum + l.failed, 0);
if (totalFailed > 0) {
  console.error(`\n❌ ${totalFailed} eval(s) failed.`);
  process.exit(1);
}
console.log('\n✅ All evals passed.');

#!/usr/bin/env tsx
// evals/runner/presence-runner.ts
// Runs all presence checks and outputs a LayerResult JSON to stdout.

import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { globSync } from 'glob';

// ESM-safe __dirname
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const PRESENCE_DIR = join(__dirname, '..', 'presence');

interface CheckResult {
  featureId: string;
  checks: Array<{ name: string; passed: boolean; detail?: string }>;
}

interface LayerResult {
  layer: 'presence';
  passed: number; failed: number; skipped: number;
  details: Array<{ featureId: string; name: string; status: 'pass' | 'fail' | 'skip'; detail?: string }>;
}

const checkFiles = globSync(`${PRESENCE_DIR}/*.check.ts`).sort();
const details: LayerResult['details'] = [];
let passed = 0, failed = 0;

for (const file of checkFiles) {
  try {
    const mod = await import(file);
    const result: CheckResult = mod.check();
    for (const c of result.checks) {
      if (c.passed) {
        passed++;
        details.push({ featureId: result.featureId, name: c.name, status: 'pass' });
      } else {
        failed++;
        details.push({ featureId: result.featureId, name: c.name, status: 'fail', detail: c.detail });
      }
    }
  } catch (e: any) {
    failed++;
    details.push({ featureId: file, name: 'runner error', status: 'fail', detail: String(e?.message ?? e) });
  }
}

const result: LayerResult = { layer: 'presence', passed, failed, skipped: 0, details };
process.stdout.write(JSON.stringify(result));

path: ~/.copilot/m-skills/eval-generator-code-app/dashboard-template.html

     /* GENERATED — replace with actual features: { id, title, priority, note? } */

   { id: 'user-search', title: 'User Search', priority: 'high' },
   { id: 'dataverse-read', title: 'Dataverse Read', priority: 'high', note: 'Not yet implemented' },

{
  "scripts": {
    "eval": "tsx evals/runner/run-evals.ts",
    "eval:presence": "tsx evals/runner/presence-runner.ts",
    "eval:unit": "vitest run --config evals/vitest.config.ts",
    "eval:security": "tsx evals/runner/security-runner.ts",
    "eval:dashboard": "npx open-cli evals/dashboard/index.html"
  }
}

{
  "devDependencies": {
    "tsx": "^4.11.0",
    "glob": "^11.0.0",
    "vitest": "^2.0.0",
    "@vitejs/plugin-react": "^4.3.0",
    "jsdom": "^24.0.0",
    "@testing-library/react": "^16.0.0",
    "@testing-library/jest-dom": "^6.4.0"
  }
}

npx tsx evals/runner/run-evals.ts

npm run eval           # run all layers (presence + unit + security)
npm run eval:presence  # presence checks only
npm run eval:unit      # Vitest unit tests only
npm run eval:security  # security analysis only

/eval-generator-code-app
Generate evals for my Code App at C:\projects\my-code-app using requirements at C:\docs\BRD.md
Eval my code app against the current plan.md
Check feature completeness for the code app in my current working directory