This is a cache of https://vccplus.brianholman.com/publications/vcc-plus-architecture/evidence/experiments/cmoc-benchmarks/benchmark-py/. It is a snapshot of the page as it appeared on 2026-11-21T00:02:58.833+0000.
experiments/cmoc-benchmarks/benchmark.py — supporting evidence · VCC Plus
← Supporting evidence

Evidence record

experiments/cmoc-benchmarks/benchmark.py

From Windows Application to Portable Emulation Platform

16,577 bytes

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Build and verify CoCo C benchmarks with an explicitly selected CMOC runtime."""
import argparse
import atexit
import hashlib
import importlib.util
import json
import math
import os
from pathlib import Path
import platform
import re
import shutil
import statistics
import subprocess
import sys
import tempfile
from datetime import datetime, timezone

ROOT = Path(__file__).resolve().parent
PROFILES = {
    'debug': ['-O0', '-fno-omit-frame-pointer'],
    'release': ['-O2'],
    'release_omit': ['-O2', '-fomit-frame-pointer'],
}
MODES = ['6809', '6309-emulation', '6309-native']


def sha(path):
    return hashlib.sha256(Path(path).read_bytes()).hexdigest()


def run(command, *, cwd=None, env=None, log=None):
    result = subprocess.run([str(x) for x in command], cwd=cwd, env=env,
                            capture_output=True, text=True, timeout=180)
    if log:
        Path(log).write_text(result.stdout + result.stderr)
    if result.returncode:
        raise RuntimeError(f'Command failed ({result.returncode}): {command[0]}\n' + result.stdout + result.stderr)
    return result.stdout


def image_sizes(binary, linkmap):
    data = Path(binary).read_bytes()
    offset = payload = 0
    spans = []
    while offset < len(data):
        if len(data) - offset < 5:
            raise ValueError('Truncated DECB block')
        kind = data[offset]
        length = int.from_bytes(data[offset + 1:offset + 3], 'big')
        address = int.from_bytes(data[offset + 3:offset + 5], 'big')
        offset += 5
        if kind == 255:
            if length or offset != len(data):
                raise ValueError('Malformed DECB execution record')
            break
        if kind or offset + length > len(data) or address + length > 65536:
            raise ValueError('Malformed DECB data block')
        payload += length
        spans.append((address, address + length))
        offset += length
    else:
        raise ValueError('Missing DECB execution record')
    sections = re.findall(r'^Section: (\S+) \((.*?)\) load at ([0-9A-Fa-f]+), length ([0-9A-Fa-f]+)$',
                          linkmap, re.M)
    if not sections or not spans:
        raise ValueError('Missing linker sections or payload')
    allocated = [(int(a, 16), int(a, 16) + int(n, 16)) for _, _, a, n in sections]
    return {
        'bin_file_bytes': len(data),
        'initialized_bytes': payload,
        'bss_bytes': sum(int(n, 16) for name, _, _, n in sections if name == 'bss'),
        'linked_code_bytes': sum(int(n, 16) for name, _, _, n in sections if name in ('code', 'start')),
        'static_ram_span_bytes': max(b for _, b in allocated) - min(a for a, _ in allocated),
    }


def symbols(text):
    return {name: int(address, 16) for name, address in
            re.findall(r'^Symbol: (\S+) \(.*?\) = ([0-9A-Fa-f]+)$', text, re.M)}


def execute(runner, rom, binary, points, output, count, mode, log):
    return json.loads(run([runner, rom, binary, *(f'{p:x}' for p in points),
                           f'{output:x}', count, mode], log=log))


def calibrate(runtime, runner, rom, folder, env, inputs):
    receipts = []
    for mode in MODES:
        work = folder / mode
        work.mkdir(parents=True)
        source = (inputs / 'calibrate.asm').read_text()
        if mode == '6309-native':
            source = source.replace('fcb $12,$12,$12', 'fcb $11,$3d,$01')
        (work / 'calibrate.asm').write_text(source)
        run([runtime / 'bin/lwasm', '--6309', '--decb', '-o', 'CAL.BIN', 'calibrate.asm'],
            cwd=work, env=env, log=work / 'build.log')
        result = execute(runner, rom, work / 'CAL.BIN', [0x2805, 0x2808, 0x2809],
                         0x2800, 1, mode, work / 'run.log')
        expected_cycles = 3 if mode == '6309-native' else 6
        if result['cycles'] != expected_cycles or result['output'] != [0x1a]:
            raise RuntimeError(f'Cycle calibration failed: {mode}: {result}')
        receipts.append({'cpu': mode, 'cycles': result['cycles'], 'expected_cycles': expected_cycles})
    return receipts


def summarize(record):
    """Only correct, deterministic complete matrices may receive a recommendation."""
    grouped = {}
    for row in record['runs']:
        key = row['host_arch'], row['cpu'], row['case'], row['profile']
        grouped.setdefault(key, []).append(row)
    result = []
    for key, rows in sorted(grouped.items()):
        host, cpu, case, profile = key
        # Sum one timing per seed: no statistical noise or repeat over-weighting.
        seeds = sorted({r['seed'] for r in rows})
        cycles = [next(r['cycles'] for r in rows if r['seed'] == seed) for seed in seeds]
        result.append(dict(host_arch=host, cpu=cpu, case=case, profile=profile,
                           mean_cycles=statistics.mean(cycles),
                           initialized_bytes=statistics.mean(r['sizes']['initialized_bytes'] for r in rows),
                           static_ram_span_bytes=statistics.mean(r['sizes']['static_ram_span_bytes'] for r in rows)))
    return result


def markdown(record):
    rows = summarize(record)
    host, cpu = record['host_arches'][0], record['cpu_modes'][0]
    lookup = {(r['case'], r['profile']): r for r in rows if r['host_arch'] == host and r['cpu'] == cpu}
    lines = ['# CoCo CMOC benchmark results', '',
             'Generated from verified executions; retained JSON contains every seed, repeat, CPU mode and host compiler slice.', '',
             f"Compiler: CMOC {record['toolchain']['cmoc']}; SHA-256 `{record['tool_hashes']['libexec/cmoc']}`.",
             "VCC source inputs and linked core are identified by SHA-256 in the numerical record.", '',
             f'## {cpu} measurements', '',
             'Cycles are means over input seeds, with deterministic repeats checked. Sizes include linked runtime helpers, startup, setup and result collection; they are initialized payload bytes, excluding DECB headers and BSS. Full static RAM spans are in JSON and exclude stack/heap reservations.', '',
             '| Workload | Debug cycles | Release cycles | Omit cycles | Omit speed gain | Release bytes | Omit bytes |',
             '| --- | ---: | ---: | ---: | ---: | ---: | ---: |']
    for case in record['cases']:
        d, r, o = [lookup[case, p] for p in PROFILES]
        gain = 100 * (1 - o['mean_cycles'] / r['mean_cycles'])
        lines.append(f"| {case} | {d['mean_cycles']:,.0f} | {r['mean_cycles']:,.0f} | {o['mean_cycles']:,.0f} | {gain:.1f}% | {r['initialized_bytes']:,.0f} | {o['initialized_bytes']:,.0f} |")
    lines += ['', '## Release versus frame-pointer omission', '']
    for arch in record['host_arches']:
        for mode in record['cpu_modes']:
            subset = {(r['case'], r['profile']): r for r in rows if r['host_arch'] == arch and r['cpu'] == mode}
            ratios = [subset[c, 'release_omit']['mean_cycles'] / subset[c, 'release']['mean_cycles'] for c in record['cases']]
            geometric = math.exp(statistics.mean(math.log(x) for x in ratios))
            total_r = sum(subset[c, 'release']['initialized_bytes'] for c in record['cases'])
            total_o = sum(subset[c, 'release_omit']['initialized_bytes'] for c in record['cases'])
            lines.append(f'- {arch}, {mode}: equal-workload geometric mean uses {100 * (1-geometric):.2f}% fewer cycles; summed initialized payload uses {100 * (1-total_o/total_r):.2f}% fewer bytes.')
    lines += ['', 'These are CPU workloads over synthetic application buffers. They do not measure device throughput, audio playback, disk latency, complete-game frame rate, physical hardware, or debugger compatibility. See README.md for the decision criteria and methodology.', '']
    return '\n'.join(lines)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--runtime', type=Path, required=True)
    parser.add_argument('--vcc-source', type=Path, required=True)
    parser.add_argument('--core', type=Path, required=True)
    parser.add_argument('--rom', type=Path, required=True)
    parser.add_argument('--output', type=Path, required=True, help='new directory; existing results are never overwritten')
    parser.add_argument('--host-arches', nargs='+', choices=['arm64', 'x86_64'], default=[platform.machine()])
    parser.add_argument('--seeds', nargs='+', type=int, default=[1, 165, 65521])
    parser.add_argument('--repeats', type=int, default=2)
    parser.add_argument('--cases', nargs='+', help='optional workload-name subset')
    parser.add_argument('--cpu-modes', nargs='+', choices=MODES, default=MODES)
    args = parser.parse_args()
    if args.repeats < 2 or args.repeats > 10 or not args.seeds or any(s < 0 or s > 65535 for s in args.seeds):
        parser.error('Use 2..10 repeats and 16-bit unsigned seeds')
    for field in ('host_arches', 'seeds', 'cpu_modes'):
        if len(getattr(args, field)) != len(set(getattr(args, field))):
            parser.error('Duplicate ' + field)
    runtime, core, rom, vcc = [p.resolve(strict=True) for p in (args.runtime, args.core, args.rom, args.vcc_source)]
    output = args.output.resolve()
    output.mkdir(parents=True, exist_ok=False)
    inputs = output / 'inputs'
    inputs.mkdir()
    for source in ['main.c','workloads.c','workloads.h','reference.py','runner.cpp','benchmark.py','calibrate.asm']:
        shutil.copyfile(ROOT / source, inputs / source)
    spec = importlib.util.spec_from_file_location('frozen_reference', inputs / 'reference.py')
    model = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(model)
    CASES, expected = model.CASES, model.expected
    names = args.cases or CASES
    if len(set(names)) != len(names) or any(n not in CASES for n in names):
        parser.error('Unknown or duplicate workload')
    env = dict(os.environ, PATH=str(runtime / 'bin') + os.pathsep + '/usr/bin:/bin')
    for key in ('CMOCPKGDATADIR', 'CMOCFLOATLIBDIR', 'VCC_CMOC_COMPAT', 'CPATH', 'C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH'):
        env.pop(key, None)
    # Reproduce bin/cmoc's space-free data alias, but select the exact compiler
    # Mach-O slice explicitly instead of relying on shell architecture inheritance.
    alias = tempfile.TemporaryDirectory(prefix='coco-benchmark-cmoc-')
    atexit.register(alias.cleanup)
    data_alias = Path(alias.name) / 'cmoc'
    data_alias.symlink_to(runtime / 'share/cmoc', target_is_directory=True)
    env['CMOCPKGDATADIR'] = str(data_alias)
    env['CMOCFLOATLIBDIR'] = str(data_alias / 'lib')
    toolchain = json.loads((runtime / 'manifest.json').read_text())
    tools = ['libexec/cmoc', 'bin/cmoc', 'bin/cpp', 'bin/mcpp', 'bin/lwasm', 'bin/lwlink']
    tools += sorted(str(p.relative_to(runtime)) for p in (runtime / 'share/cmoc/lib').glob('libcmoc-*.a'))
    record = dict(schema_version=1, timestamp_utc=datetime.now(timezone.utc).isoformat(),
                  toolchain={k:v for k,v in toolchain.items() if k in ('cmoc','lwtools','mcpp') or k.startswith('vcc')},
                  tool_hashes={p:sha(runtime / p) for p in tools}, core_sha256=sha(core),
                  source_repository_url='https://vccplus-git.brianholman.com/VCCPlus/VCCPlusOSS/src/branch/main/',
                  vcc_source_inputs_sha256={p.relative_to(vcc).as_posix():sha(p)
                      for folder in (vcc / 'src/core', vcc / 'src/devices')
                      for p in sorted(folder.rglob('*')) if p.is_file()},
                  source_hashes={p:sha(inputs / p) for p in ['main.c','workloads.c','workloads.h','reference.py','runner.cpp','benchmark.py','calibrate.asm']},
                  compiler_slice_selection='explicit arch invocation of shipped libexec/cmoc with wrapper-equivalent environment',
                  host_arches=args.host_arches, cpu_modes=args.cpu_modes, seeds=args.seeds,
                  repeats=args.repeats, profiles=PROFILES, cases=names, runs=[])
    runner = output / 'runner'
    run(['xcrun','clang++','-std=c++17','-O2','-mmacosx-version-min=14.0',
         '-I',vcc / 'src/core',inputs / 'runner.cpp',core,'-o',runner], log=output / 'runner-build.log')
    record['calibration'] = calibrate(runtime, runner, rom, output / 'calibration', env, inputs)
    print('Cycle calibration passed on all three CPU modes.', flush=True)
    for arch in args.host_arches:
        for name in names:
            case = CASES.index(name)
            for profile, flags in PROFILES.items():
                for native in sorted({m == '6309-native' for m in args.cpu_modes}):
                    for seed in args.seeds:
                        work = output / arch / name / profile / str(int(native)) / str(seed)
                        work.mkdir(parents=True)
                        for source in ('main.c','workloads.c','workloads.h'):
                            shutil.copyfile(inputs / source, work / source)
                        compiler = ['/usr/bin/arch', '-' + arch, runtime / 'libexec/cmoc']
                        common = ['--coco', '--intermediate', '--deps', '--stack-space=2048', '-Werror', *flags,
                                  f'-DBENCH_CASE={case}', f'-DBENCH_SEED={seed}', f'-DBENCH_NATIVE={int(native)}']
                        for source in ('main', 'workloads'):
                            run([*compiler,*common,'-c','-o',source+'.o',source+'.c'], cwd=work, env=env, log=work / (source+'-build.log'))
                        # Match generated project defaults: relocatable CoCo target, no forced switch strategy.
                        run([*compiler,'--coco','--intermediate','--org=2800','--limit=7800','-o','BENCH.BIN','main.o','workloads.o'],
                            cwd=work, env=env, log=work / 'link.log')
                        binary = work / 'BENCH.BIN'
                        linkmap = (work / 'BENCH.map').read_text()
                        syms = symbols(linkmap)
                        points = [syms[n] for n in ('bench_begin', 'bench_end', 'bench_done')]
                        result_address = syms['_bench_output']
                        sizes = image_sizes(binary, linkmap)
                        for mode in [m for m in args.cpu_modes if (m == '6309-native') == native]:
                            timing = None
                            want = expected(seed, case)
                            for repeat in range(args.repeats):
                                receipt = execute(runner,rom,binary,points,result_address,len(want),mode,work / f'{mode}-{repeat}.log')
                                if receipt['output'] != want:
                                    raise RuntimeError(f'Incorrect output: {arch}/{name}/{profile}/{mode}/seed={seed}: expected {want}, received {receipt["output"]}')
                                if timing is not None and receipt['cycles'] != timing:
                                    raise RuntimeError('Nondeterministic cycle count')
                                timing = receipt['cycles']
                                record['runs'].append(dict(host_arch=arch,case=name,profile=profile,cpu=mode,seed=seed,repeat=repeat,
                                                          cycles=timing,sizes=sizes,binary_sha256=sha(binary),output=receipt['output'],md=receipt['md']))
                            print(f'PASS {arch} {name} {profile} {mode} seed={seed}: {timing:,} cycles, {sizes["initialized_bytes"]} bytes', flush=True)
                        (output / 'progress.json').write_text(json.dumps(record, indent=2)+'\n')
    # Host slices must emit identical guest binaries for identical builds.
    binaries = {}
    for row in record['runs']:
        key = row['case'],row['profile'],row['cpu'],row['seed']
        if key in binaries and binaries[key] != row['binary_sha256']:
            raise RuntimeError('Host compiler slices emitted different guest binaries')
        binaries[key] = row['binary_sha256']
    if any(sha(vcc / p) != expected for p, expected in record['vcc_source_inputs_sha256'].items()):
        raise RuntimeError('Source inputs changed during measurement')
    record['status'] = 'passed'
    (output / 'results.json').write_text(json.dumps(record, indent=2)+'\n')
    (output / 'RESULTS.md').write_text(markdown(record))
    alias.cleanup()
    print(f'PASS: {len(record["runs"])} verified executions. Results: {output / "RESULTS.md"}', flush=True)


if __name__ == '__main__':
    try:
        main()
    except (RuntimeError, ValueError, OSError, subprocess.SubprocessError) as error:
        print(f'FAIL: {error}', file=sys.stderr)
        sys.exit(1)

SHA-25669cf1bf4d620564f98e355ba25a6632f94f4200f10894b4b5d4de2b5c8851356

A closer look

Open original