#!/usr/bin/env python3 import argparse import hashlib import json from pathlib import Path import re import platform import shutil import statistics import subprocess import time parser = argparse.ArgumentParser() parser.add_argument('template', type=Path, help='Existing model runtime directory with namelists and input data') parser.add_argument('--repeats', type=int, default=3) parser.add_argument('--steps', type=int, default=1440) parser.add_argument('--reference', default='exoplasim-upstream-import') parser.add_argument('--timeout', type=int, default=180) parser.add_argument('--variants', nargs='+', choices=['checked', 'checked-candidate', 'performance', 'candidate'], default=['checked', 'checked-candidate', 'performance', 'candidate']) args = parser.parse_args() if args.repeats < 1 or args.steps < 1: parser.error('--repeats and --steps must be positive') root = Path(__file__).resolve().parents[1] work = root / 'tmp' / ('simulator-benchmark-' + time.strftime('%Y%m%dT%H%M%S')) work.mkdir(parents=True) template = args.template.resolve() build_template = root / 'exoplasim/plasim/bld' flags_by_mode = {} for mode in ['checked', 'performance']: config = work / ('config-' + mode) (config / 'plasim/bld').mkdir(parents=True) for filename in ['configure.sh', 'makecheck', 'f90check.f90', 'cc_check.c']: shutil.copy2(root / 'exoplasim' / filename, config / filename) command = ['bash', 'configure.sh', '--build-mode', mode] if mode == 'performance': command.append('--native') with (config / 'configure.log').open('w') as log: subprocess.run(command, cwd=config, stdout=log, stderr=subprocess.STDOUT, check=True, timeout=60) flags_by_mode[mode] = re.search(r'^MOST_F90_OPTS=(.*)$', (config / 'most_compiler').read_text(), re.M)[1] checked = flags_by_mode['checked'] performance = flags_by_mode['performance'] baseline = subprocess.check_output(['git', 'show', args.reference + ':exoplasim/plasim/src/legmod.f90'], cwd=root, timeout=20) current_source = (root / 'exoplasim/plasim/src/legmod.f90').read_bytes() variants = [('checked', checked, baseline), ('checked-candidate', checked, current_source), ('performance', performance, baseline), ('candidate', performance, current_source)] variants = [variant for variant in variants if variant[0] in args.variants] results = {'reference': args.reference, 'reference_commit': subprocess.check_output(['git', 'rev-parse', args.reference], cwd=root, timeout=20).decode().strip(), 'candidate_sha256': hashlib.sha256(current_source).hexdigest(), 'hostname': platform.node(), 'platform': platform.platform(), 'compiler': subprocess.check_output(['gfortran', '--version'], timeout=10).decode().splitlines()[0], 'steps': args.steps, 'repeats': args.repeats, 'template': str(template), 'variants': {}, 'runs': []} for name, flags, source in variants: build = work / ('build-' + name) shutil.copytree(build_template, build) (build / 'legmod.f90').write_bytes(source) with (work / (name + '-build.log')).open('w') as log: subprocess.run(['make', '-B', 'OCEANCOUP=cpl_stub', 'MOST_F90_OPTS=' + flags], cwd=build, stdout=log, stderr=subprocess.STDOUT, check=True, timeout=180) results['variants'][name] = {'flags': flags, 'executable': str(build / 'plasim.x')} for repeat in range(args.repeats): order = variants[repeat % len(variants):] + variants[:repeat % len(variants)] for name, _, _ in order: run = work / f'{name}-{repeat}' shutil.copytree(template, run) namelist = run / 'plasim_namelist' text = namelist.read_text() for key, value in [('N_RUN_YEARS', 0), ('N_RUN_MONTHS', 0), ('N_RUN_DAYS', 0), ('N_RUN_STEPS', args.steps)]: text, count = re.subn(r'(?im)\b' + key + r'\s*=\s*\d+', f'{key} = {value}', text) if count != 1: raise ValueError(f'Expected one {key} in {namelist}, found {count}') namelist.write_text(text) for filename in ['plasim_output', 'plasim_snapshot', 'plasim_status', 'Abort_Message']: (run / filename).unlink(missing_ok=True) timing = run / 'timing.txt' with (run / 'runtime.log').open('w') as log: subprocess.run([shutil.which('time'), '-f', '%U %S %e %M', '-o', str(timing), results['variants'][name]['executable']], cwd=run, stdout=log, stderr=subprocess.STDOUT, timeout=args.timeout, check=True) if (run / 'Abort_Message').exists(): raise RuntimeError(f'Model aborted: {run}') user, system, wall, rss = map(float, timing.read_text().split()) output = run / 'plasim_output' status = run / 'plasim_status' record = dict(variant=name, repeat=repeat, user_seconds=user, system_seconds=system, wall_seconds=wall, max_rss_kib=rss, directory=str(run), output_sha256=hashlib.sha256(output.read_bytes()).hexdigest(), restart_sha256=hashlib.sha256(status.read_bytes()).hexdigest()) results['runs'].append(record) print(json.dumps(record), flush=True) (work / 'results.json').write_text(json.dumps(results, indent=2) + '\n') for name, _, _ in variants: records = [r for r in results['runs'] if r['variant'] == name] results['variants'][name]['median_user_seconds'] = statistics.median(r['user_seconds'] for r in records) results['variants'][name]['median_wall_seconds'] = statistics.median(r['wall_seconds'] for r in records) results['variants'][name]['output_repeatable'] = len({r['output_sha256'] for r in records}) == 1 results['variants'][name]['restart_repeatable'] = len({r['restart_sha256'] for r in records}) == 1 import numpy as np from exoplasim.pyburn import readfile comparisons = [] pairs = [('checked-candidate', 'checked'), ('candidate', 'performance'), ('performance', 'checked')] for name, reference_name in pairs: if name not in args.variants or reference_name not in args.variants: continue reference_run = next(r for r in results['runs'] if r['variant'] == reference_name) record = next(r for r in results['runs'] if r['variant'] == name) reference_data = readfile(str(Path(reference_run['directory']) / 'plasim_output')) data = readfile(str(Path(record['directory']) / 'plasim_output')) if data.keys() != reference_data.keys(): raise AssertionError('Output field sets differ') fields = {} for key, value in reference_data.items(): value = np.asarray(value) current = np.asarray(data[key]) if current.shape != value.shape or not np.isfinite(current).all() or not np.isfinite(value).all(): raise AssertionError(f'Invalid field {key}') delta = float(np.max(np.abs(current.astype(float) - value.astype(float)))) fields[key] = dict(max_absolute_error=delta, scaled_error=delta / max(1.0, float(np.max(np.abs(value))))) comparisons.append(dict(variant=name, reference_variant=reference_name, raw_output_identical=record['output_sha256'] == reference_run['output_sha256'], restart_identical=record['restart_sha256'] == reference_run['restart_sha256'], fields=fields)) results['comparisons'] = comparisons (work / 'results.json').write_text(json.dumps(results, indent=2) + '\n') for name, summary in results['variants'].items(): if not summary['output_repeatable'] or not summary['restart_repeatable']: raise AssertionError(f'Repeated model state differs: {name}') for comparison in comparisons: if comparison['variant'] != 'performance' and max(field['scaled_error'] for field in comparison['fields'].values()) > 1e-10: raise AssertionError(f"Simulator numerical mismatch: {comparison['variant']} vs {comparison['reference_variant']}") print(json.dumps(results['variants'], indent=2)) print('Results:', work / 'results.json')