137 lines
7.9 KiB
Python
Executable File
137 lines
7.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import platform
|
|
import shutil
|
|
import statistics
|
|
import subprocess
|
|
import time
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('template', type=Path, help='Existing model runtime directory with namelists and input data')
|
|
parser.add_argument('--repeats', type=int, default=3)
|
|
parser.add_argument('--steps', type=int, default=1440)
|
|
parser.add_argument('--reference', default='exoplasim-upstream-import')
|
|
parser.add_argument('--timeout', type=int, default=180)
|
|
parser.add_argument('--variants', nargs='+', choices=['checked', 'checked-candidate', 'performance', 'candidate'],
|
|
default=['checked', 'checked-candidate', 'performance', 'candidate'])
|
|
args = parser.parse_args()
|
|
if args.repeats < 1 or args.steps < 1:
|
|
parser.error('--repeats and --steps must be positive')
|
|
root = Path(__file__).resolve().parents[1]
|
|
work = root / 'tmp' / ('simulator-benchmark-' + time.strftime('%Y%m%dT%H%M%S'))
|
|
work.mkdir(parents=True)
|
|
template = args.template.resolve()
|
|
build_template = root / 'exoplasim/plasim/bld'
|
|
flags_by_mode = {}
|
|
for mode in ['checked', 'performance']:
|
|
config = work / ('config-' + mode)
|
|
(config / 'plasim/bld').mkdir(parents=True)
|
|
for filename in ['configure.sh', 'makecheck', 'f90check.f90', 'cc_check.c']:
|
|
shutil.copy2(root / 'exoplasim' / filename, config / filename)
|
|
command = ['bash', 'configure.sh', '--build-mode', mode]
|
|
if mode == 'performance':
|
|
command.append('--native')
|
|
with (config / 'configure.log').open('w') as log:
|
|
subprocess.run(command, cwd=config, stdout=log, stderr=subprocess.STDOUT, check=True, timeout=60)
|
|
flags_by_mode[mode] = re.search(r'^MOST_F90_OPTS=(.*)$', (config / 'most_compiler').read_text(), re.M)[1]
|
|
checked = flags_by_mode['checked']
|
|
performance = flags_by_mode['performance']
|
|
baseline = subprocess.check_output(['git', 'show', args.reference + ':exoplasim/plasim/src/legmod.f90'], cwd=root, timeout=20)
|
|
current_source = (root / 'exoplasim/plasim/src/legmod.f90').read_bytes()
|
|
variants = [('checked', checked, baseline), ('checked-candidate', checked, current_source),
|
|
('performance', performance, baseline), ('candidate', performance, current_source)]
|
|
variants = [variant for variant in variants if variant[0] in args.variants]
|
|
results = {'reference': args.reference,
|
|
'reference_commit': subprocess.check_output(['git', 'rev-parse', args.reference], cwd=root, timeout=20).decode().strip(),
|
|
'candidate_sha256': hashlib.sha256(current_source).hexdigest(),
|
|
'hostname': platform.node(), 'platform': platform.platform(),
|
|
'compiler': subprocess.check_output(['gfortran', '--version'], timeout=10).decode().splitlines()[0],
|
|
'steps': args.steps, 'repeats': args.repeats,
|
|
'template': str(template), 'variants': {}, 'runs': []}
|
|
for name, flags, source in variants:
|
|
build = work / ('build-' + name)
|
|
shutil.copytree(build_template, build)
|
|
(build / 'legmod.f90').write_bytes(source)
|
|
with (work / (name + '-build.log')).open('w') as log:
|
|
subprocess.run(['make', '-B', 'OCEANCOUP=cpl_stub', 'MOST_F90_OPTS=' + flags],
|
|
cwd=build, stdout=log, stderr=subprocess.STDOUT, check=True, timeout=180)
|
|
results['variants'][name] = {'flags': flags, 'executable': str(build / 'plasim.x')}
|
|
|
|
for repeat in range(args.repeats):
|
|
order = variants[repeat % len(variants):] + variants[:repeat % len(variants)]
|
|
for name, _, _ in order:
|
|
run = work / f'{name}-{repeat}'
|
|
shutil.copytree(template, run)
|
|
namelist = run / 'plasim_namelist'
|
|
text = namelist.read_text()
|
|
for key, value in [('N_RUN_YEARS', 0), ('N_RUN_MONTHS', 0), ('N_RUN_DAYS', 0), ('N_RUN_STEPS', args.steps)]:
|
|
text, count = re.subn(r'(?im)\b' + key + r'\s*=\s*\d+', f'{key} = {value}', text)
|
|
if count != 1:
|
|
raise ValueError(f'Expected one {key} in {namelist}, found {count}')
|
|
namelist.write_text(text)
|
|
for filename in ['plasim_output', 'plasim_snapshot', 'plasim_status', 'Abort_Message']:
|
|
(run / filename).unlink(missing_ok=True)
|
|
timing = run / 'timing.txt'
|
|
with (run / 'runtime.log').open('w') as log:
|
|
subprocess.run([shutil.which('time'), '-f', '%U %S %e %M', '-o', str(timing),
|
|
results['variants'][name]['executable']], cwd=run, stdout=log,
|
|
stderr=subprocess.STDOUT, timeout=args.timeout, check=True)
|
|
if (run / 'Abort_Message').exists():
|
|
raise RuntimeError(f'Model aborted: {run}')
|
|
user, system, wall, rss = map(float, timing.read_text().split())
|
|
output = run / 'plasim_output'
|
|
status = run / 'plasim_status'
|
|
record = dict(variant=name, repeat=repeat, user_seconds=user, system_seconds=system,
|
|
wall_seconds=wall, max_rss_kib=rss, directory=str(run),
|
|
output_sha256=hashlib.sha256(output.read_bytes()).hexdigest(),
|
|
restart_sha256=hashlib.sha256(status.read_bytes()).hexdigest())
|
|
results['runs'].append(record)
|
|
print(json.dumps(record), flush=True)
|
|
(work / 'results.json').write_text(json.dumps(results, indent=2) + '\n')
|
|
|
|
for name, _, _ in variants:
|
|
records = [r for r in results['runs'] if r['variant'] == name]
|
|
results['variants'][name]['median_user_seconds'] = statistics.median(r['user_seconds'] for r in records)
|
|
results['variants'][name]['median_wall_seconds'] = statistics.median(r['wall_seconds'] for r in records)
|
|
results['variants'][name]['output_repeatable'] = len({r['output_sha256'] for r in records}) == 1
|
|
results['variants'][name]['restart_repeatable'] = len({r['restart_sha256'] for r in records}) == 1
|
|
|
|
import numpy as np
|
|
from exoplasim.pyburn import readfile
|
|
comparisons = []
|
|
pairs = [('checked-candidate', 'checked'), ('candidate', 'performance'), ('performance', 'checked')]
|
|
for name, reference_name in pairs:
|
|
if name not in args.variants or reference_name not in args.variants:
|
|
continue
|
|
reference_run = next(r for r in results['runs'] if r['variant'] == reference_name)
|
|
record = next(r for r in results['runs'] if r['variant'] == name)
|
|
reference_data = readfile(str(Path(reference_run['directory']) / 'plasim_output'))
|
|
data = readfile(str(Path(record['directory']) / 'plasim_output'))
|
|
if data.keys() != reference_data.keys():
|
|
raise AssertionError('Output field sets differ')
|
|
fields = {}
|
|
for key, value in reference_data.items():
|
|
value = np.asarray(value)
|
|
current = np.asarray(data[key])
|
|
if current.shape != value.shape or not np.isfinite(current).all() or not np.isfinite(value).all():
|
|
raise AssertionError(f'Invalid field {key}')
|
|
delta = float(np.max(np.abs(current.astype(float) - value.astype(float))))
|
|
fields[key] = dict(max_absolute_error=delta, scaled_error=delta / max(1.0, float(np.max(np.abs(value)))))
|
|
comparisons.append(dict(variant=name, reference_variant=reference_name,
|
|
raw_output_identical=record['output_sha256'] == reference_run['output_sha256'],
|
|
restart_identical=record['restart_sha256'] == reference_run['restart_sha256'], fields=fields))
|
|
results['comparisons'] = comparisons
|
|
(work / 'results.json').write_text(json.dumps(results, indent=2) + '\n')
|
|
for name, summary in results['variants'].items():
|
|
if not summary['output_repeatable'] or not summary['restart_repeatable']:
|
|
raise AssertionError(f'Repeated model state differs: {name}')
|
|
for comparison in comparisons:
|
|
if comparison['variant'] != 'performance' and max(field['scaled_error'] for field in comparison['fields'].values()) > 1e-10:
|
|
raise AssertionError(f"Simulator numerical mismatch: {comparison['variant']} vs {comparison['reference_variant']}")
|
|
print(json.dumps(results['variants'], indent=2))
|
|
print('Results:', work / 'results.json')
|