# Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 # # Subject to the condition set forth below, permission is hereby granted to any # person obtaining a copy of this software, associated documentation and/or # data (collectively the "Software"), free of charge and under any and all # copyright rights in the Software, and any and all patent rights owned or # freely licensable by each licensor hereunder covering either (i) the # unmodified Software as contributed to or provided by such licensor, or (ii) # the Larger Works (as defined below), to deal in both # # (a) the Software, and # # (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if # one is included with the Software each a "Larger Work" to which the Software # is contributed by such licensors), # # without restriction, including without limitation the rights to copy, create # derivative works of, display, perform, and distribute the Software and make, # use, sell, offer for sale, import, export, have made, and have sold the # Software and the Larger Work(s), and to sublicense the foregoing rights on # either these or other terms. # # This license is subject to the following condition: # # The above copyright notice and either this complete permission notice or at a # minimum a reference to the UPL must be included in all copies or substantial # portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import mx import mx_benchmark import glob import json import math import os import re import shutil import subprocess import sys from os.path import join, abspath, exists SUITE = None python_vm_registry = None # By default we disabled some benchmarks, both because some don't run and # because we want to reduce the total runtime of the suites. DEFAULT_NUMPY_BENCHMARKS = [ "bench_app", "bench_core", # "bench_function_base", "bench_indexing", # "bench_io", "bench_linalg", # "bench_ma", # "bench_random", "bench_reduce", # "bench_shape_base", # "bench_ufunc", ] SKIPPED_NUMPY_BENCHMARKS = [ "bench_core.CountNonzero.time_count_nonzero(2, 1000000, )", # Times out "bench_core.CountNonzero.time_count_nonzero(3, 1000000, )", # Times out "bench_core.CountNonzero.time_count_nonzero_axis(2, 1000000, )", # Times out "bench_core.CountNonzero.time_count_nonzero_axis(3, 1000000, )", # Times out "bench_core.CountNonzero.time_count_nonzero_multi_axis(2, 1000000, )", # Times out "bench_core.CountNonzero.time_count_nonzero_multi_axis(3, 1000000, )", # Times out "bench_linalg.LinalgSmallArrays.time_det_small_array", # TODO fails with numpy.linalg.LinAlgError ] DEFAULT_PANDAS_BENCHMARKS = [ "reshape", "replace" ] SKIPPED_PANDAS_BENCHMARKS = [ "replace.ReplaceDict.time_replace_series", # Times out "replace.ReplaceList.time_replace_list", # OOM, WIP msimacek "replace.ReplaceList.time_replace_list_one_match", # OOM, WIP msimacek "reshape.Crosstab.time_crosstab_normalize_margins", # Times out "reshape.Cut.peakmem_cut_interval", # Times out "reshape.Cut.time_cut_interval", # Times out "reshape.GetDummies.time_get_dummies_1d_sparse", # Times out "reshape.PivotTable.time_pivot_table_margins", # Times out "reshape.WideToLong.time_wide_to_long_big", # Times out "reshape.Cut.time_qcut_datetime", # Transient failure GR-61245, exit code -11 "reshape.Explode.time_explode", # Transient failure GR-61245, exit code -11 ] DEFAULT_PYPERFORMANCE_BENCHMARKS = [ # "2to3", # "chameleon", "chaos", # "crypto_pyaes", # "django_template", # "dulwich_log", "fannkuch", "float", "go", "hexiom", # "html5lib", "json_dumps", "json_loads", # "logging", # "mako", "meteor_contest", "nbody", "nqueens", "pathlib", "pickle", "pickle_dict", "pickle_list", "pickle_pure_python", "pidigits", "pyflate", "regex_compile", "regex_dna", "regex_effbot", "regex_v8", "richards", "scimark", "spectral_norm", # "sqlalchemy_declarative", # "sqlalchemy_imperative", # "sqlite_synth", # "sympy", "telco", # "tornado_http", "unpack_sequence", "unpickle", "unpickle_list", # "unpickle_pure_python", # "xml_etree", ] DEFAULT_PYPY_BENCHMARKS = [ "ai", # "bm_chameleon", # "bm_dulwich_log", "bm_mako", "bm_mdp", "chaos", # "cpython_doc", "crypto_pyaes", "deltablue", "django", "eparse", "fannkuch", "float", "genshi_text", "genshi_xml", "go", "hexiom2", "html5lib", "json_bench", "meteor-contest", "nbody_modified", "nqueens", "pidigits", "pyflate-fast", "pyxl_bench", "raytrace-simple", "richards", "scimark_fft", "scimark_lu", "scimark_montecarlo", "scimark_sor", "scimark_sparsematmult", "spectral-norm", "spitfire2", "spitfire_cstringio2", # "sqlalchemy_declarative", # "sqlalchemy_imperative", # "sqlitesynth", # "sympy_expand", # "sympy_integrate", # "sympy_str", # "sympy_sum", # "telco", # "twisted_names", # "twisted_pb", # "twisted_tcp", ] def create_asv_benchmark_selection(benchmarks, skipped=()): regex = '|'.join(benchmarks) if not skipped: return regex negative_lookaheads = [re.escape(skip) + (r'\b' if not skip.endswith(')') else '') for skip in skipped] return '^(?!' + '|'.join(negative_lookaheads) + ')(' + regex + ')' class PyPerfJsonRule(mx_benchmark.Rule): """Parses a JSON file produced by PyPerf and creates a measurement result.""" def __init__(self, filenames: str, suiteName: str): self.filenames = filenames.split(",") self.suiteName = suiteName def parse(self, text: str) -> list: r = [] for filename in self.filenames: self._parse_file(r, filename) return r def _parse_file(self, r: list, filename: str): with open(self._prepend_working_dir(filename)) as fp: js = json.load(fp) benchmarks = js["benchmarks"] for benchmark in benchmarks: name = benchmark.get("metadata", js["metadata"])["name"] unit = benchmark.get("metadata", {}).get("unit") or js["metadata"]["unit"] unit = { "second": "s", "byte": "B", }[unit] metric = { "s": "time", "B": "max-rss", }[unit] for run in benchmark["runs"]: if values := run.get("values", None): if metric == "time": warmups = run.get("warmups", []) for idx, warmup in enumerate(warmups): r.append( { "bench-suite": self.suiteName, "benchmark": name, "metric.name": "warmup", "metric.unit": unit, "metric.score-function": "id", "metric.better": "lower", "metric.type": "numeric", "metric.iteration": idx, "metric.value": warmup[1], # 0 is inner_loop count } ) for value in values: r.append( { "bench-suite": self.suiteName, "benchmark": name, "metric.name": metric, "metric.unit": unit, "metric.score-function": "id", "metric.better": "lower", "metric.type": "numeric", "metric.iteration": 0, "metric.value": value, } ) class AsvJsonRule(mx_benchmark.Rule): """Parses a JSON file produced by ASV (airspeed-velocity) and creates a measurement result.""" def __init__(self, filename: str, suiteName: str): self.filename = filename self.suiteName = suiteName def parse(self, text: str) -> list: import itertools r = [] with open(self._prepend_working_dir(self.filename)) as fp: js = json.load(fp) columns = js["result_columns"] # type: list[str] peak_idx = columns.index("result") param_idx = columns.index("params") try: samples_idx = columns.index("samples") except ValueError: samples_idx = -1 for benchmark, result in js["results"].items(): param_combinations = itertools.product(*result[param_idx]) for run_idx, params in enumerate(param_combinations): peak_values = result[peak_idx] if not peak_values: continue value = peak_values[run_idx] if not value or math.isnan(value): continue r.append( { "bench-suite": self.suiteName, "benchmark": benchmark, "metric.name": "time", "metric.unit": "s", "metric.score-function": "id", "metric.better": "lower", "metric.type": "numeric", "metric.iteration": 0, "metric.value": value, "config.run-flags": " ".join(params), } ) # It may be that the samples are missing; so omit this step if 0 <= samples_idx < len(result): for iteration, value in enumerate(result[samples_idx][run_idx]): r.append( { "bench-suite": self.suiteName, "benchmark": benchmark, "metric.name": "warmup", "metric.unit": "s", "metric.score-function": "id", "metric.better": "lower", "metric.type": "numeric", "metric.iteration": iteration, "metric.value": value, "config.run-flags": " ".join(params), } ) return r class PyPyJsonRule(mx_benchmark.Rule, mx_benchmark.AveragingBenchmarkMixin): """Parses a JSON file produced by the Unladen Swallow or PyPy benchmark harness and creates a measurement result.""" def __init__(self, filename: str, suiteName: str): self.filename = filename self.suiteName = suiteName def parse(self, text: str) -> list: r = [] with open(self._prepend_working_dir(self.filename)) as fp: js = json.load(fp) for result in js["results"]: name = result[0] if result[1] == "RawResult": values = result[2]["base_times"] elif result[1] == "SimpleComparisonResult": values = [result[2]["base_time"]] else: mx.warn(f"No data found for {name} with {result[1]}") continue for iteration, value in enumerate(values): r.append( { "bench-suite": self.suiteName, "benchmark": name, "metric.name": "warmup", "metric.unit": "s", "metric.score-function": "id", "metric.better": "lower", "metric.type": "numeric", "metric.iteration": iteration, "metric.value": value, } ) self.addAverageAcrossLatestResults(r) return r class GraalPyVm(mx_benchmark.GuestVm): def __init__(self, config_name, options, host_vm=None): super(GraalPyVm, self).__init__(host_vm=host_vm) self._config_name = config_name self._options = options def name(self): return "graalpython" def config_name(self): return self._config_name def hosting_registry(self): return mx_benchmark.java_vm_registry def with_host_vm(self, host_vm): return self.__class__(self.config_name(), self._options, host_vm) def run(self, cwd, args): for arg in args: if "--vm.Xmx" in arg: mx.log(f"Setting Xmx from {arg}") break else: xmxArg = "--vm.Xmx8G" mx.log(f"Setting Xmx as {xmxArg}") args.insert(0, xmxArg) try: old_gp_arg = os.environ.get("GRAAL_PYTHON_ARGS") if old_gp_arg: os.environ["GRAAL_PYTHON_ARGS"] = old_gp_arg + " " + xmxArg else: os.environ["GRAAL_PYTHON_ARGS"] = xmxArg old_java_opts = os.environ.get("JAVA_OPTS") if old_java_opts: os.environ["JAVA_OPTS"] = old_java_opts + " " + xmxArg.replace("--vm", "-") else: os.environ["JAVA_OPTS"] = xmxArg.replace("--vm.", "-") mx.log("Running with `JAVA_OPTS={JAVA_OPTS}` and `GRAAL_PYTHON_ARGS={GRAAL_PYTHON_ARGS}`".format(**os.environ)) return self.host_vm().run_launcher("graalpy", self._options + args, cwd) finally: if old_java_opts: os.environ["JAVA_OPTS"] = old_java_opts else: del os.environ["JAVA_OPTS"] if old_gp_arg: os.environ["GRAAL_PYTHON_ARGS"] = old_gp_arg else: del os.environ["GRAAL_PYTHON_ARGS"] class PyPyVm(mx_benchmark.Vm): def config_name(self): return "launcher" def name(self): return "pypy" def interpreter(self): home = mx.get_env("PYPY_HOME") if not home: try: return ( subprocess.check_output("which pypy3", shell=True).decode().strip() ) except OSError: mx.abort("{} is not set!".format("PYPY_HOME")) return join(home, "bin", "pypy3") def run(self, cwd, args): env = os.environ.copy() xmxArg = re.compile("--vm.Xmx([0-9]+)([kKgGmM])") pypyGcMax = "8GB" for idx, arg in enumerate(args): if m := xmxArg.search(arg): args = args[:idx] + args[idx + 1 :] pypyGcMax = f"{m.group(1)}{m.group(2).upper()}B" mx.log(f"Setting PYPY_GC_MAX={pypyGcMax} via {arg}") break else: mx.log( f"Setting PYPY_GC_MAX={pypyGcMax}, use --vm.Xmx argument to override it" ) env["PYPY_GC_MAX"] = pypyGcMax return mx.run([self.interpreter()] + args, cwd=cwd, env=env) class Python3Vm(mx_benchmark.Vm): def config_name(self): return "launcher" def name(self): return "cpython" def interpreter(self): home = mx.get_env("PYTHON3_HOME") if not home: return sys.executable if exists(exe := join(home, "bin", "python3")): return exe elif exists(exe := join(home, "python3")): return exe elif exists(exe := join(home, "python")): return exe return join(home, "bin", "python") def run(self, cwd, args): for idx, arg in enumerate(args): if "--vm.Xmx" in arg: mx.warn(f"Ignoring {arg}, cannot restrict memory on CPython.") args = args[:idx] + args[idx + 1 :] break return mx.run([self.interpreter()] + args, cwd=cwd) class WildcardList: """It is not easy to track for external suites which benchmarks are available, so we just return a wildcard list and assume the caller knows what they want to run""" def __init__(self, benchmarks=None): self.benchmarks = benchmarks def __contains__(self, x): return True def __iter__(self): if not self.benchmarks: mx.abort( "Cannot iterate over benchmark names in foreign benchmark suites. " + "Leave off the benchmark name part to run all, or name the benchmarks yourself." ) else: return iter(self.benchmarks) class PySuite(mx_benchmark.TemporaryWorkdirMixin, mx_benchmark.VmBenchmarkSuite): def runAndReturnStdOut(self, benchmarks, bmSuiteArgs): ret_code, out, dims = super().runAndReturnStdOut(benchmarks, bmSuiteArgs) def _replace_host_vm(old, new): host_vm = dims.get("host-vm") if host_vm and old in host_vm: dims['host-vm'] = host_vm.replace(old, new) mx.logv(f"[DEBUG] replace 'host-vm': '{host_vm}' -> '{dims['host-vm']}'") _replace_host_vm('graalvm-ce-python', 'graalvm-ce') _replace_host_vm('graalvm-ee-python', 'graalvm-ee') return ret_code, out, dims class PyPerformanceSuite(PySuite): VERSION = "1.0.6" def name(self): return "pyperformance-suite" def group(self): return "Graal" def subgroup(self): return "graalpython" def benchmarkList(self, bmSuiteArgs): return WildcardList(DEFAULT_PYPERFORMANCE_BENCHMARKS) def rules(self, output, benchmarks, bmSuiteArgs): return [PyPerfJsonRule(output, self.name())] def createVmCommandLineArgs(self, benchmarks, runArgs): return [] def get_vm_registry(self): return python_vm_registry def _vmRun(self, vm, workdir, command, benchmarks, bmSuiteArgs): workdir = abspath(workdir) vm_venv = f"{self.name()}-{vm.name()}-{vm.config_name()}" if not hasattr(self, "prepared"): self.prepared = True vm.run(workdir, ["-m", "venv", join(workdir, vm_venv)]) mx.run( [ join(vm_venv, "bin", "pip"), "install", f"pyperformance=={self.VERSION}", ], cwd=workdir, ) if benchmarks: bms = ["-b", ",".join(benchmarks)] else: bms = ["-b", ",".join(DEFAULT_PYPERFORMANCE_BENCHMARKS)] json_file = f"{vm_venv}.json" retcode = mx.run( [ join(vm_venv, "bin", "pyperformance"), "run", "--inherit-environ", "PIP_INDEX_URL,PIP_EXTRA_INDEX_URL,PIP_TRUSTED_HOST,PIP_TIMEOUT,PIP_RETRIES,LD_LIBRARY_PATH,LIBRARY_PATH,CPATH,PATH,PYPY_GC_MAX,JAVA_OPTS,GRAAL_PYTHON_ARGS", "-o", json_file, *bms, ], cwd=workdir, nonZeroIsFatal=False, ) mx.log(f"Return code of benchmark harness: {retcode}") # run again in single shot mode for memory measurements json_file_memory = f"{vm_venv}_memory.json" retcode = mx.run( [ join(vm_venv, "bin", "pyperformance"), "run", "--debug-single-value", "--track-memory", "--inherit-environ", "PIP_INDEX_URL,PIP_EXTRA_INDEX_URL,PIP_TRUSTED_HOST,PIP_TIMEOUT,PIP_RETRIES,LD_LIBRARY_PATH,LIBRARY_PATH,CPATH,PATH,PYPY_GC_MAX,JAVA_OPTS,GRAAL_PYTHON_ARGS", "-o", json_file_memory, *bms, ], cwd=workdir, nonZeroIsFatal=False, ) mx.log(f"Return code of benchmark harness: {retcode}") shutil.copy(join(workdir, json_file), join(SUITE.dir, "raw_results.json")) shutil.copy(join(workdir, json_file_memory), join(SUITE.dir, "raw_results_memory.json")) return retcode, ",".join([join(workdir, json_file), join(workdir, json_file_memory)]) class PyPySuite(PySuite): VERSION = "0324a252cf1a" def name(self): return "pypy-suite" def group(self): return "Graal" def subgroup(self): return "graalpython" def benchmarkList(self, bmSuiteArgs): return WildcardList(DEFAULT_PYPY_BENCHMARKS) def rules(self, output, benchmarks, bmSuiteArgs): return [PyPyJsonRule(output, self.name())] def createVmCommandLineArgs(self, benchmarks, runArgs): return [] def get_vm_registry(self): return python_vm_registry def _vmRun(self, vm, workdir, command, benchmarks, bmSuiteArgs): workdir = abspath(workdir) vm_venv = f"{self.name()}-{vm.name()}-{vm.config_name()}" if not hasattr(self, "prepared"): self.prepared = True if artifact := os.environ.get("PYPY_BENCHMARKS_DIR"): shutil.copytree(artifact, join(workdir, "benchmarks")) else: mx.warn("PYPY_BENCHMARKS_DIR is not set, cloning repository") mx.run( ["hg", "clone", "https://foss.heptapod.net/pypy/benchmarks"], cwd=workdir, ) mx.run( ["hg", "up", "-C", self.VERSION], cwd=join(workdir, "benchmarks") ) # workaround for pypy's benchmarks script issues with open(join(workdir, "benchmarks", "nullpython.py")) as f: content = f.read() content = content.replace("/usr/bin/python", "/usr/bin/env python") with open(join(workdir, "benchmarks", "nullpython.py"), "w") as f: f.write(content) with open(join(workdir, "benchmarks", "benchmarks.py")) as f: content = f.read() content = content.replace( 'float(line.split(b" ")[0])', "float(line.split()[0])" ) with open(join(workdir, "benchmarks", "benchmarks.py"), "w") as f: f.write(content) vm.run(workdir, ["-m", "venv", join(workdir, vm_venv)]) json_file = f"{vm_venv}.json" if benchmarks: bms = ["-b", ",".join(benchmarks)] else: bms = ["-b", ",".join(DEFAULT_PYPY_BENCHMARKS)] retcode = mx.run( [ sys.executable, join(workdir, "benchmarks", "run_local.py"), f"{vm_venv}/bin/python", "-o", join(workdir, json_file), *bms, ], cwd=workdir, nonZeroIsFatal=False, ) shutil.copy(join(workdir, json_file), join(SUITE.dir, "raw_results.json")) mx.log(f"Return code of benchmark harness: {retcode}") return retcode, join(workdir, json_file) class NumPySuite(PySuite): VERSION = "v1.26.4" BENCHMARK_REQ = [ "asv==0.5.1", "distlib==0.3.6", "filelock==3.8.0", "platformdirs==2.5.2", "six==1.16.0", "virtualenv==20.16.3", "packaging==24.0", f"numpy=={VERSION}", ] def name(self): return "numpy-suite" def group(self): return "Graal" def subgroup(self): return "graalpython" def benchmarkList(self, bmSuiteArgs): return WildcardList(DEFAULT_NUMPY_BENCHMARKS) def rules(self, output, benchmarks, bmSuiteArgs): return [AsvJsonRule(output, self.name())] def createVmCommandLineArgs(self, benchmarks, runArgs): return [] def get_vm_registry(self): return python_vm_registry def _vmRun(self, vm, workdir, command, benchmarks, bmSuiteArgs): workdir = abspath(workdir) benchdir = join(workdir, "numpy", "benchmarks") vm_venv = f"{self.name()}-{vm.name()}-{vm.config_name()}" if not hasattr(self, "prepared"): self.prepared = True npdir = join(workdir, "numpy") if artifact := os.environ.get("NUMPY_BENCHMARKS_DIR"): shutil.copytree(artifact, npdir) else: mx.warn("NUMPY_BENCHMARKS_DIR is not set, cloning numpy repository") mx.run( [ "git", "clone", "--depth", "1", "https://github.com/numpy/numpy.git", "--branch", self.VERSION, "--single-branch", ], cwd=workdir, ) shutil.rmtree(join(npdir, ".git")) mx.run(["git", "init", "."], cwd=npdir) mx.run(["git", "config", "user.email", "you@example.com"], cwd=npdir) mx.run(["git", "config", "user.name", "YourName"], cwd=npdir) mx.run(["git", "commit", "--allow-empty", "-m", "init"], cwd=npdir) mx.run(["git", "branch", self.VERSION], cwd=npdir) mx.run(["git", "branch", "main"], cwd=npdir, nonZeroIsFatal=False) mx.run(["git", "branch", "master"], cwd=npdir, nonZeroIsFatal=False) vm.run(workdir, ["-m", "venv", join(workdir, vm_venv)]) pip = join(workdir, vm_venv, "bin", "pip") mx.run([pip, "install", *self.BENCHMARK_REQ], cwd=workdir) mx.run( [join(workdir, vm_venv, "bin", "asv"), "machine", "--yes"], cwd=benchdir ) if not benchmarks: benchmarks = DEFAULT_NUMPY_BENCHMARKS retcode = mx.run( [ join(workdir, vm_venv, "bin", "asv"), "run", "--strict", "--record-samples", "-e", "--python=same", "--set-commit-hash", self.VERSION, "-b", create_asv_benchmark_selection(benchmarks, skipped=SKIPPED_NUMPY_BENCHMARKS), ], cwd=benchdir, nonZeroIsFatal=False, ) json_file = glob.glob(join(benchdir, "results", "*", "*numpy*.json")) mx.log(f"Return code of benchmark harness: {retcode}") if json_file: json_file = json_file[0] shutil.copy(json_file, join(SUITE.dir, "raw_results.json")) return retcode, json_file else: return -1, "" class PandasSuite(PySuite): VERSION = "1.5.2" VERSION_TAG = "v" + VERSION BENCHMARK_REQ = [ "asv==0.5.1", "distlib==0.3.6", "filelock==3.8.0", "platformdirs==2.5.2", "six==1.16.0", "virtualenv==20.16.3", "jinja2", f"numpy=={NumPySuite.VERSION}", f"pandas=={VERSION}", ] def name(self): return "pandas-suite" def group(self): return "Graal" def subgroup(self): return "graalpython" def benchmarkList(self, bmSuiteArgs): return WildcardList([ "reshape", "replace", ]) def rules(self, output, benchmarks, bmSuiteArgs): return [AsvJsonRule(output, self.name())] def createVmCommandLineArgs(self, benchmarks, runArgs): return [] def get_vm_registry(self): return python_vm_registry def _vmRun(self, vm, workdir, command, benchmarks, bmSuiteArgs): workdir = abspath(workdir) benchdir = join(workdir, "pandas", "asv_bench") vm_venv = f"{self.name()}-{vm.name()}-{vm.config_name()}" if not hasattr(self, "prepared"): self.prepared = True npdir = join(workdir, "pandas") if artifact := os.environ.get("PANDAS_BENCHMARKS_DIR"): shutil.copytree(artifact, npdir) else: mx.warn("PANDAS_BENCHMARKS_DIR is not set, cloning pandas repository") repo_url = os.environ.get("PANDAS_REPO_URL", "https://github.com/pandas-dev/pandas.git") mx.log("Cloning Pandas from " + repo_url) mx.run( [ "git", "clone", "--depth", "1", repo_url, "--branch", self.VERSION_TAG, "--single-branch", ], cwd=workdir, ) shutil.rmtree(join(npdir, ".git")) pandas_benchmarks_dir = join(npdir, "asv_bench", "benchmarks") accepted = ["__init__", "pandas_vb_common"] + list(self.benchmarkList([])) removed_files = [] for f in os.listdir(pandas_benchmarks_dir): # Remove any file or directory that is not a benchmark suite we want to run. # Keep all files starting with "_" if os.path.splitext(f)[0] not in accepted: removed_files.append(f) f_path = join(pandas_benchmarks_dir, f) if os.path.isdir(f_path): shutil.rmtree(f_path) else: os.remove(f_path) mx.log("Removed Pandas benchmark files: " + repr(removed_files)) mx.run(["git", "init", "."], cwd=npdir) mx.run(["git", "config", "user.email", "you@example.com"], cwd=npdir) mx.run(["git", "config", "user.name", "YourName"], cwd=npdir) mx.run(["git", "commit", "--allow-empty", "-m", "init"], cwd=npdir) mx.run(["git", "branch", self.VERSION_TAG], cwd=npdir) mx.run(["git", "branch", "main"], cwd=npdir, nonZeroIsFatal=False) mx.run(["git", "branch", "master"], cwd=npdir, nonZeroIsFatal=False) vm.run(workdir, ["-m", "venv", join(workdir, vm_venv)]) pip = join(workdir, vm_venv, "bin", "pip") mx.run([pip, "install", *self.BENCHMARK_REQ], cwd=workdir) mx.run( [join(workdir, vm_venv, "bin", "asv"), "machine", "--yes"], cwd=benchdir ) if not benchmarks: benchmarks = DEFAULT_PANDAS_BENCHMARKS retcode = mx.run( [ join(workdir, vm_venv, "bin", "asv"), "run", "--strict", "--record-samples", "-e", "--python=same", "--set-commit-hash", self.VERSION_TAG, "-b", create_asv_benchmark_selection(benchmarks, skipped=SKIPPED_PANDAS_BENCHMARKS), ], cwd=benchdir, nonZeroIsFatal=False, ) json_file = glob.glob(join(benchdir, "results", "*", "*pandas*.json")) mx.log(f"Return code of benchmark harness: {retcode}") if json_file: json_file = json_file[0] shutil.copy(json_file, join(SUITE.dir, "raw_results.json")) return retcode, json_file else: return -1, "" def register_python_benchmarks(): global python_vm_registry, SUITE from mx_graalpython_benchmark import python_vm_registry as vm_registry python_vm_registry = vm_registry SUITE = mx.suite("graalpython") python_vm_registry.add_vm(PyPyVm()) python_vm_registry.add_vm(Python3Vm()) for config_name, options, priority in [ ("launcher", [], 5), ]: python_vm_registry.add_vm(GraalPyVm(config_name, options), SUITE, priority) mx_benchmark.add_bm_suite(PyPerformanceSuite()) mx_benchmark.add_bm_suite(PyPySuite()) mx_benchmark.add_bm_suite(NumPySuite()) mx_benchmark.add_bm_suite(PandasSuite())