scorecard.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#***************************************************************************
#                                  _   _ ____  _
#  Project                     ___| | | |  _ \| |
#                             / __| | | | |_) | |
#                            | (__| |_| |  _ <| |___
#                             \___|\___/|_| \_\_____|
#
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at https://curl.se/docs/copyright.html.
#
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
# copies of the Software, and permit persons to whom the Software is
# furnished to do so, under the terms of the COPYING file.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
# SPDX-License-Identifier: curl
#
###########################################################################
#
import argparse
import json
import logging
import os
import re
import sys
from statistics import mean
from typing import Dict, Any, Optional, List

from testenv import Env, Httpd, Nghttpx, CurlClient, Caddy, ExecResult, NghttpxQuic, RunProfile

log = logging.getLogger(__name__)


class ScoreCardException(Exception):
    pass


class ScoreCard:

    def __init__(self, env: Env,
                 httpd: Optional[Httpd],
                 nghttpx: Optional[Nghttpx],
                 caddy: Optional[Caddy],
                 verbose: int,
                 curl_verbose: int):
        self.verbose = verbose
        self.env = env
        self.httpd = httpd
        self.nghttpx = nghttpx
        self.caddy = caddy
        self._silent_curl = not curl_verbose

    def info(self, msg):
        if self.verbose > 0:
            sys.stderr.write(msg)
            sys.stderr.flush()

    def handshakes(self, proto: str) -> Dict[str, Any]:
        props = {}
        sample_size = 5
        self.info(f'TLS Handshake\n')
        for authority in [
            'curl.se', 'google.com', 'cloudflare.com', 'nghttp2.org'
        ]:
            self.info(f'  {authority}...')
            props[authority] = {}
            for ipv in ['ipv4', 'ipv6']:
                self.info(f'{ipv}...')
                c_samples = []
                hs_samples = []
                errors = []
                for i in range(sample_size):
                    curl = CurlClient(env=self.env, silent=self._silent_curl)
                    args = [
                        '--http3-only' if proto == 'h3' else '--http2',
                        f'--{ipv}', f'https://{authority}/'
                    ]
                    r = curl.run_direct(args=args, with_stats=True)
                    if r.exit_code == 0 and len(r.stats) == 1:
                        c_samples.append(r.stats[0]['time_connect'])
                        hs_samples.append(r.stats[0]['time_appconnect'])
                    else:
                        errors.append(f'exit={r.exit_code}')
                    props[authority][f'{ipv}-connect'] = mean(c_samples) \
                        if len(c_samples) else -1
                    props[authority][f'{ipv}-handshake'] = mean(hs_samples) \
                        if len(hs_samples) else -1
                    props[authority][f'{ipv}-errors'] = errors
            self.info('ok.\n')
        return props

    def _make_docs_file(self, docs_dir: str, fname: str, fsize: int):
        fpath = os.path.join(docs_dir, fname)
        data1k = 1024*'x'
        flen = 0
        with open(fpath, 'w') as fd:
            while flen < fsize:
                fd.write(data1k)
                flen += len(data1k)
        return flen

    def _check_downloads(self, r: ExecResult, count: int):
        error = ''
        if r.exit_code != 0:
            error += f'exit={r.exit_code} '
        if r.exit_code != 0 or len(r.stats) != count:
            error += f'stats={len(r.stats)}/{count} '
        fails = [s for s in r.stats if s['response_code'] != 200]
        if len(fails) > 0:
            error += f'{len(fails)} failed'
        return error if len(error) > 0 else None

    def transfer_single(self, url: str, proto: str, count: int):
        sample_size = count
        count = 1
        samples = []
        errors = []
        profiles = []
        self.info(f'single...')
        for i in range(sample_size):
            curl = CurlClient(env=self.env, silent=self._silent_curl)
            r = curl.http_download(urls=[url], alpn_proto=proto, no_save=True,
                                   with_headers=False, with_profile=True)
            err = self._check_downloads(r, count)
            if err:
                errors.append(err)
            else:
                total_size = sum([s['size_download'] for s in r.stats])
                samples.append(total_size / r.duration.total_seconds())
                profiles.append(r.profile)
        return {
            'count': count,
            'samples': sample_size,
            'speed': mean(samples) if len(samples) else -1,
            'errors': errors,
            'stats': RunProfile.AverageStats(profiles),
        }

    def transfer_serial(self, url: str, proto: str, count: int):
        sample_size = 1
        samples = []
        errors = []
        profiles = []
        url = f'{url}?[0-{count - 1}]'
        self.info(f'serial...')
        for i in range(sample_size):
            curl = CurlClient(env=self.env, silent=self._silent_curl)
            r = curl.http_download(urls=[url], alpn_proto=proto, no_save=True,
                                   with_headers=False, with_profile=True)
            err = self._check_downloads(r, count)
            if err:
                errors.append(err)
            else:
                total_size = sum([s['size_download'] for s in r.stats])
                samples.append(total_size / r.duration.total_seconds())
                profiles.append(r.profile)
        return {
            'count': count,
            'samples': sample_size,
            'speed': mean(samples) if len(samples) else -1,
            'errors': errors,
            'stats': RunProfile.AverageStats(profiles),
        }

    def transfer_parallel(self, url: str, proto: str, count: int):
        sample_size = 1
        samples = []
        errors = []
        profiles = []
        url = f'{url}?[0-{count - 1}]'
        self.info(f'parallel...')
        for i in range(sample_size):
            curl = CurlClient(env=self.env, silent=self._silent_curl)
            r = curl.http_download(urls=[url], alpn_proto=proto, no_save=True,
                                   with_headers=False,
                                   with_profile=True,
                                   extra_args=['--parallel',
                                               '--parallel-max', str(count)])
            err = self._check_downloads(r, count)
            if err:
                errors.append(err)
            else:
                total_size = sum([s['size_download'] for s in r.stats])
                samples.append(total_size / r.duration.total_seconds())
                profiles.append(r.profile)
        return {
            'count': count,
            'samples': sample_size,
            'speed': mean(samples) if len(samples) else -1,
            'errors': errors,
            'stats': RunProfile.AverageStats(profiles),
        }

    def download_url(self, label: str, url: str, proto: str, count: int):
        self.info(f'  {count}x{label}: ')
        props = {
            'single': self.transfer_single(url=url, proto=proto, count=10),
        }
        if count > 1:
            props['serial'] = self.transfer_serial(url=url, proto=proto,
                                                   count=count)
            props['parallel'] = self.transfer_parallel(url=url, proto=proto,
                                                       count=count)
        self.info(f'ok.\n')
        return props

    def downloads(self, proto: str, count: int,
                  fsizes: List[int]) -> Dict[str, Any]:
        scores = {}
        if self.httpd:
            if proto == 'h3':
                port = self.env.h3_port
                via = 'nghttpx'
                descr = f'port {port}, proxying httpd'
            else:
                port = self.env.https_port
                via = 'httpd'
                descr = f'port {port}'
            self.info(f'{via} downloads\n')
            scores[via] = {
                'description': descr,
            }
            for fsize in fsizes:
                label = self.fmt_size(fsize)
                fname = f'score{label}.data'
                self._make_docs_file(docs_dir=self.httpd.docs_dir,
                                     fname=fname, fsize=fsize)
                url = f'https://{self.env.domain1}:{port}/{fname}'
                results = self.download_url(label=label, url=url,
                                            proto=proto, count=count)
                scores[via][label] = results
        if self.caddy:
            port = self.caddy.port
            via = 'caddy'
            descr = f'port {port}'
            self.info('caddy downloads\n')
            scores[via] = {
                'description': descr,
            }
            for fsize in fsizes:
                label = self.fmt_size(fsize)
                fname = f'score{label}.data'
                self._make_docs_file(docs_dir=self.caddy.docs_dir,
                                     fname=fname, fsize=fsize)
                url = f'https://{self.env.domain1}:{port}/{fname}'
                results = self.download_url(label=label, url=url,
                                            proto=proto, count=count)
                scores[via][label] = results
        return scores

    def do_requests(self, url: str, proto: str, count: int,
                    max_parallel: int = 1):
        sample_size = 1
        samples = []
        errors = []
        profiles = []
        url = f'{url}?[0-{count - 1}]'
        extra_args = ['--parallel', '--parallel-max', str(max_parallel)] \
            if max_parallel > 1 else []
        self.info(f'{max_parallel}...')
        for i in range(sample_size):
            curl = CurlClient(env=self.env, silent=self._silent_curl)
            r = curl.http_download(urls=[url], alpn_proto=proto, no_save=True,
                                   with_headers=False, with_profile=True,
                                   extra_args=extra_args)
            err = self._check_downloads(r, count)
            if err:
                errors.append(err)
            else:
                for _ in r.stats:
                    samples.append(count / r.duration.total_seconds())
                profiles.append(r.profile)
        return {
            'count': count,
            'samples': sample_size,
            'speed': mean(samples) if len(samples) else -1,
            'errors': errors,
            'stats': RunProfile.AverageStats(profiles),
        }

    def requests_url(self, url: str, proto: str, count: int):
        self.info(f'  {url}: ')
        props = {
            '1': self.do_requests(url=url, proto=proto, count=count),
            '6': self.do_requests(url=url, proto=proto, count=count,
                                  max_parallel=6),
            '25': self.do_requests(url=url, proto=proto, count=count,
                                   max_parallel=25),
            '50': self.do_requests(url=url, proto=proto, count=count,
                                   max_parallel=50),
            '100': self.do_requests(url=url, proto=proto, count=count,
                                    max_parallel=100),
        }
        self.info(f'ok.\n')
        return props

    def requests(self, proto: str, req_count) -> Dict[str, Any]:
        scores = {}
        if self.httpd:
            if proto == 'h3':
                port = self.env.h3_port
                via = 'nghttpx'
                descr = f'port {port}, proxying httpd'
            else:
                port = self.env.https_port
                via = 'httpd'
                descr = f'port {port}'
            self.info(f'{via} requests\n')
            self._make_docs_file(docs_dir=self.httpd.docs_dir,
                                 fname='reqs10.data', fsize=10*1024)
            url1 = f'https://{self.env.domain1}:{port}/reqs10.data'
            scores[via] = {
                'description': descr,
                'count': req_count,
                '10KB': self.requests_url(url=url1, proto=proto, count=req_count),
            }
        if self.caddy:
            port = self.caddy.port
            via = 'caddy'
            descr = f'port {port}'
            self.info('caddy requests\n')
            self._make_docs_file(docs_dir=self.caddy.docs_dir,
                                 fname='req10.data', fsize=10 * 1024)
            url1 = f'https://{self.env.domain1}:{port}/req10.data'
            scores[via] = {
                'description': descr,
                'count': req_count,
                '10KB': self.requests_url(url=url1, proto=proto, count=req_count),
            }
        return scores

    def score_proto(self, proto: str,
                    handshakes: bool = True,
                    downloads: Optional[List[int]] = None,
                    download_count: int = 50,
                    req_count=5000,
                    requests: bool = True):
        self.info(f"scoring {proto}\n")
        p = {}
        if proto == 'h3':
            p['name'] = 'h3'
            if not self.env.have_h3_curl():
                raise ScoreCardException('curl does not support HTTP/3')
            for lib in ['ngtcp2', 'quiche', 'msh3', 'nghttp3']:
                if self.env.curl_uses_lib(lib):
                    p['implementation'] = lib
                    break
        elif proto == 'h2':
            p['name'] = 'h2'
            if not self.env.have_h2_curl():
                raise ScoreCardException('curl does not support HTTP/2')
            for lib in ['nghttp2', 'hyper']:
                if self.env.curl_uses_lib(lib):
                    p['implementation'] = lib
                    break
        elif proto == 'h1' or proto == 'http/1.1':
            proto = 'http/1.1'
            p['name'] = proto
            p['implementation'] = 'hyper' if self.env.curl_uses_lib('hyper')\
                else 'native'
        else:
            raise ScoreCardException(f"unknown protocol: {proto}")

        if 'implementation' not in p:
            raise ScoreCardException(f'did not recognized {p} lib')
        p['version'] = Env.curl_lib_version(p['implementation'])

        score = {
            'curl': self.env.curl_fullname(),
            'os': self.env.curl_os(),
            'protocol': p,
        }
        if handshakes:
            score['handshakes'] = self.handshakes(proto=proto)
        if downloads and len(downloads) > 0:
            score['downloads'] = self.downloads(proto=proto,
                                                count=download_count,
                                                fsizes=downloads)
        if requests:
            score['requests'] = self.requests(proto=proto, req_count=req_count)
        self.info("\n")
        return score

    def fmt_ms(self, tval):
        return f'{int(tval*1000)} ms' if tval >= 0 else '--'

    def fmt_size(self, val):
        if val >= (1024*1024*1024):
            return f'{val / (1024*1024*1024):0.000f}GB'
        elif val >= (1024 * 1024):
            return f'{val / (1024*1024):0.000f}MB'
        elif val >= 1024:
            return f'{val / 1024:0.000f}KB'
        else:
            return f'{val:0.000f}B'

    def fmt_mbs(self, val):
        return f'{val/(1024*1024):0.000f} MB/s' if val >= 0 else '--'

    def fmt_reqs(self, val):
        return f'{val:0.000f} r/s' if val >= 0 else '--'

    def print_score(self, score):
        print(f'{score["protocol"]["name"].upper()} in {score["curl"]}')
        if 'handshakes' in score:
            print(f'{"Handshakes":<24} {"ipv4":25} {"ipv6":28}')
            print(f'  {"Host":<17} {"Connect":>12} {"Handshake":>12} '
                  f'{"Connect":>12} {"Handshake":>12}     {"Errors":<20}')
            for key, val in score["handshakes"].items():
                print(f'  {key:<17} {self.fmt_ms(val["ipv4-connect"]):>12} '
                      f'{self.fmt_ms(val["ipv4-handshake"]):>12} '
                      f'{self.fmt_ms(val["ipv6-connect"]):>12} '
                      f'{self.fmt_ms(val["ipv6-handshake"]):>12}     '
                      f'{"/".join(val["ipv4-errors"] + val["ipv6-errors"]):<20}'
                      )
        if 'downloads' in score:
            # get the key names of all sizes and measurements made
            sizes = []
            measures = []
            m_names = {}
            mcol_width = 12
            mcol_sw = 17
            for server, server_score in score['downloads'].items():
                for sskey, ssval in server_score.items():
                    if isinstance(ssval, str):
                        continue
                    if sskey not in sizes:
                        sizes.append(sskey)
                    for mkey, mval in server_score[sskey].items():
                        if mkey not in measures:
                            measures.append(mkey)
                            m_names[mkey] = f'{mkey}({mval["count"]}x)'

            print('Downloads')
            print(f'  {"Server":<8} {"Size":>8}', end='')
            for m in measures: print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='')
            print(f' {"Errors":^20}')

            for server in score['downloads']:
                for size in sizes:
                    size_score = score['downloads'][server][size]
                    print(f'  {server:<8} {size:>8}', end='')
                    errors = []
                    for key, val in size_score.items():
                        if 'errors' in val:
                            errors.extend(val['errors'])
                    for m in measures:
                        if m in size_score:
                            print(f' {self.fmt_mbs(size_score[m]["speed"]):>{mcol_width}}', end='')
                            s = f'[{size_score[m]["stats"]["cpu"]:>.1f}%'\
                                f'/{self.fmt_size(size_score[m]["stats"]["rss"])}]'
                            print(f' {s:<{mcol_sw}}', end='')
                        else:
                            print(' '*mcol_width, end='')
                    if len(errors):
                        print(f' {"/".join(errors):<20}')
                    else:
                        print(f' {"-":^20}')

        if 'requests' in score:
            sizes = []
            measures = []
            m_names = {}
            mcol_width = 9
            mcol_sw = 13
            for server in score['requests']:
                server_score = score['requests'][server]
                for sskey, ssval in server_score.items():
                    if isinstance(ssval, str) or isinstance(ssval, int):
                        continue
                    if sskey not in sizes:
                        sizes.append(sskey)
                    for mkey, mval in server_score[sskey].items():
                        if mkey not in measures:
                            measures.append(mkey)
                            m_names[mkey] = f'{mkey}'

            print('Requests, max in parallel')
            print(f'  {"Server":<8} {"Size":>6} {"Reqs":>6}', end='')
            for m in measures: print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='')
            print(f' {"Errors":^10}')

            for server in score['requests']:
                for size in sizes:
                    size_score = score['requests'][server][size]
                    count = score['requests'][server]['count']
                    print(f'  {server:<8} {size:>6} {count:>6}', end='')
                    errors = []
                    for key, val in size_score.items():
                        if 'errors' in val:
                            errors.extend(val['errors'])
                    for m in measures:
                        if m in size_score:
                            print(f' {self.fmt_reqs(size_score[m]["speed"]):>{mcol_width}}', end='')
                            s = f'[{size_score[m]["stats"]["cpu"]:>.1f}%'\
                                f'/{self.fmt_size(size_score[m]["stats"]["rss"])}]'
                            print(f' {s:<{mcol_sw}}', end='')
                        else:
                            print(' '*mcol_width, end='')
                    if len(errors):
                        print(f' {"/".join(errors):<10}')
                    else:
                        print(f' {"-":^10}')


def parse_size(s):
    m = re.match(r'(\d+)(mb|kb|gb)?', s, re.IGNORECASE)
    if m is None:
        raise Exception(f'unrecognized size: {s}')
    size = int(m.group(1))
    if not m.group(2):
        pass
    elif m.group(2).lower() == 'kb':
        size *= 1024
    elif m.group(2).lower() == 'mb':
        size *= 1024 * 1024
    elif m.group(2).lower() == 'gb':
        size *= 1024 * 1024 * 1024
    return size


def main():
    parser = argparse.ArgumentParser(prog='scorecard', description="""
        Run a range of tests to give a scorecard for a HTTP protocol
        'h3' or 'h2' implementation in curl.
        """)
    parser.add_argument("-v", "--verbose", action='count', default=1,
                        help="log more output on stderr")
    parser.add_argument("-j", "--json", action='store_true',
                        default=False, help="print json instead of text")
    parser.add_argument("-H", "--handshakes", action='store_true',
                        default=False, help="evaluate handshakes only")
    parser.add_argument("-d", "--downloads", action='store_true',
                        default=False, help="evaluate downloads")
    parser.add_argument("--download", action='append', type=str,
                        default=None, help="evaluate download size")
    parser.add_argument("--download-count", action='store', type=int,
                        default=50, help="perform that many downloads")
    parser.add_argument("-r", "--requests", action='store_true',
                        default=False, help="evaluate requests")
    parser.add_argument("--request-count", action='store', type=int,
                        default=5000, help="perform that many requests")
    parser.add_argument("--httpd", action='store_true', default=False,
                        help="evaluate httpd server only")
    parser.add_argument("--caddy", action='store_true', default=False,
                        help="evaluate caddy server only")
    parser.add_argument("--curl-verbose", action='store_true',
                        default=False, help="run curl with `-v`")
    parser.add_argument("protocol", default='h2', nargs='?',
                        help="Name of protocol to score")
    args = parser.parse_args()

    if args.verbose > 0:
        console = logging.StreamHandler()
        console.setLevel(logging.INFO)
        console.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
        logging.getLogger('').addHandler(console)

    protocol = args.protocol
    handshakes = True
    downloads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024]
    if args.download is not None:
        downloads = []
        for x in args.download:
            downloads.extend([parse_size(s) for s in x.split(',')])
    requests = True
    if args.downloads or args.requests or args.handshakes:
        handshakes = args.handshakes
        if not args.downloads:
            downloads = None
        requests = args.requests

    test_httpd = protocol != 'h3'
    test_caddy = True
    if args.caddy or args.httpd:
        test_caddy = args.caddy
        test_httpd = args.httpd

    rv = 0
    env = Env()
    env.setup()
    env.test_timeout = None
    httpd = None
    nghttpx = None
    caddy = None
    try:
        if test_httpd:
            httpd = Httpd(env=env)
            assert httpd.exists(), \
                f'httpd not found: {env.httpd}'
            httpd.clear_logs()
            assert httpd.start()
            if 'h3' == protocol:
                nghttpx = NghttpxQuic(env=env)
                nghttpx.clear_logs()
                assert nghttpx.start()
        if test_caddy and env.caddy:
            caddy = Caddy(env=env)
            caddy.clear_logs()
            assert caddy.start()

        card = ScoreCard(env=env, httpd=httpd, nghttpx=nghttpx, caddy=caddy,
                         verbose=args.verbose, curl_verbose=args.curl_verbose)
        score = card.score_proto(proto=protocol,
                                 handshakes=handshakes,
                                 downloads=downloads,
                                 download_count=args.download_count,
                                 req_count=args.request_count,
                                 requests=requests)
        if args.json:
            print(json.JSONEncoder(indent=2).encode(score))
        else:
            card.print_score(score)

    except ScoreCardException as ex:
        sys.stderr.write(f"ERROR: {str(ex)}\n")
        rv = 1
    except KeyboardInterrupt:
        log.warning("aborted")
        rv = 1
    finally:
        if caddy:
            caddy.stop()
        if nghttpx:
            nghttpx.stop(wait_dead=False)
        if httpd:
            httpd.stop()
    sys.exit(rv)


if __name__ == "__main__":
    main()