1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import argparse 28import json 29import logging 30import os 31import re 32import sys 33from statistics import mean 34from typing import Dict, Any, Optional, List 35 36from testenv import Env, Httpd, Nghttpx, CurlClient, Caddy, ExecResult, NghttpxQuic, RunProfile 37 38log = logging.getLogger(__name__) 39 40 41class ScoreCardException(Exception): 42 pass 43 44 45class ScoreCard: 46 47 def __init__(self, env: Env, 48 httpd: Optional[Httpd], 49 nghttpx: Optional[Nghttpx], 50 caddy: Optional[Caddy], 51 verbose: int, 52 curl_verbose: int, 53 download_parallel: int = 0): 54 self.verbose = verbose 55 self.env = env 56 self.httpd = httpd 57 self.nghttpx = nghttpx 58 self.caddy = caddy 59 self._silent_curl = not curl_verbose 60 self._download_parallel = download_parallel 61 62 def info(self, msg): 63 if self.verbose > 0: 64 sys.stderr.write(msg) 65 sys.stderr.flush() 66 67 def handshakes(self, proto: str) -> Dict[str, Any]: 68 props = {} 69 sample_size = 5 70 self.info(f'TLS Handshake\n') 71 for authority in [ 72 'curl.se', 'google.com', 'cloudflare.com', 'nghttp2.org' 73 ]: 74 self.info(f' {authority}...') 75 props[authority] = {} 76 for ipv in ['ipv4', 'ipv6']: 77 self.info(f'{ipv}...') 78 c_samples = [] 79 hs_samples = [] 80 errors = [] 81 for i in range(sample_size): 82 curl = CurlClient(env=self.env, silent=self._silent_curl) 83 args = [ 84 '--http3-only' if proto == 'h3' else '--http2', 85 f'--{ipv}', f'https://{authority}/' 86 ] 87 r = curl.run_direct(args=args, with_stats=True) 88 if r.exit_code == 0 and len(r.stats) == 1: 89 c_samples.append(r.stats[0]['time_connect']) 90 hs_samples.append(r.stats[0]['time_appconnect']) 91 else: 92 errors.append(f'exit={r.exit_code}') 93 props[authority][f'{ipv}-connect'] = mean(c_samples) \ 94 if len(c_samples) else -1 95 props[authority][f'{ipv}-handshake'] = mean(hs_samples) \ 96 if len(hs_samples) else -1 97 props[authority][f'{ipv}-errors'] = errors 98 self.info('ok.\n') 99 return props 100 101 def _make_docs_file(self, docs_dir: str, fname: str, fsize: int): 102 fpath = os.path.join(docs_dir, fname) 103 data1k = 1024*'x' 104 flen = 0 105 with open(fpath, 'w') as fd: 106 while flen < fsize: 107 fd.write(data1k) 108 flen += len(data1k) 109 return flen 110 111 def _check_downloads(self, r: ExecResult, count: int): 112 error = '' 113 if r.exit_code != 0: 114 error += f'exit={r.exit_code} ' 115 if r.exit_code != 0 or len(r.stats) != count: 116 error += f'stats={len(r.stats)}/{count} ' 117 fails = [s for s in r.stats if s['response_code'] != 200] 118 if len(fails) > 0: 119 error += f'{len(fails)} failed' 120 return error if len(error) > 0 else None 121 122 def transfer_single(self, url: str, proto: str, count: int): 123 sample_size = count 124 count = 1 125 samples = [] 126 errors = [] 127 profiles = [] 128 self.info(f'single...') 129 for i in range(sample_size): 130 curl = CurlClient(env=self.env, silent=self._silent_curl) 131 r = curl.http_download(urls=[url], alpn_proto=proto, no_save=True, 132 with_headers=False, with_profile=True) 133 err = self._check_downloads(r, count) 134 if err: 135 errors.append(err) 136 else: 137 total_size = sum([s['size_download'] for s in r.stats]) 138 samples.append(total_size / r.duration.total_seconds()) 139 profiles.append(r.profile) 140 return { 141 'count': count, 142 'samples': sample_size, 143 'max-parallel': 1, 144 'speed': mean(samples) if len(samples) else -1, 145 'errors': errors, 146 'stats': RunProfile.AverageStats(profiles), 147 } 148 149 def transfer_serial(self, url: str, proto: str, count: int): 150 sample_size = 1 151 samples = [] 152 errors = [] 153 profiles = [] 154 url = f'{url}?[0-{count - 1}]' 155 self.info(f'serial...') 156 for i in range(sample_size): 157 curl = CurlClient(env=self.env, silent=self._silent_curl) 158 r = curl.http_download(urls=[url], alpn_proto=proto, no_save=True, 159 with_headers=False, with_profile=True) 160 err = self._check_downloads(r, count) 161 if err: 162 errors.append(err) 163 else: 164 total_size = sum([s['size_download'] for s in r.stats]) 165 samples.append(total_size / r.duration.total_seconds()) 166 profiles.append(r.profile) 167 return { 168 'count': count, 169 'samples': sample_size, 170 'max-parallel': 1, 171 'speed': mean(samples) if len(samples) else -1, 172 'errors': errors, 173 'stats': RunProfile.AverageStats(profiles), 174 } 175 176 def transfer_parallel(self, url: str, proto: str, count: int): 177 sample_size = 1 178 samples = [] 179 errors = [] 180 profiles = [] 181 max_parallel = self._download_parallel if self._download_parallel > 0 else count 182 url = f'{url}?[0-{count - 1}]' 183 self.info(f'parallel...') 184 for i in range(sample_size): 185 curl = CurlClient(env=self.env, silent=self._silent_curl) 186 r = curl.http_download(urls=[url], alpn_proto=proto, no_save=True, 187 with_headers=False, 188 with_profile=True, 189 extra_args=['--parallel', 190 '--parallel-max', str(max_parallel)]) 191 err = self._check_downloads(r, count) 192 if err: 193 errors.append(err) 194 else: 195 total_size = sum([s['size_download'] for s in r.stats]) 196 samples.append(total_size / r.duration.total_seconds()) 197 profiles.append(r.profile) 198 return { 199 'count': count, 200 'samples': sample_size, 201 'max-parallel': max_parallel, 202 'speed': mean(samples) if len(samples) else -1, 203 'errors': errors, 204 'stats': RunProfile.AverageStats(profiles), 205 } 206 207 def download_url(self, label: str, url: str, proto: str, count: int): 208 self.info(f' {count}x{label}: ') 209 props = { 210 'single': self.transfer_single(url=url, proto=proto, count=10), 211 } 212 if count > 1: 213 props['serial'] = self.transfer_serial(url=url, proto=proto, 214 count=count) 215 props['parallel'] = self.transfer_parallel(url=url, proto=proto, 216 count=count) 217 self.info(f'ok.\n') 218 return props 219 220 def downloads(self, proto: str, count: int, 221 fsizes: List[int]) -> Dict[str, Any]: 222 scores = {} 223 if self.httpd: 224 if proto == 'h3': 225 port = self.env.h3_port 226 via = 'nghttpx' 227 descr = f'port {port}, proxying httpd' 228 else: 229 port = self.env.https_port 230 via = 'httpd' 231 descr = f'port {port}' 232 self.info(f'{via} downloads\n') 233 scores[via] = { 234 'description': descr, 235 } 236 for fsize in fsizes: 237 label = self.fmt_size(fsize) 238 fname = f'score{label}.data' 239 self._make_docs_file(docs_dir=self.httpd.docs_dir, 240 fname=fname, fsize=fsize) 241 url = f'https://{self.env.domain1}:{port}/{fname}' 242 results = self.download_url(label=label, url=url, 243 proto=proto, count=count) 244 scores[via][label] = results 245 if self.caddy: 246 port = self.caddy.port 247 via = 'caddy' 248 descr = f'port {port}' 249 self.info('caddy downloads\n') 250 scores[via] = { 251 'description': descr, 252 } 253 for fsize in fsizes: 254 label = self.fmt_size(fsize) 255 fname = f'score{label}.data' 256 self._make_docs_file(docs_dir=self.caddy.docs_dir, 257 fname=fname, fsize=fsize) 258 url = f'https://{self.env.domain1}:{port}/{fname}' 259 results = self.download_url(label=label, url=url, 260 proto=proto, count=count) 261 scores[via][label] = results 262 return scores 263 264 def do_requests(self, url: str, proto: str, count: int, 265 max_parallel: int = 1): 266 sample_size = 1 267 samples = [] 268 errors = [] 269 profiles = [] 270 url = f'{url}?[0-{count - 1}]' 271 extra_args = ['--parallel', '--parallel-max', str(max_parallel)] \ 272 if max_parallel > 1 else [] 273 self.info(f'{max_parallel}...') 274 for i in range(sample_size): 275 curl = CurlClient(env=self.env, silent=self._silent_curl) 276 r = curl.http_download(urls=[url], alpn_proto=proto, no_save=True, 277 with_headers=False, with_profile=True, 278 extra_args=extra_args) 279 err = self._check_downloads(r, count) 280 if err: 281 errors.append(err) 282 else: 283 for _ in r.stats: 284 samples.append(count / r.duration.total_seconds()) 285 profiles.append(r.profile) 286 return { 287 'count': count, 288 'samples': sample_size, 289 'speed': mean(samples) if len(samples) else -1, 290 'errors': errors, 291 'stats': RunProfile.AverageStats(profiles), 292 } 293 294 def requests_url(self, url: str, proto: str, count: int): 295 self.info(f' {url}: ') 296 props = { 297 '1': self.do_requests(url=url, proto=proto, count=count), 298 '6': self.do_requests(url=url, proto=proto, count=count, 299 max_parallel=6), 300 '25': self.do_requests(url=url, proto=proto, count=count, 301 max_parallel=25), 302 '50': self.do_requests(url=url, proto=proto, count=count, 303 max_parallel=50), 304 '100': self.do_requests(url=url, proto=proto, count=count, 305 max_parallel=100), 306 } 307 self.info(f'ok.\n') 308 return props 309 310 def requests(self, proto: str, req_count) -> Dict[str, Any]: 311 scores = {} 312 if self.httpd: 313 if proto == 'h3': 314 port = self.env.h3_port 315 via = 'nghttpx' 316 descr = f'port {port}, proxying httpd' 317 else: 318 port = self.env.https_port 319 via = 'httpd' 320 descr = f'port {port}' 321 self.info(f'{via} requests\n') 322 self._make_docs_file(docs_dir=self.httpd.docs_dir, 323 fname='reqs10.data', fsize=10*1024) 324 url1 = f'https://{self.env.domain1}:{port}/reqs10.data' 325 scores[via] = { 326 'description': descr, 327 'count': req_count, 328 '10KB': self.requests_url(url=url1, proto=proto, count=req_count), 329 } 330 if self.caddy: 331 port = self.caddy.port 332 via = 'caddy' 333 descr = f'port {port}' 334 self.info('caddy requests\n') 335 self._make_docs_file(docs_dir=self.caddy.docs_dir, 336 fname='req10.data', fsize=10 * 1024) 337 url1 = f'https://{self.env.domain1}:{port}/req10.data' 338 scores[via] = { 339 'description': descr, 340 'count': req_count, 341 '10KB': self.requests_url(url=url1, proto=proto, count=req_count), 342 } 343 return scores 344 345 def score_proto(self, proto: str, 346 handshakes: bool = True, 347 downloads: Optional[List[int]] = None, 348 download_count: int = 50, 349 req_count=5000, 350 requests: bool = True): 351 self.info(f"scoring {proto}\n") 352 p = {} 353 if proto == 'h3': 354 p['name'] = 'h3' 355 if not self.env.have_h3_curl(): 356 raise ScoreCardException('curl does not support HTTP/3') 357 for lib in ['ngtcp2', 'quiche', 'msh3', 'nghttp3']: 358 if self.env.curl_uses_lib(lib): 359 p['implementation'] = lib 360 break 361 elif proto == 'h2': 362 p['name'] = 'h2' 363 if not self.env.have_h2_curl(): 364 raise ScoreCardException('curl does not support HTTP/2') 365 for lib in ['nghttp2', 'hyper']: 366 if self.env.curl_uses_lib(lib): 367 p['implementation'] = lib 368 break 369 elif proto == 'h1' or proto == 'http/1.1': 370 proto = 'http/1.1' 371 p['name'] = proto 372 p['implementation'] = 'hyper' if self.env.curl_uses_lib('hyper')\ 373 else 'native' 374 else: 375 raise ScoreCardException(f"unknown protocol: {proto}") 376 377 if 'implementation' not in p: 378 raise ScoreCardException(f'did not recognized {p} lib') 379 p['version'] = Env.curl_lib_version(p['implementation']) 380 381 score = { 382 'curl': self.env.curl_fullname(), 383 'os': self.env.curl_os(), 384 'protocol': p, 385 } 386 if handshakes: 387 score['handshakes'] = self.handshakes(proto=proto) 388 if downloads and len(downloads) > 0: 389 score['downloads'] = self.downloads(proto=proto, 390 count=download_count, 391 fsizes=downloads) 392 if requests: 393 score['requests'] = self.requests(proto=proto, req_count=req_count) 394 self.info("\n") 395 return score 396 397 def fmt_ms(self, tval): 398 return f'{int(tval*1000)} ms' if tval >= 0 else '--' 399 400 def fmt_size(self, val): 401 if val >= (1024*1024*1024): 402 return f'{val / (1024*1024*1024):0.000f}GB' 403 elif val >= (1024 * 1024): 404 return f'{val / (1024*1024):0.000f}MB' 405 elif val >= 1024: 406 return f'{val / 1024:0.000f}KB' 407 else: 408 return f'{val:0.000f}B' 409 410 def fmt_mbs(self, val): 411 return f'{val/(1024*1024):0.000f} MB/s' if val >= 0 else '--' 412 413 def fmt_reqs(self, val): 414 return f'{val:0.000f} r/s' if val >= 0 else '--' 415 416 def print_score(self, score): 417 print(f'{score["protocol"]["name"].upper()} in {score["curl"]}') 418 if 'handshakes' in score: 419 print(f'{"Handshakes":<24} {"ipv4":25} {"ipv6":28}') 420 print(f' {"Host":<17} {"Connect":>12} {"Handshake":>12} ' 421 f'{"Connect":>12} {"Handshake":>12} {"Errors":<20}') 422 for key, val in score["handshakes"].items(): 423 print(f' {key:<17} {self.fmt_ms(val["ipv4-connect"]):>12} ' 424 f'{self.fmt_ms(val["ipv4-handshake"]):>12} ' 425 f'{self.fmt_ms(val["ipv6-connect"]):>12} ' 426 f'{self.fmt_ms(val["ipv6-handshake"]):>12} ' 427 f'{"/".join(val["ipv4-errors"] + val["ipv6-errors"]):<20}' 428 ) 429 if 'downloads' in score: 430 # get the key names of all sizes and measurements made 431 sizes = [] 432 measures = [] 433 m_names = {} 434 mcol_width = 12 435 mcol_sw = 17 436 for server, server_score in score['downloads'].items(): 437 for sskey, ssval in server_score.items(): 438 if isinstance(ssval, str): 439 continue 440 if sskey not in sizes: 441 sizes.append(sskey) 442 for mkey, mval in server_score[sskey].items(): 443 if mkey not in measures: 444 measures.append(mkey) 445 m_names[mkey] = f'{mkey}({mval["count"]}x{mval["max-parallel"]})' 446 447 print('Downloads') 448 print(f' {"Server":<8} {"Size":>8}', end='') 449 for m in measures: print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='') 450 print(f' {"Errors":^20}') 451 452 for server in score['downloads']: 453 for size in sizes: 454 size_score = score['downloads'][server][size] 455 print(f' {server:<8} {size:>8}', end='') 456 errors = [] 457 for key, val in size_score.items(): 458 if 'errors' in val: 459 errors.extend(val['errors']) 460 for m in measures: 461 if m in size_score: 462 print(f' {self.fmt_mbs(size_score[m]["speed"]):>{mcol_width}}', end='') 463 s = f'[{size_score[m]["stats"]["cpu"]:>.1f}%'\ 464 f'/{self.fmt_size(size_score[m]["stats"]["rss"])}]' 465 print(f' {s:<{mcol_sw}}', end='') 466 else: 467 print(' '*mcol_width, end='') 468 if len(errors): 469 print(f' {"/".join(errors):<20}') 470 else: 471 print(f' {"-":^20}') 472 473 if 'requests' in score: 474 sizes = [] 475 measures = [] 476 m_names = {} 477 mcol_width = 9 478 mcol_sw = 13 479 for server in score['requests']: 480 server_score = score['requests'][server] 481 for sskey, ssval in server_score.items(): 482 if isinstance(ssval, str) or isinstance(ssval, int): 483 continue 484 if sskey not in sizes: 485 sizes.append(sskey) 486 for mkey, mval in server_score[sskey].items(): 487 if mkey not in measures: 488 measures.append(mkey) 489 m_names[mkey] = f'{mkey}' 490 491 print('Requests, max in parallel') 492 print(f' {"Server":<8} {"Size":>6} {"Reqs":>6}', end='') 493 for m in measures: print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='') 494 print(f' {"Errors":^10}') 495 496 for server in score['requests']: 497 for size in sizes: 498 size_score = score['requests'][server][size] 499 count = score['requests'][server]['count'] 500 print(f' {server:<8} {size:>6} {count:>6}', end='') 501 errors = [] 502 for key, val in size_score.items(): 503 if 'errors' in val: 504 errors.extend(val['errors']) 505 for m in measures: 506 if m in size_score: 507 print(f' {self.fmt_reqs(size_score[m]["speed"]):>{mcol_width}}', end='') 508 s = f'[{size_score[m]["stats"]["cpu"]:>.1f}%'\ 509 f'/{self.fmt_size(size_score[m]["stats"]["rss"])}]' 510 print(f' {s:<{mcol_sw}}', end='') 511 else: 512 print(' '*mcol_width, end='') 513 if len(errors): 514 print(f' {"/".join(errors):<10}') 515 else: 516 print(f' {"-":^10}') 517 518 519def parse_size(s): 520 m = re.match(r'(\d+)(mb|kb|gb)?', s, re.IGNORECASE) 521 if m is None: 522 raise Exception(f'unrecognized size: {s}') 523 size = int(m.group(1)) 524 if not m.group(2): 525 pass 526 elif m.group(2).lower() == 'kb': 527 size *= 1024 528 elif m.group(2).lower() == 'mb': 529 size *= 1024 * 1024 530 elif m.group(2).lower() == 'gb': 531 size *= 1024 * 1024 * 1024 532 return size 533 534 535def main(): 536 parser = argparse.ArgumentParser(prog='scorecard', description=""" 537 Run a range of tests to give a scorecard for a HTTP protocol 538 'h3' or 'h2' implementation in curl. 539 """) 540 parser.add_argument("-v", "--verbose", action='count', default=1, 541 help="log more output on stderr") 542 parser.add_argument("-j", "--json", action='store_true', 543 default=False, help="print json instead of text") 544 parser.add_argument("-H", "--handshakes", action='store_true', 545 default=False, help="evaluate handshakes only") 546 parser.add_argument("-d", "--downloads", action='store_true', 547 default=False, help="evaluate downloads") 548 parser.add_argument("--download", action='append', type=str, 549 default=None, help="evaluate download size") 550 parser.add_argument("--download-count", action='store', type=int, 551 default=50, help="perform that many downloads") 552 parser.add_argument("--download-parallel", action='store', type=int, 553 default=0, help="perform that many downloads in parallel (default all)") 554 parser.add_argument("-r", "--requests", action='store_true', 555 default=False, help="evaluate requests") 556 parser.add_argument("--request-count", action='store', type=int, 557 default=5000, help="perform that many requests") 558 parser.add_argument("--httpd", action='store_true', default=False, 559 help="evaluate httpd server only") 560 parser.add_argument("--caddy", action='store_true', default=False, 561 help="evaluate caddy server only") 562 parser.add_argument("--curl-verbose", action='store_true', 563 default=False, help="run curl with `-v`") 564 parser.add_argument("protocol", default='h2', nargs='?', 565 help="Name of protocol to score") 566 args = parser.parse_args() 567 568 if args.verbose > 0: 569 console = logging.StreamHandler() 570 console.setLevel(logging.INFO) 571 console.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) 572 logging.getLogger('').addHandler(console) 573 574 protocol = args.protocol 575 handshakes = True 576 downloads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024] 577 if args.download is not None: 578 downloads = [] 579 for x in args.download: 580 downloads.extend([parse_size(s) for s in x.split(',')]) 581 requests = True 582 if args.downloads or args.requests or args.handshakes: 583 handshakes = args.handshakes 584 if not args.downloads: 585 downloads = None 586 requests = args.requests 587 588 test_httpd = protocol != 'h3' 589 test_caddy = True 590 if args.caddy or args.httpd: 591 test_caddy = args.caddy 592 test_httpd = args.httpd 593 594 rv = 0 595 env = Env() 596 env.setup() 597 env.test_timeout = None 598 httpd = None 599 nghttpx = None 600 caddy = None 601 try: 602 if test_httpd: 603 httpd = Httpd(env=env) 604 assert httpd.exists(), \ 605 f'httpd not found: {env.httpd}' 606 httpd.clear_logs() 607 assert httpd.start() 608 if 'h3' == protocol: 609 nghttpx = NghttpxQuic(env=env) 610 nghttpx.clear_logs() 611 assert nghttpx.start() 612 if test_caddy and env.caddy: 613 caddy = Caddy(env=env) 614 caddy.clear_logs() 615 assert caddy.start() 616 617 card = ScoreCard(env=env, httpd=httpd, nghttpx=nghttpx, caddy=caddy, 618 verbose=args.verbose, curl_verbose=args.curl_verbose, 619 download_parallel=args.download_parallel) 620 score = card.score_proto(proto=protocol, 621 handshakes=handshakes, 622 downloads=downloads, 623 download_count=args.download_count, 624 req_count=args.request_count, 625 requests=requests) 626 if args.json: 627 print(json.JSONEncoder(indent=2).encode(score)) 628 else: 629 card.print_score(score) 630 631 except ScoreCardException as ex: 632 sys.stderr.write(f"ERROR: {str(ex)}\n") 633 rv = 1 634 except KeyboardInterrupt: 635 log.warning("aborted") 636 rv = 1 637 finally: 638 if caddy: 639 caddy.stop() 640 if nghttpx: 641 nghttpx.stop(wait_dead=False) 642 if httpd: 643 httpd.stop() 644 sys.exit(rv) 645 646 647if __name__ == "__main__": 648 main() 649
