1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import argparse 28import json 29import logging 30import os 31import re 32import sys 33from statistics import mean 34from typing import Dict, Any, Optional, List 35 36from testenv import Env, Httpd, CurlClient, Caddy, ExecResult, NghttpxQuic, RunProfile 37 38log = logging.getLogger(__name__) 39 40 41class ScoreCardError(Exception): 42 pass 43 44 45class ScoreCard: 46 47 def __init__(self, env: Env, 48 protocol: str, 49 server_descr: str, 50 server_port: int, 51 verbose: int, 52 curl_verbose: int, 53 download_parallel: int = 0, 54 server_addr: Optional[str] = None): 55 self.verbose = verbose 56 self.env = env 57 self.protocol = protocol 58 self.server_descr = server_descr 59 self.server_addr = server_addr 60 self.server_port = server_port 61 self._silent_curl = not curl_verbose 62 self._download_parallel = download_parallel 63 64 def info(self, msg): 65 if self.verbose > 0: 66 sys.stderr.write(msg) 67 sys.stderr.flush() 68 69 def handshakes(self) -> Dict[str, Any]: 70 props = {} 71 sample_size = 5 72 self.info('TLS Handshake\n') 73 for authority in [ 74 'curl.se', 'google.com', 'cloudflare.com', 'nghttp2.org' 75 ]: 76 self.info(f' {authority}...') 77 props[authority] = {} 78 for ipv in ['ipv4', 'ipv6']: 79 self.info(f'{ipv}...') 80 c_samples = [] 81 hs_samples = [] 82 errors = [] 83 for _ in range(sample_size): 84 curl = CurlClient(env=self.env, silent=self._silent_curl, 85 server_addr=self.server_addr) 86 args = [ 87 '--http3-only' if self.protocol == 'h3' else '--http2', 88 f'--{ipv}', f'https://{authority}/' 89 ] 90 r = curl.run_direct(args=args, with_stats=True) 91 if r.exit_code == 0 and len(r.stats) == 1: 92 c_samples.append(r.stats[0]['time_connect']) 93 hs_samples.append(r.stats[0]['time_appconnect']) 94 else: 95 errors.append(f'exit={r.exit_code}') 96 props[authority][f'{ipv}-connect'] = mean(c_samples) \ 97 if len(c_samples) else -1 98 props[authority][f'{ipv}-handshake'] = mean(hs_samples) \ 99 if len(hs_samples) else -1 100 props[authority][f'{ipv}-errors'] = errors 101 self.info('ok.\n') 102 return props 103 104 def _make_docs_file(self, docs_dir: str, fname: str, fsize: int): 105 fpath = os.path.join(docs_dir, fname) 106 data1k = 1024*'x' 107 flen = 0 108 with open(fpath, 'w') as fd: 109 while flen < fsize: 110 fd.write(data1k) 111 flen += len(data1k) 112 return fpath 113 114 def setup_resources(self, server_docs: str, 115 downloads: Optional[List[int]] = None): 116 for fsize in downloads: 117 label = self.fmt_size(fsize) 118 fname = f'score{label}.data' 119 self._make_docs_file(docs_dir=server_docs, 120 fname=fname, fsize=fsize) 121 self._make_docs_file(docs_dir=server_docs, 122 fname='reqs10.data', fsize=10*1024) 123 124 def _check_downloads(self, r: ExecResult, count: int): 125 error = '' 126 if r.exit_code != 0: 127 error += f'exit={r.exit_code} ' 128 if r.exit_code != 0 or len(r.stats) != count: 129 error += f'stats={len(r.stats)}/{count} ' 130 fails = [s for s in r.stats if s['response_code'] != 200] 131 if len(fails) > 0: 132 error += f'{len(fails)} failed' 133 return error if len(error) > 0 else None 134 135 def transfer_single(self, url: str, count: int): 136 sample_size = count 137 count = 1 138 samples = [] 139 errors = [] 140 profiles = [] 141 self.info('single...') 142 for _ in range(sample_size): 143 curl = CurlClient(env=self.env, silent=self._silent_curl, 144 server_addr=self.server_addr) 145 r = curl.http_download(urls=[url], alpn_proto=self.protocol, 146 no_save=True, with_headers=False, 147 with_profile=True) 148 err = self._check_downloads(r, count) 149 if err: 150 errors.append(err) 151 else: 152 total_size = sum([s['size_download'] for s in r.stats]) 153 samples.append(total_size / r.duration.total_seconds()) 154 profiles.append(r.profile) 155 return { 156 'count': count, 157 'samples': sample_size, 158 'max-parallel': 1, 159 'speed': mean(samples) if len(samples) else -1, 160 'errors': errors, 161 'stats': RunProfile.AverageStats(profiles), 162 } 163 164 def transfer_serial(self, url: str, count: int): 165 sample_size = 1 166 samples = [] 167 errors = [] 168 profiles = [] 169 url = f'{url}?[0-{count - 1}]' 170 self.info('serial...') 171 for _ in range(sample_size): 172 curl = CurlClient(env=self.env, silent=self._silent_curl, 173 server_addr=self.server_addr) 174 r = curl.http_download(urls=[url], alpn_proto=self.protocol, 175 no_save=True, 176 with_headers=False, with_profile=True) 177 err = self._check_downloads(r, count) 178 if err: 179 errors.append(err) 180 else: 181 total_size = sum([s['size_download'] for s in r.stats]) 182 samples.append(total_size / r.duration.total_seconds()) 183 profiles.append(r.profile) 184 return { 185 'count': count, 186 'samples': sample_size, 187 'max-parallel': 1, 188 'speed': mean(samples) if len(samples) else -1, 189 'errors': errors, 190 'stats': RunProfile.AverageStats(profiles), 191 } 192 193 def transfer_parallel(self, url: str, count: int): 194 sample_size = 1 195 samples = [] 196 errors = [] 197 profiles = [] 198 max_parallel = self._download_parallel if self._download_parallel > 0 else count 199 url = f'{url}?[0-{count - 1}]' 200 self.info('parallel...') 201 for _ in range(sample_size): 202 curl = CurlClient(env=self.env, silent=self._silent_curl, 203 server_addr=self.server_addr) 204 r = curl.http_download(urls=[url], alpn_proto=self.protocol, 205 no_save=True, 206 with_headers=False, 207 with_profile=True, 208 extra_args=[ 209 '--parallel', 210 '--parallel-max', str(max_parallel) 211 ]) 212 err = self._check_downloads(r, count) 213 if err: 214 errors.append(err) 215 else: 216 total_size = sum([s['size_download'] for s in r.stats]) 217 samples.append(total_size / r.duration.total_seconds()) 218 profiles.append(r.profile) 219 return { 220 'count': count, 221 'samples': sample_size, 222 'max-parallel': max_parallel, 223 'speed': mean(samples) if len(samples) else -1, 224 'errors': errors, 225 'stats': RunProfile.AverageStats(profiles), 226 } 227 228 def download_url(self, label: str, url: str, count: int): 229 self.info(f' {count}x{label}: ') 230 props = { 231 'single': self.transfer_single(url=url, count=10), 232 } 233 if count > 1: 234 props['serial'] = self.transfer_serial(url=url, count=count) 235 props['parallel'] = self.transfer_parallel(url=url, count=count) 236 self.info('ok.\n') 237 return props 238 239 def downloads(self, count: int, fsizes: List[int]) -> Dict[str, Any]: 240 scores = {} 241 for fsize in fsizes: 242 label = self.fmt_size(fsize) 243 fname = f'score{label}.data' 244 url = f'https://{self.env.domain1}:{self.server_port}/{fname}' 245 scores[label] = self.download_url(label=label, url=url, count=count) 246 return scores 247 248 def _check_uploads(self, r: ExecResult, count: int): 249 error = '' 250 if r.exit_code != 0: 251 error += f'exit={r.exit_code} ' 252 if r.exit_code != 0 or len(r.stats) != count: 253 error += f'stats={len(r.stats)}/{count} ' 254 fails = [s for s in r.stats if s['response_code'] != 200] 255 if len(fails) > 0: 256 error += f'{len(fails)} failed' 257 for f in fails: 258 error += f'[{f["response_code"]}]' 259 return error if len(error) > 0 else None 260 261 def upload_single(self, url: str, fpath: str, count: int): 262 sample_size = count 263 count = 1 264 samples = [] 265 errors = [] 266 profiles = [] 267 self.info('single...') 268 for _ in range(sample_size): 269 curl = CurlClient(env=self.env, silent=self._silent_curl, 270 server_addr=self.server_addr) 271 r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, 272 with_headers=False, with_profile=True) 273 err = self._check_uploads(r, count) 274 if err: 275 errors.append(err) 276 else: 277 total_size = sum([s['size_upload'] for s in r.stats]) 278 samples.append(total_size / r.duration.total_seconds()) 279 profiles.append(r.profile) 280 return { 281 'count': count, 282 'samples': sample_size, 283 'max-parallel': 1, 284 'speed': mean(samples) if len(samples) else -1, 285 'errors': errors, 286 'stats': RunProfile.AverageStats(profiles) if len(profiles) else {}, 287 } 288 289 def upload_serial(self, url: str, fpath: str, count: int): 290 sample_size = 1 291 samples = [] 292 errors = [] 293 profiles = [] 294 url = f'{url}?id=[0-{count - 1}]' 295 self.info('serial...') 296 for _ in range(sample_size): 297 curl = CurlClient(env=self.env, silent=self._silent_curl, 298 server_addr=self.server_addr) 299 r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, 300 with_headers=False, with_profile=True) 301 err = self._check_uploads(r, count) 302 if err: 303 errors.append(err) 304 else: 305 total_size = sum([s['size_upload'] for s in r.stats]) 306 samples.append(total_size / r.duration.total_seconds()) 307 profiles.append(r.profile) 308 return { 309 'count': count, 310 'samples': sample_size, 311 'max-parallel': 1, 312 'speed': mean(samples) if len(samples) else -1, 313 'errors': errors, 314 'stats': RunProfile.AverageStats(profiles) if len(profiles) else {}, 315 } 316 317 def upload_parallel(self, url: str, fpath: str, count: int): 318 sample_size = 1 319 samples = [] 320 errors = [] 321 profiles = [] 322 max_parallel = count 323 url = f'{url}?id=[0-{count - 1}]' 324 self.info('parallel...') 325 for _ in range(sample_size): 326 curl = CurlClient(env=self.env, silent=self._silent_curl, 327 server_addr=self.server_addr) 328 r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, 329 with_headers=False, with_profile=True, 330 extra_args=[ 331 '--parallel', 332 '--parallel-max', str(max_parallel) 333 ]) 334 err = self._check_uploads(r, count) 335 if err: 336 errors.append(err) 337 else: 338 total_size = sum([s['size_upload'] for s in r.stats]) 339 samples.append(total_size / r.duration.total_seconds()) 340 profiles.append(r.profile) 341 return { 342 'count': count, 343 'samples': sample_size, 344 'max-parallel': max_parallel, 345 'speed': mean(samples) if len(samples) else -1, 346 'errors': errors, 347 'stats': RunProfile.AverageStats(profiles) if len(profiles) else {}, 348 } 349 350 def upload_url(self, label: str, url: str, fpath: str, count: int): 351 self.info(f' {count}x{label}: ') 352 props = { 353 'single': self.upload_single(url=url, fpath=fpath, count=10), 354 } 355 if count > 1: 356 props['serial'] = self.upload_serial(url=url, fpath=fpath, count=count) 357 props['parallel'] = self.upload_parallel(url=url, fpath=fpath, count=count) 358 self.info('ok.\n') 359 return props 360 361 def uploads(self, count: int, fsizes: List[int]) -> Dict[str, Any]: 362 scores = {} 363 url = f'https://{self.env.domain2}:{self.server_port}/curltest/put' 364 fpaths = {} 365 for fsize in fsizes: 366 label = self.fmt_size(fsize) 367 fname = f'upload{label}.data' 368 fpaths[label] = self._make_docs_file(docs_dir=self.env.gen_dir, 369 fname=fname, fsize=fsize) 370 371 for label, fpath in fpaths.items(): 372 scores[label] = self.upload_url(label=label, url=url, fpath=fpath, 373 count=count) 374 return scores 375 376 def do_requests(self, url: str, count: int, max_parallel: int = 1): 377 sample_size = 1 378 samples = [] 379 errors = [] 380 profiles = [] 381 url = f'{url}?[0-{count - 1}]' 382 extra_args = [ 383 '-w', '%{response_code},\\n', 384 ] 385 if max_parallel > 1: 386 extra_args.extend([ 387 '--parallel', '--parallel-max', str(max_parallel) 388 ]) 389 self.info(f'{max_parallel}...') 390 for _ in range(sample_size): 391 curl = CurlClient(env=self.env, silent=self._silent_curl, 392 server_addr=self.server_addr) 393 r = curl.http_download(urls=[url], alpn_proto=self.protocol, no_save=True, 394 with_headers=False, with_profile=True, 395 with_stats=False, extra_args=extra_args) 396 if r.exit_code != 0: 397 errors.append(f'exit={r.exit_code}') 398 else: 399 samples.append(count / r.duration.total_seconds()) 400 non_200s = 0 401 for line in r.stdout.splitlines(): 402 if not line.startswith('200,'): 403 non_200s += 1 404 if non_200s > 0: 405 errors.append(f'responses != 200: {non_200s}') 406 profiles.append(r.profile) 407 return { 408 'count': count, 409 'samples': sample_size, 410 'speed': mean(samples) if len(samples) else -1, 411 'errors': errors, 412 'stats': RunProfile.AverageStats(profiles), 413 } 414 415 def requests_url(self, url: str, count: int): 416 self.info(f' {url}: ') 417 props = {} 418 # 300 is max in curl, see tool_main.h 419 for m in [1, 6, 25, 50, 100, 300]: 420 props[str(m)] = self.do_requests(url=url, count=count, max_parallel=m) 421 self.info('ok.\n') 422 return props 423 424 def requests(self, req_count) -> Dict[str, Any]: 425 url = f'https://{self.env.domain1}:{self.server_port}/reqs10.data' 426 return { 427 'count': req_count, 428 '10KB': self.requests_url(url=url, count=req_count), 429 } 430 431 def score(self, 432 handshakes: bool = True, 433 downloads: Optional[List[int]] = None, 434 download_count: int = 50, 435 uploads: Optional[List[int]] = None, 436 upload_count: int = 50, 437 req_count=5000, 438 requests: bool = True): 439 self.info(f"scoring {self.protocol} against {self.server_descr}\n") 440 p = {} 441 if self.protocol == 'h3': 442 p['name'] = 'h3' 443 if not self.env.have_h3_curl(): 444 raise ScoreCardError('curl does not support HTTP/3') 445 for lib in ['ngtcp2', 'quiche', 'msh3', 'nghttp3']: 446 if self.env.curl_uses_lib(lib): 447 p['implementation'] = lib 448 break 449 elif self.protocol == 'h2': 450 p['name'] = 'h2' 451 if not self.env.have_h2_curl(): 452 raise ScoreCardError('curl does not support HTTP/2') 453 for lib in ['nghttp2', 'hyper']: 454 if self.env.curl_uses_lib(lib): 455 p['implementation'] = lib 456 break 457 elif self.protocol == 'h1' or self.protocol == 'http/1.1': 458 proto = 'http/1.1' 459 p['name'] = proto 460 p['implementation'] = 'hyper' if self.env.curl_uses_lib('hyper')\ 461 else 'native' 462 else: 463 raise ScoreCardError(f"unknown protocol: {self.protocol}") 464 465 if 'implementation' not in p: 466 raise ScoreCardError(f'did not recognized {p} lib') 467 p['version'] = Env.curl_lib_version(p['implementation']) 468 469 score = { 470 'curl': self.env.curl_fullname(), 471 'os': self.env.curl_os(), 472 'protocol': p, 473 'server': self.server_descr, 474 } 475 if handshakes: 476 score['handshakes'] = self.handshakes() 477 if downloads and len(downloads) > 0: 478 score['downloads'] = self.downloads(count=download_count, 479 fsizes=downloads) 480 if uploads and len(uploads) > 0: 481 score['uploads'] = self.uploads(count=upload_count, 482 fsizes=uploads) 483 if requests: 484 score['requests'] = self.requests(req_count=req_count) 485 self.info("\n") 486 return score 487 488 def fmt_ms(self, tval): 489 return f'{int(tval*1000)} ms' if tval >= 0 else '--' 490 491 def fmt_size(self, val): 492 if val >= (1024*1024*1024): 493 return f'{val / (1024*1024*1024):0.000f}GB' 494 elif val >= (1024 * 1024): 495 return f'{val / (1024*1024):0.000f}MB' 496 elif val >= 1024: 497 return f'{val / 1024:0.000f}KB' 498 else: 499 return f'{val:0.000f}B' 500 501 def fmt_mbs(self, val): 502 return f'{val/(1024*1024):0.000f} MB/s' if val >= 0 else '--' 503 504 def fmt_reqs(self, val): 505 return f'{val:0.000f} r/s' if val >= 0 else '--' 506 507 def print_score(self, score): 508 print(f'{score["protocol"]["name"].upper()} in {score["curl"]}') 509 if 'handshakes' in score: 510 print(f'{"Handshakes":<24} {"ipv4":25} {"ipv6":28}') 511 print(f' {"Host":<17} {"Connect":>12} {"Handshake":>12} ' 512 f'{"Connect":>12} {"Handshake":>12} {"Errors":<20}') 513 for key, val in score["handshakes"].items(): 514 print(f' {key:<17} {self.fmt_ms(val["ipv4-connect"]):>12} ' 515 f'{self.fmt_ms(val["ipv4-handshake"]):>12} ' 516 f'{self.fmt_ms(val["ipv6-connect"]):>12} ' 517 f'{self.fmt_ms(val["ipv6-handshake"]):>12} ' 518 f'{"/".join(val["ipv4-errors"] + val["ipv6-errors"]):<20}' 519 ) 520 if 'downloads' in score: 521 # get the key names of all sizes and measurements made 522 sizes = [] 523 measures = [] 524 m_names = {} 525 mcol_width = 12 526 mcol_sw = 17 527 for sskey, ssval in score['downloads'].items(): 528 if isinstance(ssval, str): 529 continue 530 if sskey not in sizes: 531 sizes.append(sskey) 532 for mkey, mval in score['downloads'][sskey].items(): 533 if mkey not in measures: 534 measures.append(mkey) 535 m_names[mkey] = f'{mkey}({mval["count"]}x{mval["max-parallel"]})' 536 print(f'Downloads from {score["server"]}') 537 print(f' {"Size":>8}', end='') 538 for m in measures: 539 print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='') 540 print(f' {"Errors":^20}') 541 542 for size in score['downloads']: 543 size_score = score['downloads'][size] 544 print(f' {size:>8}', end='') 545 errors = [] 546 for val in size_score.values(): 547 if 'errors' in val: 548 errors.extend(val['errors']) 549 for m in measures: 550 if m in size_score: 551 print(f' {self.fmt_mbs(size_score[m]["speed"]):>{mcol_width}}', end='') 552 s = f'[{size_score[m]["stats"]["cpu"]:>.1f}%'\ 553 f'/{self.fmt_size(size_score[m]["stats"]["rss"])}]' 554 print(f' {s:<{mcol_sw}}', end='') 555 else: 556 print(' '*mcol_width, end='') 557 if len(errors): 558 print(f' {"/".join(errors):<20}') 559 else: 560 print(f' {"-":^20}') 561 562 if 'uploads' in score: 563 # get the key names of all sizes and measurements made 564 sizes = [] 565 measures = [] 566 m_names = {} 567 mcol_width = 12 568 mcol_sw = 17 569 for sskey, ssval in score['uploads'].items(): 570 if isinstance(ssval, str): 571 continue 572 if sskey not in sizes: 573 sizes.append(sskey) 574 for mkey, mval in ssval.items(): 575 if mkey not in measures: 576 measures.append(mkey) 577 m_names[mkey] = f'{mkey}({mval["count"]}x{mval["max-parallel"]})' 578 579 print(f'Uploads to {score["server"]}') 580 print(f' {"Size":>8}', end='') 581 for m in measures: 582 print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='') 583 print(f' {"Errors":^20}') 584 585 for size in sizes: 586 size_score = score['uploads'][size] 587 print(f' {size:>8}', end='') 588 errors = [] 589 for val in size_score.values(): 590 if 'errors' in val: 591 errors.extend(val['errors']) 592 for m in measures: 593 if m in size_score: 594 print(f' {self.fmt_mbs(size_score[m]["speed"]):>{mcol_width}}', end='') 595 stats = size_score[m]["stats"] 596 if 'cpu' in stats: 597 s = f'[{stats["cpu"]:>.1f}%/{self.fmt_size(stats["rss"])}]' 598 else: 599 s = '[???/???]' 600 print(f' {s:<{mcol_sw}}', end='') 601 else: 602 print(' '*mcol_width, end='') 603 if len(errors): 604 print(f' {"/".join(errors):<20}') 605 else: 606 print(f' {"-":^20}') 607 608 if 'requests' in score: 609 sizes = [] 610 measures = [] 611 m_names = {} 612 mcol_width = 9 613 mcol_sw = 13 614 for sskey, ssval in score['requests'].items(): 615 if isinstance(ssval, (str, int)): 616 continue 617 if sskey not in sizes: 618 sizes.append(sskey) 619 for mkey in score['requests'][sskey]: 620 if mkey not in measures: 621 measures.append(mkey) 622 m_names[mkey] = f'{mkey}' 623 624 print('Requests (max parallel) to {score["server"]}') 625 print(f' {"Size":>6} {"Reqs":>6}', end='') 626 for m in measures: 627 print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='') 628 print(f' {"Errors":^10}') 629 630 for size in sizes: 631 size_score = score['requests'][size] 632 count = score['requests']['count'] 633 print(f' {size:>6} {count:>6}', end='') 634 errors = [] 635 for val in size_score.values(): 636 if 'errors' in val: 637 errors.extend(val['errors']) 638 for m in measures: 639 if m in size_score: 640 print(f' {self.fmt_reqs(size_score[m]["speed"]):>{mcol_width}}', end='') 641 s = f'[{size_score[m]["stats"]["cpu"]:>.1f}%'\ 642 f'/{self.fmt_size(size_score[m]["stats"]["rss"])}]' 643 print(f' {s:<{mcol_sw}}', end='') 644 else: 645 print(' '*mcol_width, end='') 646 if len(errors): 647 print(f' {"/".join(errors):<10}') 648 else: 649 print(f' {"-":^10}') 650 651 652def parse_size(s): 653 m = re.match(r'(\d+)(mb|kb|gb)?', s, re.IGNORECASE) 654 if m is None: 655 raise Exception(f'unrecognized size: {s}') 656 size = int(m.group(1)) 657 if not m.group(2): 658 pass 659 elif m.group(2).lower() == 'kb': 660 size *= 1024 661 elif m.group(2).lower() == 'mb': 662 size *= 1024 * 1024 663 elif m.group(2).lower() == 'gb': 664 size *= 1024 * 1024 * 1024 665 return size 666 667 668def main(): 669 parser = argparse.ArgumentParser(prog='scorecard', description=""" 670 Run a range of tests to give a scorecard for a HTTP protocol 671 'h3' or 'h2' implementation in curl. 672 """) 673 parser.add_argument("-v", "--verbose", action='count', default=1, 674 help="log more output on stderr") 675 parser.add_argument("-j", "--json", action='store_true', 676 default=False, help="print json instead of text") 677 parser.add_argument("-H", "--handshakes", action='store_true', 678 default=False, help="evaluate handshakes only") 679 parser.add_argument("-d", "--downloads", action='store_true', 680 default=False, help="evaluate downloads") 681 parser.add_argument("--download", action='append', type=str, 682 default=None, help="evaluate download size") 683 parser.add_argument("--download-count", action='store', type=int, 684 default=50, help="perform that many downloads") 685 parser.add_argument("--download-parallel", action='store', type=int, 686 default=0, help="perform that many downloads in parallel (default all)") 687 parser.add_argument("-u", "--uploads", action='store_true', 688 default=False, help="evaluate uploads") 689 parser.add_argument("--upload", action='append', type=str, 690 default=None, help="evaluate upload size") 691 parser.add_argument("--upload-count", action='store', type=int, 692 default=50, help="perform that many uploads") 693 parser.add_argument("-r", "--requests", action='store_true', 694 default=False, help="evaluate requests") 695 parser.add_argument("--request-count", action='store', type=int, 696 default=5000, help="perform that many requests") 697 parser.add_argument("--httpd", action='store_true', default=False, 698 help="evaluate httpd server only") 699 parser.add_argument("--caddy", action='store_true', default=False, 700 help="evaluate caddy server only") 701 parser.add_argument("--curl-verbose", action='store_true', 702 default=False, help="run curl with `-v`") 703 parser.add_argument("protocol", default='h2', nargs='?', 704 help="Name of protocol to score") 705 parser.add_argument("--start-only", action='store_true', default=False, 706 help="only start the servers") 707 parser.add_argument("--remote", action='store', type=str, 708 default=None, help="score against the remote server at <ip>:<port>") 709 args = parser.parse_args() 710 711 if args.verbose > 0: 712 console = logging.StreamHandler() 713 console.setLevel(logging.INFO) 714 console.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) 715 logging.getLogger('').addHandler(console) 716 717 protocol = args.protocol 718 handshakes = True 719 downloads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024] 720 if args.download is not None: 721 downloads = [] 722 for x in args.download: 723 downloads.extend([parse_size(s) for s in x.split(',')]) 724 725 uploads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024] 726 if args.upload is not None: 727 uploads = [] 728 for x in args.upload: 729 uploads.extend([parse_size(s) for s in x.split(',')]) 730 731 requests = True 732 if args.downloads or args.uploads or args.requests or args.handshakes: 733 handshakes = args.handshakes 734 if not args.downloads: 735 downloads = None 736 if not args.uploads: 737 uploads = None 738 requests = args.requests 739 740 test_httpd = protocol != 'h3' 741 test_caddy = True 742 if args.caddy or args.httpd: 743 test_caddy = args.caddy 744 test_httpd = args.httpd 745 746 rv = 0 747 env = Env() 748 env.setup() 749 env.test_timeout = None 750 httpd = None 751 nghttpx = None 752 caddy = None 753 try: 754 cards = [] 755 756 if args.remote: 757 m = re.match(r'^(.+):(\d+)$', args.remote) 758 if m is None: 759 raise ScoreCardError(f'unable to parse ip:port from --remote {args.remote}') 760 test_httpd = False 761 test_caddy = False 762 remote_addr = m.group(1) 763 remote_port = int(m.group(2)) 764 card = ScoreCard(env=env, 765 protocol=protocol, 766 server_descr=f'Server at {args.remote}', 767 server_addr=remote_addr, 768 server_port=remote_port, 769 verbose=args.verbose, curl_verbose=args.curl_verbose, 770 download_parallel=args.download_parallel) 771 cards.append(card) 772 773 if test_httpd: 774 httpd = Httpd(env=env) 775 assert httpd.exists(), \ 776 f'httpd not found: {env.httpd}' 777 httpd.clear_logs() 778 server_docs = httpd.docs_dir 779 assert httpd.start() 780 if protocol == 'h3': 781 nghttpx = NghttpxQuic(env=env) 782 nghttpx.clear_logs() 783 assert nghttpx.start() 784 server_descr = f'nghttpx: https:{env.h3_port} [backend httpd: {env.httpd_version()}, https:{env.https_port}]' 785 server_port = env.h3_port 786 else: 787 server_descr = f'httpd: {env.httpd_version()}, http:{env.http_port} https:{env.https_port}' 788 server_port = env.https_port 789 card = ScoreCard(env=env, 790 protocol=protocol, 791 server_descr=server_descr, 792 server_port=server_port, 793 verbose=args.verbose, curl_verbose=args.curl_verbose, 794 download_parallel=args.download_parallel) 795 card.setup_resources(server_docs, downloads) 796 cards.append(card) 797 798 if test_caddy and env.caddy: 799 backend = '' 800 if uploads and httpd is None: 801 backend = f' [backend httpd: {env.httpd_version()}, http:{env.http_port} https:{env.https_port}]' 802 httpd = Httpd(env=env) 803 assert httpd.exists(), \ 804 f'httpd not found: {env.httpd}' 805 httpd.clear_logs() 806 assert httpd.start() 807 caddy = Caddy(env=env) 808 caddy.clear_logs() 809 assert caddy.start() 810 server_descr = f'Caddy: {env.caddy_version()}, http:{env.caddy_http_port} https:{env.caddy_https_port}{backend}' 811 server_port = caddy.port 812 server_docs = caddy.docs_dir 813 card = ScoreCard(env=env, 814 protocol=protocol, 815 server_descr=server_descr, 816 server_port=server_port, 817 verbose=args.verbose, curl_verbose=args.curl_verbose, 818 download_parallel=args.download_parallel) 819 card.setup_resources(server_docs, downloads) 820 cards.append(card) 821 822 if args.start_only: 823 print('started servers:') 824 for card in cards: 825 print(f'{card.server_descr}') 826 sys.stderr.write('press [RETURN] to finish') 827 sys.stderr.flush() 828 sys.stdin.readline() 829 else: 830 for card in cards: 831 score = card.score(handshakes=handshakes, 832 downloads=downloads, 833 download_count=args.download_count, 834 uploads=uploads, 835 upload_count=args.upload_count, 836 req_count=args.request_count, 837 requests=requests) 838 if args.json: 839 print(json.JSONEncoder(indent=2).encode(score)) 840 else: 841 card.print_score(score) 842 843 except ScoreCardError as ex: 844 sys.stderr.write(f"ERROR: {ex}\n") 845 rv = 1 846 except KeyboardInterrupt: 847 log.warning("aborted") 848 rv = 1 849 finally: 850 if caddy: 851 caddy.stop() 852 if nghttpx: 853 nghttpx.stop(wait_dead=False) 854 if httpd: 855 httpd.stop() 856 sys.exit(rv) 857 858 859if __name__ == "__main__": 860 main() 861
