1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import argparse 28import json 29import logging 30import os 31import re 32import sys 33from statistics import mean 34from typing import Dict, Any, Optional, List 35 36from testenv import Env, Httpd, CurlClient, Caddy, ExecResult, NghttpxQuic, RunProfile 37 38log = logging.getLogger(__name__) 39 40 41class ScoreCardError(Exception): 42 pass 43 44 45class ScoreCard: 46 47 def __init__(self, env: Env, 48 protocol: str, 49 server_descr: str, 50 server_port: int, 51 verbose: int, 52 curl_verbose: int, 53 download_parallel: int = 0, 54 server_addr: Optional[str] = None): 55 self.verbose = verbose 56 self.env = env 57 self.protocol = protocol 58 self.server_descr = server_descr 59 self.server_addr = server_addr 60 self.server_port = server_port 61 self._silent_curl = not curl_verbose 62 self._download_parallel = download_parallel 63 64 def info(self, msg): 65 if self.verbose > 0: 66 sys.stderr.write(msg) 67 sys.stderr.flush() 68 69 def handshakes(self) -> Dict[str, Any]: 70 props = {} 71 sample_size = 5 72 self.info('TLS Handshake\n') 73 for authority in [ 74 'curl.se', 'google.com', 'cloudflare.com', 'nghttp2.org' 75 ]: 76 self.info(f' {authority}...') 77 props[authority] = {} 78 for ipv in ['ipv4', 'ipv6']: 79 self.info(f'{ipv}...') 80 c_samples = [] 81 hs_samples = [] 82 errors = [] 83 for _ in range(sample_size): 84 curl = CurlClient(env=self.env, silent=self._silent_curl, 85 server_addr=self.server_addr) 86 args = [ 87 '--http3-only' if self.protocol == 'h3' else '--http2', 88 f'--{ipv}', f'https://{authority}/' 89 ] 90 r = curl.run_direct(args=args, with_stats=True) 91 if r.exit_code == 0 and len(r.stats) == 1: 92 c_samples.append(r.stats[0]['time_connect']) 93 hs_samples.append(r.stats[0]['time_appconnect']) 94 else: 95 errors.append(f'exit={r.exit_code}') 96 props[authority][f'{ipv}-connect'] = mean(c_samples) \ 97 if len(c_samples) else -1 98 props[authority][f'{ipv}-handshake'] = mean(hs_samples) \ 99 if len(hs_samples) else -1 100 props[authority][f'{ipv}-errors'] = errors 101 self.info('ok.\n') 102 return props 103 104 def _make_docs_file(self, docs_dir: str, fname: str, fsize: int): 105 fpath = os.path.join(docs_dir, fname) 106 data1k = 1024*'x' 107 flen = 0 108 with open(fpath, 'w') as fd: 109 while flen < fsize: 110 fd.write(data1k) 111 flen += len(data1k) 112 return fpath 113 114 def setup_resources(self, server_docs: str, 115 downloads: Optional[List[int]] = None): 116 for fsize in downloads: 117 label = self.fmt_size(fsize) 118 fname = f'score{label}.data' 119 self._make_docs_file(docs_dir=server_docs, 120 fname=fname, fsize=fsize) 121 self._make_docs_file(docs_dir=server_docs, 122 fname='reqs10.data', fsize=10*1024) 123 124 def _check_downloads(self, r: ExecResult, count: int): 125 error = '' 126 if r.exit_code != 0: 127 error += f'exit={r.exit_code} ' 128 if r.exit_code != 0 or len(r.stats) != count: 129 error += f'stats={len(r.stats)}/{count} ' 130 fails = [s for s in r.stats if s['response_code'] != 200] 131 if len(fails) > 0: 132 error += f'{len(fails)} failed' 133 return error if len(error) > 0 else None 134 135 def transfer_single(self, url: str, count: int): 136 sample_size = count 137 count = 1 138 samples = [] 139 errors = [] 140 profiles = [] 141 self.info('single...') 142 for _ in range(sample_size): 143 curl = CurlClient(env=self.env, silent=self._silent_curl, 144 server_addr=self.server_addr) 145 r = curl.http_download(urls=[url], alpn_proto=self.protocol, 146 no_save=True, with_headers=False, 147 with_profile=True) 148 err = self._check_downloads(r, count) 149 if err: 150 errors.append(err) 151 else: 152 total_size = sum([s['size_download'] for s in r.stats]) 153 samples.append(total_size / r.duration.total_seconds()) 154 profiles.append(r.profile) 155 return { 156 'count': count, 157 'samples': sample_size, 158 'max-parallel': 1, 159 'speed': mean(samples) if len(samples) else -1, 160 'errors': errors, 161 'stats': RunProfile.AverageStats(profiles), 162 } 163 164 def transfer_serial(self, url: str, count: int): 165 sample_size = 1 166 samples = [] 167 errors = [] 168 profiles = [] 169 url = f'{url}?[0-{count - 1}]' 170 self.info('serial...') 171 for _ in range(sample_size): 172 curl = CurlClient(env=self.env, silent=self._silent_curl, 173 server_addr=self.server_addr) 174 r = curl.http_download(urls=[url], alpn_proto=self.protocol, 175 no_save=True, 176 with_headers=False, with_profile=True) 177 err = self._check_downloads(r, count) 178 if err: 179 errors.append(err) 180 else: 181 total_size = sum([s['size_download'] for s in r.stats]) 182 samples.append(total_size / r.duration.total_seconds()) 183 profiles.append(r.profile) 184 return { 185 'count': count, 186 'samples': sample_size, 187 'max-parallel': 1, 188 'speed': mean(samples) if len(samples) else -1, 189 'errors': errors, 190 'stats': RunProfile.AverageStats(profiles), 191 } 192 193 def transfer_parallel(self, url: str, count: int): 194 sample_size = 1 195 samples = [] 196 errors = [] 197 profiles = [] 198 max_parallel = self._download_parallel if self._download_parallel > 0 else count 199 url = f'{url}?[0-{count - 1}]' 200 self.info('parallel...') 201 for _ in range(sample_size): 202 curl = CurlClient(env=self.env, silent=self._silent_curl, 203 server_addr=self.server_addr) 204 r = curl.http_download(urls=[url], alpn_proto=self.protocol, 205 no_save=True, 206 with_headers=False, 207 with_profile=True, 208 extra_args=[ 209 '--parallel', 210 '--parallel-max', str(max_parallel) 211 ]) 212 err = self._check_downloads(r, count) 213 if err: 214 errors.append(err) 215 else: 216 total_size = sum([s['size_download'] for s in r.stats]) 217 samples.append(total_size / r.duration.total_seconds()) 218 profiles.append(r.profile) 219 return { 220 'count': count, 221 'samples': sample_size, 222 'max-parallel': max_parallel, 223 'speed': mean(samples) if len(samples) else -1, 224 'errors': errors, 225 'stats': RunProfile.AverageStats(profiles), 226 } 227 228 def download_url(self, label: str, url: str, count: int): 229 self.info(f' {count}x{label}: ') 230 props = { 231 'single': self.transfer_single(url=url, count=10), 232 } 233 if count > 1: 234 props['serial'] = self.transfer_serial(url=url, count=count) 235 props['parallel'] = self.transfer_parallel(url=url, count=count) 236 self.info('ok.\n') 237 return props 238 239 def downloads(self, count: int, fsizes: List[int]) -> Dict[str, Any]: 240 scores = {} 241 for fsize in fsizes: 242 label = self.fmt_size(fsize) 243 fname = f'score{label}.data' 244 url = f'https://{self.env.domain1}:{self.server_port}/{fname}' 245 scores[label] = self.download_url(label=label, url=url, count=count) 246 return scores 247 248 def _check_uploads(self, r: ExecResult, count: int): 249 error = '' 250 if r.exit_code != 0: 251 error += f'exit={r.exit_code} ' 252 if r.exit_code != 0 or len(r.stats) != count: 253 error += f'stats={len(r.stats)}/{count} ' 254 fails = [s for s in r.stats if s['response_code'] != 200] 255 if len(fails) > 0: 256 error += f'{len(fails)} failed' 257 for f in fails: 258 error += f'[{f["response_code"]}]' 259 return error if len(error) > 0 else None 260 261 def upload_single(self, url: str, fpath: str, count: int): 262 sample_size = count 263 count = 1 264 samples = [] 265 errors = [] 266 profiles = [] 267 self.info('single...') 268 for _ in range(sample_size): 269 curl = CurlClient(env=self.env, silent=self._silent_curl, 270 server_addr=self.server_addr) 271 r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, 272 with_headers=False, with_profile=True) 273 err = self._check_uploads(r, count) 274 if err: 275 errors.append(err) 276 else: 277 total_size = sum([s['size_upload'] for s in r.stats]) 278 samples.append(total_size / r.duration.total_seconds()) 279 profiles.append(r.profile) 280 return { 281 'count': count, 282 'samples': sample_size, 283 'max-parallel': 1, 284 'speed': mean(samples) if len(samples) else -1, 285 'errors': errors, 286 'stats': RunProfile.AverageStats(profiles) if len(profiles) else {}, 287 } 288 289 def upload_serial(self, url: str, fpath: str, count: int): 290 sample_size = 1 291 samples = [] 292 errors = [] 293 profiles = [] 294 url = f'{url}?id=[0-{count - 1}]' 295 self.info('serial...') 296 for _ in range(sample_size): 297 curl = CurlClient(env=self.env, silent=self._silent_curl, 298 server_addr=self.server_addr) 299 r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, 300 with_headers=False, with_profile=True) 301 err = self._check_uploads(r, count) 302 if err: 303 errors.append(err) 304 else: 305 total_size = sum([s['size_upload'] for s in r.stats]) 306 samples.append(total_size / r.duration.total_seconds()) 307 profiles.append(r.profile) 308 return { 309 'count': count, 310 'samples': sample_size, 311 'max-parallel': 1, 312 'speed': mean(samples) if len(samples) else -1, 313 'errors': errors, 314 'stats': RunProfile.AverageStats(profiles) if len(profiles) else {}, 315 } 316 317 def upload_parallel(self, url: str, fpath: str, count: int): 318 sample_size = 1 319 samples = [] 320 errors = [] 321 profiles = [] 322 max_parallel = count 323 url = f'{url}?id=[0-{count - 1}]' 324 self.info('parallel...') 325 for _ in range(sample_size): 326 curl = CurlClient(env=self.env, silent=self._silent_curl, 327 server_addr=self.server_addr) 328 r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol, 329 with_headers=False, with_profile=True, 330 extra_args=[ 331 '--parallel', 332 '--parallel-max', str(max_parallel) 333 ]) 334 err = self._check_uploads(r, count) 335 if err: 336 errors.append(err) 337 else: 338 total_size = sum([s['size_upload'] for s in r.stats]) 339 samples.append(total_size / r.duration.total_seconds()) 340 profiles.append(r.profile) 341 return { 342 'count': count, 343 'samples': sample_size, 344 'max-parallel': max_parallel, 345 'speed': mean(samples) if len(samples) else -1, 346 'errors': errors, 347 'stats': RunProfile.AverageStats(profiles) if len(profiles) else {}, 348 } 349 350 def upload_url(self, label: str, url: str, fpath: str, count: int): 351 self.info(f' {count}x{label}: ') 352 props = { 353 'single': self.upload_single(url=url, fpath=fpath, count=10), 354 } 355 if count > 1: 356 props['serial'] = self.upload_serial(url=url, fpath=fpath, count=count) 357 props['parallel'] = self.upload_parallel(url=url, fpath=fpath, count=count) 358 self.info('ok.\n') 359 return props 360 361 def uploads(self, count: int, fsizes: List[int]) -> Dict[str, Any]: 362 scores = {} 363 url = f'https://{self.env.domain2}:{self.server_port}/curltest/put' 364 fpaths = {} 365 for fsize in fsizes: 366 label = self.fmt_size(fsize) 367 fname = f'upload{label}.data' 368 fpaths[label] = self._make_docs_file(docs_dir=self.env.gen_dir, 369 fname=fname, fsize=fsize) 370 371 for label, fpath in fpaths.items(): 372 scores[label] = self.upload_url(label=label, url=url, fpath=fpath, 373 count=count) 374 return scores 375 376 def do_requests(self, url: str, count: int, max_parallel: int = 1): 377 sample_size = 1 378 samples = [] 379 errors = [] 380 profiles = [] 381 url = f'{url}?[0-{count - 1}]' 382 extra_args = [ 383 '-w', '%{response_code},\\n', 384 ] 385 if max_parallel > 1: 386 extra_args.extend([ 387 '--parallel', '--parallel-max', str(max_parallel) 388 ]) 389 self.info(f'{max_parallel}...') 390 for _ in range(sample_size): 391 curl = CurlClient(env=self.env, silent=self._silent_curl, 392 server_addr=self.server_addr) 393 r = curl.http_download(urls=[url], alpn_proto=self.protocol, no_save=True, 394 with_headers=False, with_profile=True, 395 with_stats=False, extra_args=extra_args) 396 if r.exit_code != 0: 397 errors.append(f'exit={r.exit_code}') 398 else: 399 samples.append(count / r.duration.total_seconds()) 400 non_200s = 0 401 for line in r.stdout.splitlines(): 402 if not line.startswith('200,'): 403 non_200s += 1 404 if non_200s > 0: 405 errors.append(f'responses != 200: {non_200s}') 406 profiles.append(r.profile) 407 return { 408 'count': count, 409 'samples': sample_size, 410 'speed': mean(samples) if len(samples) else -1, 411 'errors': errors, 412 'stats': RunProfile.AverageStats(profiles), 413 } 414 415 def requests_url(self, url: str, count: int): 416 self.info(f' {url}: ') 417 props = {} 418 # 300 is max in curl, see tool_main.h 419 for m in [1, 6, 25, 50, 100, 300]: 420 props[str(m)] = self.do_requests(url=url, count=count, max_parallel=m) 421 self.info('ok.\n') 422 return props 423 424 def requests(self, req_count) -> Dict[str, Any]: 425 url = f'https://{self.env.domain1}:{self.server_port}/reqs10.data' 426 return { 427 'count': req_count, 428 '10KB': self.requests_url(url=url, count=req_count), 429 } 430 431 def score(self, 432 handshakes: bool = True, 433 downloads: Optional[List[int]] = None, 434 download_count: int = 50, 435 uploads: Optional[List[int]] = None, 436 upload_count: int = 50, 437 req_count=5000, 438 requests: bool = True): 439 self.info(f"scoring {self.protocol} against {self.server_descr}\n") 440 p = {} 441 if self.protocol == 'h3': 442 p['name'] = 'h3' 443 if not self.env.have_h3_curl(): 444 raise ScoreCardError('curl does not support HTTP/3') 445 for lib in ['ngtcp2', 'quiche', 'msh3', 'nghttp3']: 446 if self.env.curl_uses_lib(lib): 447 p['implementation'] = lib 448 break 449 elif self.protocol == 'h2': 450 p['name'] = 'h2' 451 if not self.env.have_h2_curl(): 452 raise ScoreCardError('curl does not support HTTP/2') 453 for lib in ['nghttp2']: 454 if self.env.curl_uses_lib(lib): 455 p['implementation'] = lib 456 break 457 elif self.protocol == 'h1' or self.protocol == 'http/1.1': 458 proto = 'http/1.1' 459 p['name'] = proto 460 p['implementation'] = 'native' 461 else: 462 raise ScoreCardError(f"unknown protocol: {self.protocol}") 463 464 if 'implementation' not in p: 465 raise ScoreCardError(f'did not recognized {p} lib') 466 p['version'] = Env.curl_lib_version(p['implementation']) 467 468 score = { 469 'curl': self.env.curl_fullname(), 470 'os': self.env.curl_os(), 471 'protocol': p, 472 'server': self.server_descr, 473 } 474 if handshakes: 475 score['handshakes'] = self.handshakes() 476 if downloads and len(downloads) > 0: 477 score['downloads'] = self.downloads(count=download_count, 478 fsizes=downloads) 479 if uploads and len(uploads) > 0: 480 score['uploads'] = self.uploads(count=upload_count, 481 fsizes=uploads) 482 if requests: 483 score['requests'] = self.requests(req_count=req_count) 484 self.info("\n") 485 return score 486 487 def fmt_ms(self, tval): 488 return f'{int(tval*1000)} ms' if tval >= 0 else '--' 489 490 def fmt_size(self, val): 491 if val >= (1024*1024*1024): 492 return f'{val / (1024*1024*1024):0.000f}GB' 493 elif val >= (1024 * 1024): 494 return f'{val / (1024*1024):0.000f}MB' 495 elif val >= 1024: 496 return f'{val / 1024:0.000f}KB' 497 else: 498 return f'{val:0.000f}B' 499 500 def fmt_mbs(self, val): 501 return f'{val/(1024*1024):0.000f} MB/s' if val >= 0 else '--' 502 503 def fmt_reqs(self, val): 504 return f'{val:0.000f} r/s' if val >= 0 else '--' 505 506 def print_score(self, score): 507 print(f'{score["protocol"]["name"].upper()} in {score["curl"]}') 508 if 'handshakes' in score: 509 print(f'{"Handshakes":<24} {"ipv4":25} {"ipv6":28}') 510 print(f' {"Host":<17} {"Connect":>12} {"Handshake":>12} ' 511 f'{"Connect":>12} {"Handshake":>12} {"Errors":<20}') 512 for key, val in score["handshakes"].items(): 513 print(f' {key:<17} {self.fmt_ms(val["ipv4-connect"]):>12} ' 514 f'{self.fmt_ms(val["ipv4-handshake"]):>12} ' 515 f'{self.fmt_ms(val["ipv6-connect"]):>12} ' 516 f'{self.fmt_ms(val["ipv6-handshake"]):>12} ' 517 f'{"/".join(val["ipv4-errors"] + val["ipv6-errors"]):<20}' 518 ) 519 if 'downloads' in score: 520 # get the key names of all sizes and measurements made 521 sizes = [] 522 measures = [] 523 m_names = {} 524 mcol_width = 12 525 mcol_sw = 17 526 for sskey, ssval in score['downloads'].items(): 527 if isinstance(ssval, str): 528 continue 529 if sskey not in sizes: 530 sizes.append(sskey) 531 for mkey, mval in score['downloads'][sskey].items(): 532 if mkey not in measures: 533 measures.append(mkey) 534 m_names[mkey] = f'{mkey}({mval["count"]}x{mval["max-parallel"]})' 535 print(f'Downloads from {score["server"]}') 536 print(f' {"Size":>8}', end='') 537 for m in measures: 538 print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='') 539 print(f' {"Errors":^20}') 540 541 for size in score['downloads']: 542 size_score = score['downloads'][size] 543 print(f' {size:>8}', end='') 544 errors = [] 545 for val in size_score.values(): 546 if 'errors' in val: 547 errors.extend(val['errors']) 548 for m in measures: 549 if m in size_score: 550 print(f' {self.fmt_mbs(size_score[m]["speed"]):>{mcol_width}}', end='') 551 s = f'[{size_score[m]["stats"]["cpu"]:>.1f}%'\ 552 f'/{self.fmt_size(size_score[m]["stats"]["rss"])}]' 553 print(f' {s:<{mcol_sw}}', end='') 554 else: 555 print(' '*mcol_width, end='') 556 if len(errors): 557 print(f' {"/".join(errors):<20}') 558 else: 559 print(f' {"-":^20}') 560 561 if 'uploads' in score: 562 # get the key names of all sizes and measurements made 563 sizes = [] 564 measures = [] 565 m_names = {} 566 mcol_width = 12 567 mcol_sw = 17 568 for sskey, ssval in score['uploads'].items(): 569 if isinstance(ssval, str): 570 continue 571 if sskey not in sizes: 572 sizes.append(sskey) 573 for mkey, mval in ssval.items(): 574 if mkey not in measures: 575 measures.append(mkey) 576 m_names[mkey] = f'{mkey}({mval["count"]}x{mval["max-parallel"]})' 577 578 print(f'Uploads to {score["server"]}') 579 print(f' {"Size":>8}', end='') 580 for m in measures: 581 print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='') 582 print(f' {"Errors":^20}') 583 584 for size in sizes: 585 size_score = score['uploads'][size] 586 print(f' {size:>8}', end='') 587 errors = [] 588 for val in size_score.values(): 589 if 'errors' in val: 590 errors.extend(val['errors']) 591 for m in measures: 592 if m in size_score: 593 print(f' {self.fmt_mbs(size_score[m]["speed"]):>{mcol_width}}', end='') 594 stats = size_score[m]["stats"] 595 if 'cpu' in stats: 596 s = f'[{stats["cpu"]:>.1f}%/{self.fmt_size(stats["rss"])}]' 597 else: 598 s = '[???/???]' 599 print(f' {s:<{mcol_sw}}', end='') 600 else: 601 print(' '*mcol_width, end='') 602 if len(errors): 603 print(f' {"/".join(errors):<20}') 604 else: 605 print(f' {"-":^20}') 606 607 if 'requests' in score: 608 sizes = [] 609 measures = [] 610 m_names = {} 611 mcol_width = 9 612 mcol_sw = 13 613 for sskey, ssval in score['requests'].items(): 614 if isinstance(ssval, (str, int)): 615 continue 616 if sskey not in sizes: 617 sizes.append(sskey) 618 for mkey in score['requests'][sskey]: 619 if mkey not in measures: 620 measures.append(mkey) 621 m_names[mkey] = f'{mkey}' 622 623 print('Requests (max parallel) to {score["server"]}') 624 print(f' {"Size":>6} {"Reqs":>6}', end='') 625 for m in measures: 626 print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='') 627 print(f' {"Errors":^10}') 628 629 for size in sizes: 630 size_score = score['requests'][size] 631 count = score['requests']['count'] 632 print(f' {size:>6} {count:>6}', end='') 633 errors = [] 634 for val in size_score.values(): 635 if 'errors' in val: 636 errors.extend(val['errors']) 637 for m in measures: 638 if m in size_score: 639 print(f' {self.fmt_reqs(size_score[m]["speed"]):>{mcol_width}}', end='') 640 s = f'[{size_score[m]["stats"]["cpu"]:>.1f}%'\ 641 f'/{self.fmt_size(size_score[m]["stats"]["rss"])}]' 642 print(f' {s:<{mcol_sw}}', end='') 643 else: 644 print(' '*mcol_width, end='') 645 if len(errors): 646 print(f' {"/".join(errors):<10}') 647 else: 648 print(f' {"-":^10}') 649 650 651def parse_size(s): 652 m = re.match(r'(\d+)(mb|kb|gb)?', s, re.IGNORECASE) 653 if m is None: 654 raise Exception(f'unrecognized size: {s}') 655 size = int(m.group(1)) 656 if not m.group(2): 657 pass 658 elif m.group(2).lower() == 'kb': 659 size *= 1024 660 elif m.group(2).lower() == 'mb': 661 size *= 1024 * 1024 662 elif m.group(2).lower() == 'gb': 663 size *= 1024 * 1024 * 1024 664 return size 665 666 667def main(): 668 parser = argparse.ArgumentParser(prog='scorecard', description=""" 669 Run a range of tests to give a scorecard for a HTTP protocol 670 'h3' or 'h2' implementation in curl. 671 """) 672 parser.add_argument("-v", "--verbose", action='count', default=1, 673 help="log more output on stderr") 674 parser.add_argument("-j", "--json", action='store_true', 675 default=False, help="print json instead of text") 676 parser.add_argument("-H", "--handshakes", action='store_true', 677 default=False, help="evaluate handshakes only") 678 parser.add_argument("-d", "--downloads", action='store_true', 679 default=False, help="evaluate downloads") 680 parser.add_argument("--download", action='append', type=str, 681 default=None, help="evaluate download size") 682 parser.add_argument("--download-count", action='store', type=int, 683 default=50, help="perform that many downloads") 684 parser.add_argument("--download-parallel", action='store', type=int, 685 default=0, help="perform that many downloads in parallel (default all)") 686 parser.add_argument("-u", "--uploads", action='store_true', 687 default=False, help="evaluate uploads") 688 parser.add_argument("--upload", action='append', type=str, 689 default=None, help="evaluate upload size") 690 parser.add_argument("--upload-count", action='store', type=int, 691 default=50, help="perform that many uploads") 692 parser.add_argument("-r", "--requests", action='store_true', 693 default=False, help="evaluate requests") 694 parser.add_argument("--request-count", action='store', type=int, 695 default=5000, help="perform that many requests") 696 parser.add_argument("--httpd", action='store_true', default=False, 697 help="evaluate httpd server only") 698 parser.add_argument("--caddy", action='store_true', default=False, 699 help="evaluate caddy server only") 700 parser.add_argument("--curl-verbose", action='store_true', 701 default=False, help="run curl with `-v`") 702 parser.add_argument("protocol", default='h2', nargs='?', 703 help="Name of protocol to score") 704 parser.add_argument("--start-only", action='store_true', default=False, 705 help="only start the servers") 706 parser.add_argument("--remote", action='store', type=str, 707 default=None, help="score against the remote server at <ip>:<port>") 708 args = parser.parse_args() 709 710 if args.verbose > 0: 711 console = logging.StreamHandler() 712 console.setLevel(logging.INFO) 713 console.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) 714 logging.getLogger('').addHandler(console) 715 716 protocol = args.protocol 717 handshakes = True 718 downloads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024] 719 if args.download is not None: 720 downloads = [] 721 for x in args.download: 722 downloads.extend([parse_size(s) for s in x.split(',')]) 723 724 uploads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024] 725 if args.upload is not None: 726 uploads = [] 727 for x in args.upload: 728 uploads.extend([parse_size(s) for s in x.split(',')]) 729 730 requests = True 731 if args.downloads or args.uploads or args.requests or args.handshakes: 732 handshakes = args.handshakes 733 if not args.downloads: 734 downloads = None 735 if not args.uploads: 736 uploads = None 737 requests = args.requests 738 739 test_httpd = protocol != 'h3' 740 test_caddy = True 741 if args.caddy or args.httpd: 742 test_caddy = args.caddy 743 test_httpd = args.httpd 744 745 rv = 0 746 env = Env() 747 env.setup() 748 env.test_timeout = None 749 httpd = None 750 nghttpx = None 751 caddy = None 752 try: 753 cards = [] 754 755 if args.remote: 756 m = re.match(r'^(.+):(\d+)$', args.remote) 757 if m is None: 758 raise ScoreCardError(f'unable to parse ip:port from --remote {args.remote}') 759 test_httpd = False 760 test_caddy = False 761 remote_addr = m.group(1) 762 remote_port = int(m.group(2)) 763 card = ScoreCard(env=env, 764 protocol=protocol, 765 server_descr=f'Server at {args.remote}', 766 server_addr=remote_addr, 767 server_port=remote_port, 768 verbose=args.verbose, curl_verbose=args.curl_verbose, 769 download_parallel=args.download_parallel) 770 cards.append(card) 771 772 if test_httpd: 773 httpd = Httpd(env=env) 774 assert httpd.exists(), \ 775 f'httpd not found: {env.httpd}' 776 httpd.clear_logs() 777 server_docs = httpd.docs_dir 778 assert httpd.start() 779 if protocol == 'h3': 780 nghttpx = NghttpxQuic(env=env) 781 nghttpx.clear_logs() 782 assert nghttpx.start() 783 server_descr = f'nghttpx: https:{env.h3_port} [backend httpd: {env.httpd_version()}, https:{env.https_port}]' 784 server_port = env.h3_port 785 else: 786 server_descr = f'httpd: {env.httpd_version()}, http:{env.http_port} https:{env.https_port}' 787 server_port = env.https_port 788 card = ScoreCard(env=env, 789 protocol=protocol, 790 server_descr=server_descr, 791 server_port=server_port, 792 verbose=args.verbose, curl_verbose=args.curl_verbose, 793 download_parallel=args.download_parallel) 794 card.setup_resources(server_docs, downloads) 795 cards.append(card) 796 797 if test_caddy and env.caddy: 798 backend = '' 799 if uploads and httpd is None: 800 backend = f' [backend httpd: {env.httpd_version()}, http:{env.http_port} https:{env.https_port}]' 801 httpd = Httpd(env=env) 802 assert httpd.exists(), \ 803 f'httpd not found: {env.httpd}' 804 httpd.clear_logs() 805 assert httpd.start() 806 caddy = Caddy(env=env) 807 caddy.clear_logs() 808 assert caddy.start() 809 server_descr = f'Caddy: {env.caddy_version()}, http:{env.caddy_http_port} https:{env.caddy_https_port}{backend}' 810 server_port = caddy.port 811 server_docs = caddy.docs_dir 812 card = ScoreCard(env=env, 813 protocol=protocol, 814 server_descr=server_descr, 815 server_port=server_port, 816 verbose=args.verbose, curl_verbose=args.curl_verbose, 817 download_parallel=args.download_parallel) 818 card.setup_resources(server_docs, downloads) 819 cards.append(card) 820 821 if args.start_only: 822 print('started servers:') 823 for card in cards: 824 print(f'{card.server_descr}') 825 sys.stderr.write('press [RETURN] to finish') 826 sys.stderr.flush() 827 sys.stdin.readline() 828 else: 829 for card in cards: 830 score = card.score(handshakes=handshakes, 831 downloads=downloads, 832 download_count=args.download_count, 833 uploads=uploads, 834 upload_count=args.upload_count, 835 req_count=args.request_count, 836 requests=requests) 837 if args.json: 838 print(json.JSONEncoder(indent=2).encode(score)) 839 else: 840 card.print_score(score) 841 842 except ScoreCardError as ex: 843 sys.stderr.write(f"ERROR: {ex}\n") 844 rv = 1 845 except KeyboardInterrupt: 846 log.warning("aborted") 847 rv = 1 848 finally: 849 if caddy: 850 caddy.stop() 851 if nghttpx: 852 nghttpx.stop(wait_dead=False) 853 if httpd: 854 httpd.stop() 855 sys.exit(rv) 856 857 858if __name__ == "__main__": 859 main() 860
