1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import json 28import logging 29import os 30import psutil 31import re 32import shutil 33import subprocess 34from statistics import mean, fmean 35from datetime import timedelta, datetime 36from typing import List, Optional, Dict, Union, Any 37from urllib.parse import urlparse 38 39from .env import Env 40 41 42log = logging.getLogger(__name__) 43 44 45class RunProfile: 46 47 STAT_KEYS = ['cpu', 'rss', 'vsz'] 48 49 @classmethod 50 def AverageStats(cls, profiles: List['RunProfile']): 51 avg = {} 52 stats = [p.stats for p in profiles] 53 for key in cls.STAT_KEYS: 54 avg[key] = mean([s[key] for s in stats]) 55 return avg 56 57 def __init__(self, pid: int, started_at: datetime, run_dir): 58 self._pid = pid 59 self._started_at = started_at 60 self._duration = timedelta(seconds=0) 61 self._run_dir = run_dir 62 self._samples = [] 63 self._psu = None 64 self._stats = None 65 66 @property 67 def duration(self) -> timedelta: 68 return self._duration 69 70 @property 71 def stats(self) -> Optional[Dict[str,Any]]: 72 return self._stats 73 74 def sample(self): 75 elapsed = datetime.now() - self._started_at 76 try: 77 if self._psu is None: 78 self._psu = psutil.Process(pid=self._pid) 79 mem = self._psu.memory_info() 80 self._samples.append({ 81 'time': elapsed, 82 'cpu': self._psu.cpu_percent(), 83 'vsz': mem.vms, 84 'rss': mem.rss, 85 }) 86 except psutil.NoSuchProcess: 87 pass 88 89 def finish(self): 90 self._duration = datetime.now() - self._started_at 91 if len(self._samples) > 0: 92 weights = [s['time'].total_seconds() for s in self._samples] 93 self._stats = {} 94 for key in self.STAT_KEYS: 95 self._stats[key] = fmean([s[key] for s in self._samples], weights) 96 else: 97 self._stats = None 98 self._psu = None 99 100 def __repr__(self): 101 return f'RunProfile[pid={self._pid}, '\ 102 f'duration={self.duration.total_seconds():.3f}s, '\ 103 f'stats={self.stats}]' 104 105 106class ExecResult: 107 108 def __init__(self, args: List[str], exit_code: int, 109 stdout: List[str], stderr: List[str], 110 duration: Optional[timedelta] = None, 111 with_stats: bool = False, 112 exception: Optional[str] = None, 113 profile: Optional[RunProfile] = None): 114 self._args = args 115 self._exit_code = exit_code 116 self._exception = exception 117 self._stdout = stdout 118 self._stderr = stderr 119 self._profile = profile 120 self._duration = duration if duration is not None else timedelta() 121 self._response = None 122 self._responses = [] 123 self._results = {} 124 self._assets = [] 125 self._stats = [] 126 self._json_out = None 127 self._with_stats = with_stats 128 if with_stats: 129 self._parse_stats() 130 else: 131 # noinspection PyBroadException 132 try: 133 out = ''.join(self._stdout) 134 self._json_out = json.loads(out) 135 except: 136 pass 137 138 def __repr__(self): 139 return f"ExecResult[code={self.exit_code}, exception={self._exception}, "\ 140 f"args={self._args}, stdout={self._stdout}, stderr={self._stderr}]" 141 142 def _parse_stats(self): 143 self._stats = [] 144 for l in self._stdout: 145 try: 146 self._stats.append(json.loads(l)) 147 except: 148 log.error(f'not a JSON stat: {l}') 149 break 150 151 @property 152 def exit_code(self) -> int: 153 return self._exit_code 154 155 @property 156 def args(self) -> List[str]: 157 return self._args 158 159 @property 160 def outraw(self) -> bytes: 161 return ''.join(self._stdout).encode() 162 163 @property 164 def stdout(self) -> str: 165 return ''.join(self._stdout) 166 167 @property 168 def json(self) -> Optional[Dict]: 169 """Output as JSON dictionary or None if not parseable.""" 170 return self._json_out 171 172 @property 173 def stderr(self) -> str: 174 return ''.join(self._stderr) 175 176 @property 177 def trace_lines(self) -> List[str]: 178 return self._stderr 179 180 @property 181 def duration(self) -> timedelta: 182 return self._duration 183 184 @property 185 def profile(self) -> Optional[RunProfile]: 186 return self._profile 187 188 @property 189 def response(self) -> Optional[Dict]: 190 return self._response 191 192 @property 193 def responses(self) -> List[Dict]: 194 return self._responses 195 196 @property 197 def results(self) -> Dict: 198 return self._results 199 200 @property 201 def assets(self) -> List: 202 return self._assets 203 204 @property 205 def with_stats(self) -> bool: 206 return self._with_stats 207 208 @property 209 def stats(self) -> List: 210 return self._stats 211 212 @property 213 def total_connects(self) -> Optional[int]: 214 if len(self.stats): 215 n = 0 216 for stat in self.stats: 217 n += stat['num_connects'] 218 return n 219 return None 220 221 def add_response(self, resp: Dict): 222 self._response = resp 223 self._responses.append(resp) 224 225 def add_results(self, results: Dict): 226 self._results.update(results) 227 if 'response' in results: 228 self.add_response(results['response']) 229 230 def add_assets(self, assets: List): 231 self._assets.extend(assets) 232 233 def check_exit_code(self, code: Union[int, bool]): 234 if code is True: 235 assert self.exit_code == 0, f'expected exit code {code}, '\ 236 f'got {self.exit_code}\n{self.dump_logs()}' 237 elif code is False: 238 assert self.exit_code != 0, f'expected exit code {code}, '\ 239 f'got {self.exit_code}\n{self.dump_logs()}' 240 else: 241 assert self.exit_code == code, f'expected exit code {code}, '\ 242 f'got {self.exit_code}\n{self.dump_logs()}' 243 244 def check_response(self, http_status: Optional[int] = 200, 245 count: Optional[int] = 1, 246 protocol: Optional[str] = None, 247 exitcode: Optional[int] = 0, 248 connect_count: Optional[int] = None): 249 if exitcode: 250 self.check_exit_code(exitcode) 251 if self.with_stats and isinstance(exitcode, int): 252 for idx, x in enumerate(self.stats): 253 if 'exitcode' in x: 254 assert int(x['exitcode']) == exitcode, \ 255 f'response #{idx} exitcode: expected {exitcode}, '\ 256 f'got {x["exitcode"]}\n{self.dump_logs()}' 257 258 if self.with_stats: 259 assert len(self.stats) == count, \ 260 f'response count: expected {count}, ' \ 261 f'got {len(self.stats)}\n{self.dump_logs()}' 262 else: 263 assert len(self.responses) == count, \ 264 f'response count: expected {count}, ' \ 265 f'got {len(self.responses)}\n{self.dump_logs()}' 266 if http_status is not None: 267 if self.with_stats: 268 for idx, x in enumerate(self.stats): 269 assert 'http_code' in x, \ 270 f'response #{idx} reports no http_code\n{self.dump_stat(x)}' 271 assert x['http_code'] == http_status, \ 272 f'response #{idx} http_code: expected {http_status}, '\ 273 f'got {x["http_code"]}\n{self.dump_stat(x)}' 274 else: 275 for idx, x in enumerate(self.responses): 276 assert x['status'] == http_status, \ 277 f'response #{idx} status: expected {http_status},'\ 278 f'got {x["status"]}\n{self.dump_stat(x)}' 279 if protocol is not None: 280 if self.with_stats: 281 http_version = None 282 if protocol == 'HTTP/1.1': 283 http_version = '1.1' 284 elif protocol == 'HTTP/2': 285 http_version = '2' 286 elif protocol == 'HTTP/3': 287 http_version = '3' 288 if http_version is not None: 289 for idx, x in enumerate(self.stats): 290 assert x['http_version'] == http_version, \ 291 f'response #{idx} protocol: expected http/{http_version},' \ 292 f'got version {x["http_version"]}\n{self.dump_stat(x)}' 293 else: 294 for idx, x in enumerate(self.responses): 295 assert x['protocol'] == protocol, \ 296 f'response #{idx} protocol: expected {protocol},'\ 297 f'got {x["protocol"]}\n{self.dump_logs()}' 298 if connect_count is not None: 299 assert self.total_connects == connect_count, \ 300 f'expected {connect_count}, but {self.total_connects} '\ 301 f'were made\n{self.dump_logs()}' 302 303 def check_stats(self, count: int, http_status: Optional[int] = None, 304 exitcode: Optional[int] = None): 305 if exitcode is None: 306 self.check_exit_code(0) 307 assert len(self.stats) == count, \ 308 f'stats count: expected {count}, got {len(self.stats)}\n{self.dump_logs()}' 309 if http_status is not None: 310 for idx, x in enumerate(self.stats): 311 assert 'http_code' in x, \ 312 f'status #{idx} reports no http_code\n{self.dump_stat(x)}' 313 assert x['http_code'] == http_status, \ 314 f'status #{idx} http_code: expected {http_status}, '\ 315 f'got {x["http_code"]}\n{self.dump_stat(x)}' 316 if exitcode is not None: 317 for idx, x in enumerate(self.stats): 318 if 'exitcode' in x: 319 assert x['exitcode'] == exitcode, \ 320 f'status #{idx} exitcode: expected {exitcode}, '\ 321 f'got {x["exitcode"]}\n{self.dump_stat(x)}' 322 323 def dump_logs(self): 324 lines = ['>>--stdout ----------------------------------------------\n'] 325 lines.extend(self._stdout) 326 lines.append('>>--stderr ----------------------------------------------\n') 327 lines.extend(self._stderr) 328 lines.append('<<-------------------------------------------------------\n') 329 return ''.join(lines) 330 331 def dump_stat(self, x): 332 lines = [ 333 'json stat from curl:', 334 json.JSONEncoder(indent=2).encode(x), 335 ] 336 if 'xfer_id' in x: 337 xfer_id = x['xfer_id'] 338 lines.append(f'>>--xfer {xfer_id} trace:\n') 339 lines.extend(self.xfer_trace_for(xfer_id)) 340 else: 341 lines.append('>>--full trace-------------------------------------------\n') 342 lines.extend(self._stderr) 343 lines.append('<<-------------------------------------------------------\n') 344 return ''.join(lines) 345 346 def xfer_trace_for(self, xfer_id) -> List[str]: 347 pat = re.compile(f'^[^[]* \\[{xfer_id}-.*$') 348 return [line for line in self._stderr if pat.match(line)] 349 350 351class CurlClient: 352 353 ALPN_ARG = { 354 'http/0.9': '--http0.9', 355 'http/1.0': '--http1.0', 356 'http/1.1': '--http1.1', 357 'h2': '--http2', 358 'h2c': '--http2', 359 'h3': '--http3-only', 360 } 361 362 def __init__(self, env: Env, run_dir: Optional[str] = None, 363 timeout: Optional[float] = None, silent: bool = False): 364 self.env = env 365 self._timeout = timeout if timeout else env.test_timeout 366 self._curl = os.environ['CURL'] if 'CURL' in os.environ else env.curl 367 self._run_dir = run_dir if run_dir else os.path.join(env.gen_dir, 'curl') 368 self._stdoutfile = f'{self._run_dir}/curl.stdout' 369 self._stderrfile = f'{self._run_dir}/curl.stderr' 370 self._headerfile = f'{self._run_dir}/curl.headers' 371 self._log_path = f'{self._run_dir}/curl.log' 372 self._silent = silent 373 self._rmrf(self._run_dir) 374 self._mkpath(self._run_dir) 375 376 @property 377 def run_dir(self) -> str: 378 return self._run_dir 379 380 def download_file(self, i: int) -> str: 381 return os.path.join(self.run_dir, f'download_{i}.data') 382 383 def _rmf(self, path): 384 if os.path.exists(path): 385 return os.remove(path) 386 387 def _rmrf(self, path): 388 if os.path.exists(path): 389 return shutil.rmtree(path) 390 391 def _mkpath(self, path): 392 if not os.path.exists(path): 393 return os.makedirs(path) 394 395 def get_proxy_args(self, proto: str = 'http/1.1', 396 proxys: bool = True, tunnel: bool = False, 397 use_ip: bool = False): 398 proxy_name = '127.0.0.1' if use_ip else self.env.proxy_domain 399 if proxys: 400 pport = self.env.pts_port(proto) if tunnel else self.env.proxys_port 401 xargs = [ 402 '--proxy', f'https://{proxy_name}:{pport}/', 403 '--resolve', f'{proxy_name}:{pport}:127.0.0.1', 404 '--proxy-cacert', self.env.ca.cert_file, 405 ] 406 if proto == 'h2': 407 xargs.append('--proxy-http2') 408 else: 409 xargs = [ 410 '--proxy', f'http://{proxy_name}:{self.env.proxy_port}/', 411 '--resolve', f'{proxy_name}:{self.env.proxy_port}:127.0.0.1', 412 ] 413 if tunnel: 414 xargs.append('--proxytunnel') 415 return xargs 416 417 def http_get(self, url: str, extra_args: Optional[List[str]] = None, 418 alpn_proto: Optional[str] = None, 419 def_tracing: bool = True, 420 with_stats: bool = False, 421 with_profile: bool = False): 422 return self._raw(url, options=extra_args, 423 with_stats=with_stats, 424 alpn_proto=alpn_proto, 425 def_tracing=def_tracing, 426 with_profile=with_profile) 427 428 def http_download(self, urls: List[str], 429 alpn_proto: Optional[str] = None, 430 with_stats: bool = True, 431 with_headers: bool = False, 432 with_profile: bool = False, 433 no_save: bool = False, 434 extra_args: List[str] = None): 435 if extra_args is None: 436 extra_args = [] 437 if no_save: 438 extra_args.extend([ 439 '-o', '/dev/null', 440 ]) 441 else: 442 extra_args.extend([ 443 '-o', 'download_#1.data', 444 ]) 445 # remove any existing ones 446 for i in range(100): 447 self._rmf(self.download_file(i)) 448 if with_stats: 449 extra_args.extend([ 450 '-w', '%{json}\\n' 451 ]) 452 return self._raw(urls, alpn_proto=alpn_proto, options=extra_args, 453 with_stats=with_stats, 454 with_headers=with_headers, 455 with_profile=with_profile) 456 457 def http_upload(self, urls: List[str], data: str, 458 alpn_proto: Optional[str] = None, 459 with_stats: bool = True, 460 with_headers: bool = False, 461 with_profile: bool = False, 462 extra_args: Optional[List[str]] = None): 463 if extra_args is None: 464 extra_args = [] 465 extra_args.extend([ 466 '--data-binary', data, '-o', 'download_#1.data', 467 ]) 468 if with_stats: 469 extra_args.extend([ 470 '-w', '%{json}\\n' 471 ]) 472 return self._raw(urls, alpn_proto=alpn_proto, options=extra_args, 473 with_stats=with_stats, 474 with_headers=with_headers, 475 with_profile=with_profile) 476 477 def http_delete(self, urls: List[str], 478 alpn_proto: Optional[str] = None, 479 with_stats: bool = True, 480 with_profile: bool = False, 481 extra_args: Optional[List[str]] = None): 482 if extra_args is None: 483 extra_args = [] 484 extra_args.extend([ 485 '-X', 'DELETE', '-o', '/dev/null', 486 ]) 487 if with_stats: 488 extra_args.extend([ 489 '-w', '%{json}\\n' 490 ]) 491 return self._raw(urls, alpn_proto=alpn_proto, options=extra_args, 492 with_stats=with_stats, 493 with_headers=False, 494 with_profile=with_profile) 495 496 def http_put(self, urls: List[str], data=None, fdata=None, 497 alpn_proto: Optional[str] = None, 498 with_stats: bool = True, 499 with_headers: bool = False, 500 with_profile: bool = False, 501 extra_args: Optional[List[str]] = None): 502 if extra_args is None: 503 extra_args = [] 504 if fdata is not None: 505 extra_args.extend(['-T', fdata]) 506 elif data is not None: 507 extra_args.extend(['-T', '-']) 508 extra_args.extend([ 509 '-o', 'download_#1.data', 510 ]) 511 if with_stats: 512 extra_args.extend([ 513 '-w', '%{json}\\n' 514 ]) 515 return self._raw(urls, intext=data, 516 alpn_proto=alpn_proto, options=extra_args, 517 with_stats=with_stats, 518 with_headers=with_headers, 519 with_profile=with_profile) 520 521 def http_form(self, urls: List[str], form: Dict[str, str], 522 alpn_proto: Optional[str] = None, 523 with_stats: bool = True, 524 with_headers: bool = False, 525 extra_args: Optional[List[str]] = None): 526 if extra_args is None: 527 extra_args = [] 528 for key, val in form.items(): 529 extra_args.extend(['-F', f'{key}={val}']) 530 extra_args.extend([ 531 '-o', 'download_#1.data', 532 ]) 533 if with_stats: 534 extra_args.extend([ 535 '-w', '%{json}\\n' 536 ]) 537 return self._raw(urls, alpn_proto=alpn_proto, options=extra_args, 538 with_stats=with_stats, 539 with_headers=with_headers) 540 541 def ftp_get(self, urls: List[str], 542 with_stats: bool = True, 543 with_profile: bool = False, 544 no_save: bool = False, 545 extra_args: List[str] = None): 546 if extra_args is None: 547 extra_args = [] 548 if no_save: 549 extra_args.extend([ 550 '-o', '/dev/null', 551 ]) 552 else: 553 extra_args.extend([ 554 '-o', 'download_#1.data', 555 ]) 556 # remove any existing ones 557 for i in range(100): 558 self._rmf(self.download_file(i)) 559 if with_stats: 560 extra_args.extend([ 561 '-w', '%{json}\\n' 562 ]) 563 return self._raw(urls, options=extra_args, 564 with_stats=with_stats, 565 with_headers=False, 566 with_profile=with_profile) 567 568 def ftp_ssl_get(self, urls: List[str], 569 with_stats: bool = True, 570 with_profile: bool = False, 571 no_save: bool = False, 572 extra_args: List[str] = None): 573 if extra_args is None: 574 extra_args = [] 575 extra_args.extend([ 576 '--ssl-reqd', 577 ]) 578 return self.ftp_get(urls=urls, with_stats=with_stats, 579 with_profile=with_profile, no_save=no_save, 580 extra_args=extra_args) 581 582 def ftp_upload(self, urls: List[str], fupload, 583 with_stats: bool = True, 584 with_profile: bool = False, 585 extra_args: List[str] = None): 586 if extra_args is None: 587 extra_args = [] 588 extra_args.extend([ 589 '--upload-file', fupload 590 ]) 591 if with_stats: 592 extra_args.extend([ 593 '-w', '%{json}\\n' 594 ]) 595 return self._raw(urls, options=extra_args, 596 with_stats=with_stats, 597 with_headers=False, 598 with_profile=with_profile) 599 600 def ftp_ssl_upload(self, urls: List[str], fupload, 601 with_stats: bool = True, 602 with_profile: bool = False, 603 extra_args: List[str] = None): 604 if extra_args is None: 605 extra_args = [] 606 extra_args.extend([ 607 '--ssl-reqd', 608 ]) 609 return self.ftp_upload(urls=urls, fupload=fupload, 610 with_stats=with_stats, with_profile=with_profile, 611 extra_args=extra_args) 612 613 def response_file(self, idx: int): 614 return os.path.join(self._run_dir, f'download_{idx}.data') 615 616 def run_direct(self, args, with_stats: bool = False, with_profile: bool = False): 617 my_args = [self._curl] 618 if with_stats: 619 my_args.extend([ 620 '-w', '%{json}\\n' 621 ]) 622 my_args.extend([ 623 '-o', 'download.data', 624 ]) 625 my_args.extend(args) 626 return self._run(args=my_args, with_stats=with_stats, with_profile=with_profile) 627 628 def _run(self, args, intext='', with_stats: bool = False, with_profile: bool = True): 629 self._rmf(self._stdoutfile) 630 self._rmf(self._stderrfile) 631 self._rmf(self._headerfile) 632 started_at = datetime.now() 633 exception = None 634 profile = None 635 started_at = datetime.now() 636 try: 637 with open(self._stdoutfile, 'w') as cout: 638 with open(self._stderrfile, 'w') as cerr: 639 if with_profile: 640 end_at = started_at + timedelta(seconds=self._timeout) \ 641 if self._timeout else None 642 log.info(f'starting: {args}') 643 p = subprocess.Popen(args, stderr=cerr, stdout=cout, 644 cwd=self._run_dir, shell=False) 645 profile = RunProfile(p.pid, started_at, self._run_dir) 646 if intext is not None and False: 647 p.communicate(input=intext.encode(), timeout=1) 648 ptimeout = 0.0 649 while True: 650 try: 651 p.wait(timeout=ptimeout) 652 break 653 except subprocess.TimeoutExpired: 654 if end_at and datetime.now() >= end_at: 655 p.kill() 656 raise subprocess.TimeoutExpired(cmd=args, timeout=self._timeout) 657 profile.sample() 658 ptimeout = 0.01 659 exitcode = p.returncode 660 profile.finish() 661 log.info(f'done: exit={exitcode}, profile={profile}') 662 else: 663 p = subprocess.run(args, stderr=cerr, stdout=cout, 664 cwd=self._run_dir, shell=False, 665 input=intext.encode() if intext else None, 666 timeout=self._timeout) 667 exitcode = p.returncode 668 except subprocess.TimeoutExpired: 669 now = datetime.now() 670 duration = now - started_at 671 log.warning(f'Timeout at {now} after {duration.total_seconds()}s ' 672 f'(configured {self._timeout}s): {args}') 673 exitcode = -1 674 exception = 'TimeoutExpired' 675 coutput = open(self._stdoutfile).readlines() 676 cerrput = open(self._stderrfile).readlines() 677 return ExecResult(args=args, exit_code=exitcode, exception=exception, 678 stdout=coutput, stderr=cerrput, 679 duration=datetime.now() - started_at, 680 with_stats=with_stats, 681 profile=profile) 682 683 def _raw(self, urls, intext='', timeout=None, options=None, insecure=False, 684 alpn_proto: Optional[str] = None, 685 force_resolve=True, 686 with_stats=False, 687 with_headers=True, 688 def_tracing=True, 689 with_profile=False): 690 args = self._complete_args( 691 urls=urls, timeout=timeout, options=options, insecure=insecure, 692 alpn_proto=alpn_proto, force_resolve=force_resolve, 693 with_headers=with_headers, def_tracing=def_tracing) 694 r = self._run(args, intext=intext, with_stats=with_stats, 695 with_profile=with_profile) 696 if r.exit_code == 0 and with_headers: 697 self._parse_headerfile(self._headerfile, r=r) 698 if r.json: 699 r.response["json"] = r.json 700 return r 701 702 def _complete_args(self, urls, timeout=None, options=None, 703 insecure=False, force_resolve=True, 704 alpn_proto: Optional[str] = None, 705 with_headers: bool = True, 706 def_tracing: bool = True): 707 if not isinstance(urls, list): 708 urls = [urls] 709 710 args = [self._curl, "-s", "--path-as-is"] 711 if with_headers: 712 args.extend(["-D", self._headerfile]) 713 if def_tracing is not False and not self._silent: 714 args.extend(['-v', '--trace-ids', '--trace-time']) 715 if self.env.verbose > 1: 716 args.extend(['--trace-config', 'http/2,http/3,h2-proxy,h1-proxy']) 717 pass 718 719 active_options = options 720 if options is not None and '--next' in options: 721 active_options = options[options.index('--next') + 1:] 722 723 for url in urls: 724 u = urlparse(urls[0]) 725 if options: 726 args.extend(options) 727 if alpn_proto is not None: 728 if alpn_proto not in self.ALPN_ARG: 729 raise Exception(f'unknown ALPN protocol: "{alpn_proto}"') 730 args.append(self.ALPN_ARG[alpn_proto]) 731 732 if u.scheme == 'http': 733 pass 734 elif insecure: 735 args.append('--insecure') 736 elif active_options and "--cacert" in active_options: 737 pass 738 elif u.hostname: 739 args.extend(["--cacert", self.env.ca.cert_file]) 740 741 if force_resolve and u.hostname and u.hostname != 'localhost' \ 742 and not re.match(r'^(\d+|\[|:).*', u.hostname): 743 port = u.port if u.port else 443 744 args.extend(["--resolve", f"{u.hostname}:{port}:127.0.0.1"]) 745 if timeout is not None and int(timeout) > 0: 746 args.extend(["--connect-timeout", str(int(timeout))]) 747 args.append(url) 748 return args 749 750 def _parse_headerfile(self, headerfile: str, r: ExecResult = None) -> ExecResult: 751 lines = open(headerfile).readlines() 752 if r is None: 753 r = ExecResult(args=[], exit_code=0, stdout=[], stderr=[]) 754 755 response = None 756 757 def fin_response(resp): 758 if resp: 759 r.add_response(resp) 760 761 expected = ['status'] 762 for line in lines: 763 line = line.strip() 764 if re.match(r'^$', line): 765 if 'trailer' in expected: 766 # end of trailers 767 fin_response(response) 768 response = None 769 expected = ['status'] 770 elif 'header' in expected: 771 # end of header, another status or trailers might follow 772 expected = ['status', 'trailer'] 773 else: 774 assert False, f"unexpected line: '{line}'" 775 continue 776 if 'status' in expected: 777 # log.debug("reading 1st response line: %s", line) 778 m = re.match(r'^(\S+) (\d+)( .*)?$', line) 779 if m: 780 fin_response(response) 781 response = { 782 "protocol": m.group(1), 783 "status": int(m.group(2)), 784 "description": m.group(3), 785 "header": {}, 786 "trailer": {}, 787 "body": r.outraw 788 } 789 expected = ['header'] 790 continue 791 if 'trailer' in expected: 792 m = re.match(r'^([^:]+):\s*(.*)$', line) 793 if m: 794 response['trailer'][m.group(1).lower()] = m.group(2) 795 continue 796 if 'header' in expected: 797 m = re.match(r'^([^:]+):\s*(.*)$', line) 798 if m: 799 response['header'][m.group(1).lower()] = m.group(2) 800 continue 801 assert False, f"unexpected line: '{line}, expected: {expected}'" 802 803 fin_response(response) 804 return r 805
