1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import logging 28import os 29import re 30import shutil 31import socket 32import subprocess 33import tempfile 34from configparser import ConfigParser, ExtendedInterpolation 35from typing import Optional 36 37from .certs import CertificateSpec, Credentials, TestCA 38from .ports import alloc_ports 39 40 41log = logging.getLogger(__name__) 42 43 44def init_config_from(conf_path): 45 if os.path.isfile(conf_path): 46 config = ConfigParser(interpolation=ExtendedInterpolation()) 47 config.read(conf_path) 48 return config 49 return None 50 51 52TESTS_HTTPD_PATH = os.path.dirname(os.path.dirname(__file__)) 53TOP_PATH = os.path.join(os.getcwd(), os.path.pardir) 54DEF_CONFIG = init_config_from(os.path.join(TOP_PATH, 'tests', 'http', 'config.ini')) 55CURL = os.path.join(TOP_PATH, 'src', 'curl') 56 57 58class EnvConfig: 59 60 def __init__(self): 61 self.tests_dir = TESTS_HTTPD_PATH 62 self.gen_dir = os.path.join(self.tests_dir, 'gen') 63 self.project_dir = os.path.dirname(os.path.dirname(self.tests_dir)) 64 self.build_dir = TOP_PATH 65 self.config = DEF_CONFIG 66 # check cur and its features 67 self.curl = CURL 68 if 'CURL' in os.environ: 69 self.curl = os.environ['CURL'] 70 self.curl_props = { 71 'version_string': '', 72 'version': '', 73 'os': '', 74 'fullname': '', 75 'features_string': '', 76 'features': set(), 77 'protocols_string': '', 78 'protocols': set(), 79 'libs': set(), 80 'lib_versions': set(), 81 } 82 self.curl_is_debug = False 83 self.curl_protos = [] 84 p = subprocess.run(args=[self.curl, '-V'], 85 capture_output=True, text=True) 86 if p.returncode != 0: 87 raise RuntimeError(f'{self.curl} -V failed with exit code: {p.returncode}') 88 if p.stderr.startswith('WARNING:'): 89 self.curl_is_debug = True 90 for line in p.stdout.splitlines(keepends=False): 91 if line.startswith('curl '): 92 self.curl_props['version_string'] = line 93 m = re.match(r'^curl (?P<version>\S+) (?P<os>\S+) (?P<libs>.*)$', line) 94 if m: 95 self.curl_props['fullname'] = m.group(0) 96 self.curl_props['version'] = m.group('version') 97 self.curl_props['os'] = m.group('os') 98 self.curl_props['lib_versions'] = { 99 lib.lower() for lib in m.group('libs').split(' ') 100 } 101 self.curl_props['libs'] = { 102 re.sub(r'/[a-z0-9.-]*', '', lib) for lib in self.curl_props['lib_versions'] 103 } 104 if line.startswith('Features: '): 105 self.curl_props['features_string'] = line[10:] 106 self.curl_props['features'] = { 107 feat.lower() for feat in line[10:].split(' ') 108 } 109 if line.startswith('Protocols: '): 110 self.curl_props['protocols_string'] = line[11:] 111 self.curl_props['protocols'] = { 112 prot.lower() for prot in line[11:].split(' ') 113 } 114 115 self.ports = alloc_ports(port_specs={ 116 'ftp': socket.SOCK_STREAM, 117 'ftps': socket.SOCK_STREAM, 118 'http': socket.SOCK_STREAM, 119 'https': socket.SOCK_STREAM, 120 'nghttpx_https': socket.SOCK_STREAM, 121 'proxy': socket.SOCK_STREAM, 122 'proxys': socket.SOCK_STREAM, 123 'h2proxys': socket.SOCK_STREAM, 124 'caddy': socket.SOCK_STREAM, 125 'caddys': socket.SOCK_STREAM, 126 'ws': socket.SOCK_STREAM, 127 }) 128 self.httpd = self.config['httpd']['httpd'] 129 self.apachectl = self.config['httpd']['apachectl'] 130 self.apxs = self.config['httpd']['apxs'] 131 if len(self.apxs) == 0: 132 self.apxs = None 133 self._httpd_version = None 134 135 self.examples_pem = { 136 'key': 'xxx', 137 'cert': 'xxx', 138 } 139 self.htdocs_dir = os.path.join(self.gen_dir, 'htdocs') 140 self.tld = 'http.curl.se' 141 self.domain1 = f"one.{self.tld}" 142 self.domain1brotli = f"brotli.one.{self.tld}" 143 self.domain2 = f"two.{self.tld}" 144 self.ftp_domain = f"ftp.{self.tld}" 145 self.proxy_domain = f"proxy.{self.tld}" 146 self.cert_specs = [ 147 CertificateSpec(domains=[self.domain1, self.domain1brotli, 'localhost', '127.0.0.1'], key_type='rsa2048'), 148 CertificateSpec(domains=[self.domain2], key_type='rsa2048'), 149 CertificateSpec(domains=[self.ftp_domain], key_type='rsa2048'), 150 CertificateSpec(domains=[self.proxy_domain, '127.0.0.1'], key_type='rsa2048'), 151 CertificateSpec(name="clientsX", sub_specs=[ 152 CertificateSpec(name="user1", client=True), 153 ]), 154 ] 155 156 self.nghttpx = self.config['nghttpx']['nghttpx'] 157 if len(self.nghttpx.strip()) == 0: 158 self.nghttpx = None 159 self._nghttpx_version = None 160 self.nghttpx_with_h3 = False 161 if self.nghttpx is not None: 162 p = subprocess.run(args=[self.nghttpx, '-v'], 163 capture_output=True, text=True) 164 if p.returncode != 0: 165 # not a working nghttpx 166 self.nghttpx = None 167 else: 168 self._nghttpx_version = re.sub(r'^nghttpx\s*', '', p.stdout.strip()) 169 self.nghttpx_with_h3 = re.match(r'.* nghttp3/.*', p.stdout.strip()) is not None 170 log.debug(f'nghttpx -v: {p.stdout}') 171 172 self.caddy = self.config['caddy']['caddy'] 173 self._caddy_version = None 174 if len(self.caddy.strip()) == 0: 175 self.caddy = None 176 if self.caddy is not None: 177 try: 178 p = subprocess.run(args=[self.caddy, 'version'], 179 capture_output=True, text=True) 180 if p.returncode != 0: 181 # not a working caddy 182 self.caddy = None 183 m = re.match(r'v?(\d+\.\d+\.\d+).*', p.stdout) 184 if m: 185 self._caddy_version = m.group(1) 186 else: 187 raise RuntimeError(f'Unable to determine cadd version from: {p.stdout}') 188 # TODO: specify specific exceptions here 189 except: # noqa: E722 190 self.caddy = None 191 192 self.vsftpd = self.config['vsftpd']['vsftpd'] 193 self._vsftpd_version = None 194 if self.vsftpd is not None: 195 try: 196 with tempfile.TemporaryFile('w+') as tmp: 197 p = subprocess.run(args=[self.vsftpd, '-v'], 198 capture_output=True, text=True, stdin=tmp) 199 if p.returncode != 0: 200 # not a working vsftpd 201 self.vsftpd = None 202 if p.stderr: 203 ver_text = p.stderr 204 else: 205 # Oddly, some versions of vsftpd write to stdin (!) 206 # instead of stderr, which is odd but works. If there 207 # is nothing on stderr, read the file on stdin and use 208 # any data there instead. 209 tmp.seek(0) 210 ver_text = tmp.read() 211 m = re.match(r'vsftpd: version (\d+\.\d+\.\d+)', ver_text) 212 if m: 213 self._vsftpd_version = m.group(1) 214 elif len(p.stderr) == 0: 215 # vsftp does not use stdout or stderr for printing its version... -.- 216 self._vsftpd_version = 'unknown' 217 else: 218 raise Exception(f'Unable to determine VsFTPD version from: {p.stderr}') 219 except Exception: 220 self.vsftpd = None 221 222 self._tcpdump = shutil.which('tcpdump') 223 224 @property 225 def httpd_version(self): 226 if self._httpd_version is None and self.apxs is not None: 227 try: 228 p = subprocess.run(args=[self.apxs, '-q', 'HTTPD_VERSION'], 229 capture_output=True, text=True) 230 if p.returncode != 0: 231 log.error(f'{self.apxs} failed to query HTTPD_VERSION: {p}') 232 else: 233 self._httpd_version = p.stdout.strip() 234 except Exception: 235 log.exception(f'{self.apxs} failed to run') 236 return self._httpd_version 237 238 def versiontuple(self, v): 239 v = re.sub(r'(\d+\.\d+(\.\d+)?)(-\S+)?', r'\1', v) 240 return tuple(map(int, v.split('.'))) 241 242 def httpd_is_at_least(self, minv): 243 if self.httpd_version is None: 244 return False 245 hv = self.versiontuple(self.httpd_version) 246 return hv >= self.versiontuple(minv) 247 248 def caddy_is_at_least(self, minv): 249 if self.caddy_version is None: 250 return False 251 hv = self.versiontuple(self.caddy_version) 252 return hv >= self.versiontuple(minv) 253 254 def is_complete(self) -> bool: 255 return os.path.isfile(self.httpd) and \ 256 os.path.isfile(self.apachectl) and \ 257 self.apxs is not None and \ 258 os.path.isfile(self.apxs) 259 260 def get_incomplete_reason(self) -> Optional[str]: 261 if self.httpd is None or len(self.httpd.strip()) == 0: 262 return 'httpd not configured, see `--with-test-httpd=<path>`' 263 if not os.path.isfile(self.httpd): 264 return f'httpd ({self.httpd}) not found' 265 if not os.path.isfile(self.apachectl): 266 return f'apachectl ({self.apachectl}) not found' 267 if self.apxs is None: 268 return "command apxs not found (commonly provided in apache2-dev)" 269 if not os.path.isfile(self.apxs): 270 return f"apxs ({self.apxs}) not found" 271 return None 272 273 @property 274 def nghttpx_version(self): 275 return self._nghttpx_version 276 277 @property 278 def caddy_version(self): 279 return self._caddy_version 280 281 @property 282 def vsftpd_version(self): 283 return self._vsftpd_version 284 285 @property 286 def tcpdmp(self) -> Optional[str]: 287 return self._tcpdump 288 289 290class Env: 291 292 CONFIG = EnvConfig() 293 294 @staticmethod 295 def setup_incomplete() -> bool: 296 return not Env.CONFIG.is_complete() 297 298 @staticmethod 299 def incomplete_reason() -> Optional[str]: 300 return Env.CONFIG.get_incomplete_reason() 301 302 @staticmethod 303 def have_nghttpx() -> bool: 304 return Env.CONFIG.nghttpx is not None 305 306 @staticmethod 307 def have_h3_server() -> bool: 308 return Env.CONFIG.nghttpx_with_h3 309 310 @staticmethod 311 def have_ssl_curl() -> bool: 312 return Env.curl_has_feature('ssl') or Env.curl_has_feature('multissl') 313 314 @staticmethod 315 def have_h2_curl() -> bool: 316 return 'http2' in Env.CONFIG.curl_props['features'] 317 318 @staticmethod 319 def have_h3_curl() -> bool: 320 return 'http3' in Env.CONFIG.curl_props['features'] 321 322 @staticmethod 323 def curl_uses_lib(libname: str) -> bool: 324 return libname.lower() in Env.CONFIG.curl_props['libs'] 325 326 @staticmethod 327 def curl_uses_ossl_quic() -> bool: 328 if Env.have_h3_curl(): 329 return not Env.curl_uses_lib('ngtcp2') and Env.curl_uses_lib('nghttp3') 330 return False 331 332 @staticmethod 333 def curl_version_string() -> str: 334 return Env.CONFIG.curl_props['version_string'] 335 336 @staticmethod 337 def curl_features_string() -> str: 338 return Env.CONFIG.curl_props['features_string'] 339 340 @staticmethod 341 def curl_has_feature(feature: str) -> bool: 342 return feature.lower() in Env.CONFIG.curl_props['features'] 343 344 @staticmethod 345 def curl_protocols_string() -> str: 346 return Env.CONFIG.curl_props['protocols_string'] 347 348 @staticmethod 349 def curl_has_protocol(protocol: str) -> bool: 350 return protocol.lower() in Env.CONFIG.curl_props['protocols'] 351 352 @staticmethod 353 def curl_lib_version(libname: str) -> str: 354 prefix = f'{libname.lower()}/' 355 for lversion in Env.CONFIG.curl_props['lib_versions']: 356 if lversion.startswith(prefix): 357 return lversion[len(prefix):] 358 return 'unknown' 359 360 @staticmethod 361 def curl_lib_version_at_least(libname: str, min_version) -> bool: 362 lversion = Env.curl_lib_version(libname) 363 if lversion != 'unknown': 364 return Env.CONFIG.versiontuple(min_version) <= \ 365 Env.CONFIG.versiontuple(lversion) 366 return False 367 368 @staticmethod 369 def curl_os() -> str: 370 return Env.CONFIG.curl_props['os'] 371 372 @staticmethod 373 def curl_fullname() -> str: 374 return Env.CONFIG.curl_props['fullname'] 375 376 @staticmethod 377 def curl_version() -> str: 378 return Env.CONFIG.curl_props['version'] 379 380 @staticmethod 381 def curl_is_debug() -> bool: 382 return Env.CONFIG.curl_is_debug 383 384 @staticmethod 385 def have_h3() -> bool: 386 return Env.have_h3_curl() and Env.have_h3_server() 387 388 @staticmethod 389 def httpd_version() -> str: 390 return Env.CONFIG.httpd_version 391 392 @staticmethod 393 def nghttpx_version() -> str: 394 return Env.CONFIG.nghttpx_version 395 396 @staticmethod 397 def caddy_version() -> str: 398 return Env.CONFIG.caddy_version 399 400 @staticmethod 401 def caddy_is_at_least(minv) -> bool: 402 return Env.CONFIG.caddy_is_at_least(minv) 403 404 @staticmethod 405 def httpd_is_at_least(minv) -> bool: 406 return Env.CONFIG.httpd_is_at_least(minv) 407 408 @staticmethod 409 def has_caddy() -> bool: 410 return Env.CONFIG.caddy is not None 411 412 @staticmethod 413 def has_vsftpd() -> bool: 414 return Env.CONFIG.vsftpd is not None 415 416 @staticmethod 417 def vsftpd_version() -> str: 418 return Env.CONFIG.vsftpd_version 419 420 @staticmethod 421 def tcpdump() -> Optional[str]: 422 return Env.CONFIG.tcpdmp 423 424 def __init__(self, pytestconfig=None): 425 self._verbose = pytestconfig.option.verbose \ 426 if pytestconfig is not None else 0 427 self._ca = None 428 self._test_timeout = 300.0 if self._verbose > 1 else 60.0 # seconds 429 430 def issue_certs(self): 431 if self._ca is None: 432 ca_dir = os.path.join(self.CONFIG.gen_dir, 'ca') 433 self._ca = TestCA.create_root(name=self.CONFIG.tld, 434 store_dir=ca_dir, 435 key_type="rsa2048") 436 self._ca.issue_certs(self.CONFIG.cert_specs) 437 438 def setup(self): 439 os.makedirs(self.gen_dir, exist_ok=True) 440 os.makedirs(self.htdocs_dir, exist_ok=True) 441 self.issue_certs() 442 443 def get_credentials(self, domain) -> Optional[Credentials]: 444 creds = self.ca.get_credentials_for_name(domain) 445 if len(creds) > 0: 446 return creds[0] 447 return None 448 449 @property 450 def verbose(self) -> int: 451 return self._verbose 452 453 @property 454 def test_timeout(self) -> Optional[float]: 455 return self._test_timeout 456 457 @test_timeout.setter 458 def test_timeout(self, val: Optional[float]): 459 self._test_timeout = val 460 461 @property 462 def gen_dir(self) -> str: 463 return self.CONFIG.gen_dir 464 465 @property 466 def project_dir(self) -> str: 467 return self.CONFIG.project_dir 468 469 @property 470 def build_dir(self) -> str: 471 return self.CONFIG.build_dir 472 473 @property 474 def ca(self): 475 return self._ca 476 477 @property 478 def htdocs_dir(self) -> str: 479 return self.CONFIG.htdocs_dir 480 481 @property 482 def tld(self) -> str: 483 return self.CONFIG.tld 484 485 @property 486 def domain1(self) -> str: 487 return self.CONFIG.domain1 488 489 @property 490 def domain1brotli(self) -> str: 491 return self.CONFIG.domain1brotli 492 493 @property 494 def domain2(self) -> str: 495 return self.CONFIG.domain2 496 497 @property 498 def ftp_domain(self) -> str: 499 return self.CONFIG.ftp_domain 500 501 @property 502 def proxy_domain(self) -> str: 503 return self.CONFIG.proxy_domain 504 505 @property 506 def http_port(self) -> int: 507 return self.CONFIG.ports['http'] 508 509 @property 510 def https_port(self) -> int: 511 return self.CONFIG.ports['https'] 512 513 @property 514 def nghttpx_https_port(self) -> int: 515 return self.CONFIG.ports['nghttpx_https'] 516 517 @property 518 def h3_port(self) -> int: 519 return self.https_port 520 521 @property 522 def proxy_port(self) -> int: 523 return self.CONFIG.ports['proxy'] 524 525 @property 526 def proxys_port(self) -> int: 527 return self.CONFIG.ports['proxys'] 528 529 @property 530 def ftp_port(self) -> int: 531 return self.CONFIG.ports['ftp'] 532 533 @property 534 def ftps_port(self) -> int: 535 return self.CONFIG.ports['ftps'] 536 537 @property 538 def h2proxys_port(self) -> int: 539 return self.CONFIG.ports['h2proxys'] 540 541 def pts_port(self, proto: str = 'http/1.1') -> int: 542 # proxy tunnel port 543 return self.CONFIG.ports['h2proxys' if proto == 'h2' else 'proxys'] 544 545 @property 546 def caddy(self) -> str: 547 return self.CONFIG.caddy 548 549 @property 550 def caddy_https_port(self) -> int: 551 return self.CONFIG.ports['caddys'] 552 553 @property 554 def caddy_http_port(self) -> int: 555 return self.CONFIG.ports['caddy'] 556 557 @property 558 def vsftpd(self) -> str: 559 return self.CONFIG.vsftpd 560 561 @property 562 def ws_port(self) -> int: 563 return self.CONFIG.ports['ws'] 564 565 @property 566 def curl(self) -> str: 567 return self.CONFIG.curl 568 569 @property 570 def httpd(self) -> str: 571 return self.CONFIG.httpd 572 573 @property 574 def apachectl(self) -> str: 575 return self.CONFIG.apachectl 576 577 @property 578 def apxs(self) -> str: 579 return self.CONFIG.apxs 580 581 @property 582 def nghttpx(self) -> Optional[str]: 583 return self.CONFIG.nghttpx 584 585 @property 586 def slow_network(self) -> bool: 587 return "CURL_DBG_SOCK_WBLOCK" in os.environ or \ 588 "CURL_DBG_SOCK_WPARTIAL" in os.environ 589 590 @property 591 def ci_run(self) -> bool: 592 return "CURL_CI" in os.environ 593 594 def port_for(self, alpn_proto: Optional[str] = None): 595 if alpn_proto is None or \ 596 alpn_proto in ['h2', 'http/1.1', 'http/1.0', 'http/0.9']: 597 return self.https_port 598 if alpn_proto in ['h3']: 599 return self.h3_port 600 return self.http_port 601 602 def authority_for(self, domain: str, alpn_proto: Optional[str] = None): 603 return f'{domain}:{self.port_for(alpn_proto=alpn_proto)}' 604 605 def make_data_file(self, indir: str, fname: str, fsize: int, 606 line_length: int = 1024) -> str: 607 if line_length < 11: 608 raise RuntimeError('line_length less than 11 not supported') 609 fpath = os.path.join(indir, fname) 610 s10 = "0123456789" 611 s = round((line_length / 10) + 1) * s10 612 s = s[0:line_length-11] 613 with open(fpath, 'w') as fd: 614 for i in range(int(fsize / line_length)): 615 fd.write(f"{i:09d}-{s}\n") 616 remain = int(fsize % line_length) 617 if remain != 0: 618 i = int(fsize / line_length) + 1 619 fd.write(f"{i:09d}-{s}"[0:remain-1] + "\n") 620 return fpath 621
