1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import logging 28import os 29import re 30import socket 31import subprocess 32import sys 33from configparser import ConfigParser, ExtendedInterpolation 34from datetime import timedelta 35from typing import Optional 36 37import pytest 38 39from .certs import CertificateSpec, TestCA, Credentials 40from .ports import alloc_ports 41 42 43log = logging.getLogger(__name__) 44 45 46def init_config_from(conf_path): 47 if os.path.isfile(conf_path): 48 config = ConfigParser(interpolation=ExtendedInterpolation()) 49 config.read(conf_path) 50 return config 51 return None 52 53 54TESTS_HTTPD_PATH = os.path.dirname(os.path.dirname(__file__)) 55DEF_CONFIG = init_config_from(os.path.join(TESTS_HTTPD_PATH, 'config.ini')) 56 57TOP_PATH = os.path.dirname(os.path.dirname(TESTS_HTTPD_PATH)) 58CURL = os.path.join(TOP_PATH, 'src/curl') 59 60 61class EnvConfig: 62 63 def __init__(self): 64 self.tests_dir = TESTS_HTTPD_PATH 65 self.gen_dir = os.path.join(self.tests_dir, 'gen') 66 self.project_dir = os.path.dirname(os.path.dirname(self.tests_dir)) 67 self.config = DEF_CONFIG 68 # check cur and its features 69 self.curl = CURL 70 if 'CURL' in os.environ: 71 self.curl = os.environ['CURL'] 72 self.curl_props = { 73 'version': None, 74 'os': None, 75 'fullname': None, 76 'features': [], 77 'protocols': [], 78 'libs': [], 79 'lib_versions': [], 80 } 81 self.curl_is_debug = False 82 self.curl_protos = [] 83 p = subprocess.run(args=[self.curl, '-V'], 84 capture_output=True, text=True) 85 if p.returncode != 0: 86 assert False, f'{self.curl} -V failed with exit code: {p.returncode}' 87 if p.stderr.startswith('WARNING:'): 88 self.curl_is_debug = True 89 for l in p.stdout.splitlines(keepends=False): 90 if l.startswith('curl '): 91 m = re.match(r'^curl (?P<version>\S+) (?P<os>\S+) (?P<libs>.*)$', l) 92 if m: 93 self.curl_props['fullname'] = m.group(0) 94 self.curl_props['version'] = m.group('version') 95 self.curl_props['os'] = m.group('os') 96 self.curl_props['lib_versions'] = [ 97 lib.lower() for lib in m.group('libs').split(' ') 98 ] 99 self.curl_props['libs'] = [ 100 re.sub(r'/.*', '', lib) for lib in self.curl_props['lib_versions'] 101 ] 102 if l.startswith('Features: '): 103 self.curl_props['features'] = [ 104 feat.lower() for feat in l[10:].split(' ') 105 ] 106 if l.startswith('Protocols: '): 107 self.curl_props['protocols'] = [ 108 prot.lower() for prot in l[11:].split(' ') 109 ] 110 111 self.ports = alloc_ports(port_specs={ 112 'ftp': socket.SOCK_STREAM, 113 'ftps': socket.SOCK_STREAM, 114 'http': socket.SOCK_STREAM, 115 'https': socket.SOCK_STREAM, 116 'proxy': socket.SOCK_STREAM, 117 'proxys': socket.SOCK_STREAM, 118 'h2proxys': socket.SOCK_STREAM, 119 'caddy': socket.SOCK_STREAM, 120 'caddys': socket.SOCK_STREAM, 121 'ws': socket.SOCK_STREAM, 122 }) 123 self.httpd = self.config['httpd']['httpd'] 124 self.apachectl = self.config['httpd']['apachectl'] 125 self.apxs = self.config['httpd']['apxs'] 126 if len(self.apxs) == 0: 127 self.apxs = None 128 self._httpd_version = None 129 130 self.examples_pem = { 131 'key': 'xxx', 132 'cert': 'xxx', 133 } 134 self.htdocs_dir = os.path.join(self.gen_dir, 'htdocs') 135 self.tld = 'http.curl.se' 136 self.domain1 = f"one.{self.tld}" 137 self.domain1brotli = f"brotli.one.{self.tld}" 138 self.domain2 = f"two.{self.tld}" 139 self.ftp_domain = f"ftp.{self.tld}" 140 self.proxy_domain = f"proxy.{self.tld}" 141 self.cert_specs = [ 142 CertificateSpec(domains=[self.domain1, self.domain1brotli, 'localhost', '127.0.0.1'], key_type='rsa2048'), 143 CertificateSpec(domains=[self.domain2], key_type='rsa2048'), 144 CertificateSpec(domains=[self.ftp_domain], key_type='rsa2048'), 145 CertificateSpec(domains=[self.proxy_domain, '127.0.0.1'], key_type='rsa2048'), 146 CertificateSpec(name="clientsX", sub_specs=[ 147 CertificateSpec(name="user1", client=True), 148 ]), 149 ] 150 151 self.nghttpx = self.config['nghttpx']['nghttpx'] 152 if len(self.nghttpx.strip()) == 0: 153 self.nghttpx = None 154 self._nghttpx_version = None 155 self.nghttpx_with_h3 = False 156 if self.nghttpx is not None: 157 p = subprocess.run(args=[self.nghttpx, '-v'], 158 capture_output=True, text=True) 159 if p.returncode != 0: 160 # not a working nghttpx 161 self.nghttpx = None 162 else: 163 self._nghttpx_version = re.sub(r'^nghttpx\s*', '', p.stdout.strip()) 164 self.nghttpx_with_h3 = re.match(r'.* nghttp3/.*', p.stdout.strip()) is not None 165 log.debug(f'nghttpx -v: {p.stdout}') 166 167 self.caddy = self.config['caddy']['caddy'] 168 self._caddy_version = None 169 if len(self.caddy.strip()) == 0: 170 self.caddy = None 171 if self.caddy is not None: 172 try: 173 p = subprocess.run(args=[self.caddy, 'version'], 174 capture_output=True, text=True) 175 if p.returncode != 0: 176 # not a working caddy 177 self.caddy = None 178 m = re.match(r'v?(\d+\.\d+\.\d+).*', p.stdout) 179 if m: 180 self._caddy_version = m.group(1) 181 else: 182 raise f'Unable to determine cadd version from: {p.stdout}' 183 except: 184 self.caddy = None 185 186 self.vsftpd = self.config['vsftpd']['vsftpd'] 187 self._vsftpd_version = None 188 if self.vsftpd is not None: 189 try: 190 p = subprocess.run(args=[self.vsftpd, '-v'], 191 capture_output=True, text=True) 192 if p.returncode != 0: 193 # not a working vsftpd 194 self.vsftpd = None 195 m = re.match(r'vsftpd: version (\d+\.\d+\.\d+)', p.stderr) 196 if m: 197 self._vsftpd_version = m.group(1) 198 elif len(p.stderr) == 0: 199 # vsftp does not use stdout or stderr for printing its version... -.- 200 self._vsftpd_version = 'unknown' 201 else: 202 raise Exception(f'Unable to determine VsFTPD version from: {p.stderr}') 203 except Exception as e: 204 self.vsftpd = None 205 206 @property 207 def httpd_version(self): 208 if self._httpd_version is None and self.apxs is not None: 209 try: 210 p = subprocess.run(args=[self.apxs, '-q', 'HTTPD_VERSION'], 211 capture_output=True, text=True) 212 if p.returncode != 0: 213 log.error(f'{self.apxs} failed to query HTTPD_VERSION: {p}') 214 else: 215 self._httpd_version = p.stdout.strip() 216 except Exception as e: 217 log.error(f'{self.apxs} failed to run: {e}') 218 return self._httpd_version 219 220 def versiontuple(self, v): 221 v = re.sub(r'(\d+\.\d+(\.\d+)?)(-\S+)?', r'\1', v) 222 return tuple(map(int, v.split('.'))) 223 224 def httpd_is_at_least(self, minv): 225 if self.httpd_version is None: 226 return False 227 hv = self.versiontuple(self.httpd_version) 228 return hv >= self.versiontuple(minv) 229 230 def caddy_is_at_least(self, minv): 231 if self.caddy_version is None: 232 return False 233 hv = self.versiontuple(self.caddy_version) 234 return hv >= self.versiontuple(minv) 235 236 def is_complete(self) -> bool: 237 return os.path.isfile(self.httpd) and \ 238 os.path.isfile(self.apachectl) and \ 239 self.apxs is not None and \ 240 os.path.isfile(self.apxs) 241 242 def get_incomplete_reason(self) -> Optional[str]: 243 if self.httpd is None or len(self.httpd.strip()) == 0: 244 return f'httpd not configured, see `--with-test-httpd=<path>`' 245 if not os.path.isfile(self.httpd): 246 return f'httpd ({self.httpd}) not found' 247 if not os.path.isfile(self.apachectl): 248 return f'apachectl ({self.apachectl}) not found' 249 if self.apxs is None: 250 return f"command apxs not found (commonly provided in apache2-dev)" 251 if not os.path.isfile(self.apxs): 252 return f"apxs ({self.apxs}) not found" 253 return None 254 255 @property 256 def nghttpx_version(self): 257 return self._nghttpx_version 258 259 @property 260 def caddy_version(self): 261 return self._caddy_version 262 263 @property 264 def vsftpd_version(self): 265 return self._vsftpd_version 266 267 268class Env: 269 270 CONFIG = EnvConfig() 271 272 @staticmethod 273 def setup_incomplete() -> bool: 274 return not Env.CONFIG.is_complete() 275 276 @staticmethod 277 def incomplete_reason() -> Optional[str]: 278 return Env.CONFIG.get_incomplete_reason() 279 280 @staticmethod 281 def have_nghttpx() -> bool: 282 return Env.CONFIG.nghttpx is not None 283 284 @staticmethod 285 def have_h3_server() -> bool: 286 return Env.CONFIG.nghttpx_with_h3 287 288 @staticmethod 289 def have_ssl_curl() -> bool: 290 return 'ssl' in Env.CONFIG.curl_props['features'] 291 292 @staticmethod 293 def have_h2_curl() -> bool: 294 return 'http2' in Env.CONFIG.curl_props['features'] 295 296 @staticmethod 297 def have_h3_curl() -> bool: 298 return 'http3' in Env.CONFIG.curl_props['features'] 299 300 @staticmethod 301 def curl_uses_lib(libname: str) -> bool: 302 return libname.lower() in Env.CONFIG.curl_props['libs'] 303 304 @staticmethod 305 def curl_uses_ossl_quic() -> bool: 306 if Env.have_h3_curl(): 307 return not Env.curl_uses_lib('ngtcp2') and Env.curl_uses_lib('nghttp3') 308 return False 309 310 @staticmethod 311 def curl_has_feature(feature: str) -> bool: 312 return feature.lower() in Env.CONFIG.curl_props['features'] 313 314 @staticmethod 315 def curl_has_protocol(protocol: str) -> bool: 316 return protocol.lower() in Env.CONFIG.curl_props['protocols'] 317 318 @staticmethod 319 def curl_lib_version(libname: str) -> str: 320 prefix = f'{libname.lower()}/' 321 for lversion in Env.CONFIG.curl_props['lib_versions']: 322 if lversion.startswith(prefix): 323 return lversion[len(prefix):] 324 return 'unknown' 325 326 @staticmethod 327 def curl_lib_version_at_least(libname: str, min_version) -> str: 328 lversion = Env.curl_lib_version(libname) 329 if lversion != 'unknown': 330 return Env.CONFIG.versiontuple(min_version) <= \ 331 Env.CONFIG.versiontuple(lversion) 332 return False 333 334 @staticmethod 335 def curl_os() -> str: 336 return Env.CONFIG.curl_props['os'] 337 338 @staticmethod 339 def curl_fullname() -> str: 340 return Env.CONFIG.curl_props['fullname'] 341 342 @staticmethod 343 def curl_version() -> str: 344 return Env.CONFIG.curl_props['version'] 345 346 @staticmethod 347 def curl_is_debug() -> bool: 348 return Env.CONFIG.curl_is_debug 349 350 @staticmethod 351 def have_h3() -> bool: 352 return Env.have_h3_curl() and Env.have_h3_server() 353 354 @staticmethod 355 def httpd_version() -> str: 356 return Env.CONFIG.httpd_version 357 358 @staticmethod 359 def nghttpx_version() -> str: 360 return Env.CONFIG.nghttpx_version 361 362 @staticmethod 363 def caddy_version() -> str: 364 return Env.CONFIG.caddy_version 365 366 @staticmethod 367 def caddy_is_at_least(minv) -> bool: 368 return Env.CONFIG.caddy_is_at_least(minv) 369 370 @staticmethod 371 def httpd_is_at_least(minv) -> bool: 372 return Env.CONFIG.httpd_is_at_least(minv) 373 374 @staticmethod 375 def has_caddy() -> bool: 376 return Env.CONFIG.caddy is not None 377 378 @staticmethod 379 def has_vsftpd() -> bool: 380 return Env.CONFIG.vsftpd is not None 381 382 @staticmethod 383 def vsftpd_version() -> str: 384 return Env.CONFIG.vsftpd_version 385 386 def __init__(self, pytestconfig=None): 387 self._verbose = pytestconfig.option.verbose \ 388 if pytestconfig is not None else 0 389 self._ca = None 390 self._test_timeout = 300.0 if self._verbose > 1 else 60.0 # seconds 391 392 def issue_certs(self): 393 if self._ca is None: 394 ca_dir = os.path.join(self.CONFIG.gen_dir, 'ca') 395 self._ca = TestCA.create_root(name=self.CONFIG.tld, 396 store_dir=ca_dir, 397 key_type="rsa2048") 398 self._ca.issue_certs(self.CONFIG.cert_specs) 399 400 def setup(self): 401 os.makedirs(self.gen_dir, exist_ok=True) 402 os.makedirs(self.htdocs_dir, exist_ok=True) 403 self.issue_certs() 404 405 def get_credentials(self, domain) -> Optional[Credentials]: 406 creds = self.ca.get_credentials_for_name(domain) 407 if len(creds) > 0: 408 return creds[0] 409 return None 410 411 @property 412 def verbose(self) -> int: 413 return self._verbose 414 415 @property 416 def test_timeout(self) -> Optional[float]: 417 return self._test_timeout 418 419 @test_timeout.setter 420 def test_timeout(self, val: Optional[float]): 421 self._test_timeout = val 422 423 @property 424 def gen_dir(self) -> str: 425 return self.CONFIG.gen_dir 426 427 @property 428 def project_dir(self) -> str: 429 return self.CONFIG.project_dir 430 431 @property 432 def ca(self): 433 return self._ca 434 435 @property 436 def htdocs_dir(self) -> str: 437 return self.CONFIG.htdocs_dir 438 439 @property 440 def domain1(self) -> str: 441 return self.CONFIG.domain1 442 443 @property 444 def domain1brotli(self) -> str: 445 return self.CONFIG.domain1brotli 446 447 @property 448 def domain2(self) -> str: 449 return self.CONFIG.domain2 450 451 @property 452 def ftp_domain(self) -> str: 453 return self.CONFIG.ftp_domain 454 455 @property 456 def proxy_domain(self) -> str: 457 return self.CONFIG.proxy_domain 458 459 @property 460 def http_port(self) -> int: 461 return self.CONFIG.ports['http'] 462 463 @property 464 def https_port(self) -> int: 465 return self.CONFIG.ports['https'] 466 467 @property 468 def h3_port(self) -> int: 469 return self.https_port 470 471 @property 472 def proxy_port(self) -> int: 473 return self.CONFIG.ports['proxy'] 474 475 @property 476 def proxys_port(self) -> int: 477 return self.CONFIG.ports['proxys'] 478 479 @property 480 def ftp_port(self) -> int: 481 return self.CONFIG.ports['ftp'] 482 483 @property 484 def ftps_port(self) -> int: 485 return self.CONFIG.ports['ftps'] 486 487 @property 488 def h2proxys_port(self) -> int: 489 return self.CONFIG.ports['h2proxys'] 490 491 def pts_port(self, proto: str = 'http/1.1') -> int: 492 # proxy tunnel port 493 return self.CONFIG.ports['h2proxys' if proto == 'h2' else 'proxys'] 494 495 @property 496 def caddy(self) -> str: 497 return self.CONFIG.caddy 498 499 @property 500 def caddy_https_port(self) -> int: 501 return self.CONFIG.ports['caddys'] 502 503 @property 504 def caddy_http_port(self) -> int: 505 return self.CONFIG.ports['caddy'] 506 507 @property 508 def vsftpd(self) -> str: 509 return self.CONFIG.vsftpd 510 511 @property 512 def ws_port(self) -> int: 513 return self.CONFIG.ports['ws'] 514 515 @property 516 def curl(self) -> str: 517 return self.CONFIG.curl 518 519 @property 520 def httpd(self) -> str: 521 return self.CONFIG.httpd 522 523 @property 524 def apachectl(self) -> str: 525 return self.CONFIG.apachectl 526 527 @property 528 def apxs(self) -> str: 529 return self.CONFIG.apxs 530 531 @property 532 def nghttpx(self) -> Optional[str]: 533 return self.CONFIG.nghttpx 534 535 @property 536 def slow_network(self) -> bool: 537 return "CURL_DBG_SOCK_WBLOCK" in os.environ or \ 538 "CURL_DBG_SOCK_WPARTIAL" in os.environ 539 540 @property 541 def ci_run(self) -> bool: 542 return "CURL_CI" in os.environ 543 544 def authority_for(self, domain: str, alpn_proto: Optional[str] = None): 545 if alpn_proto is None or \ 546 alpn_proto in ['h2', 'http/1.1', 'http/1.0', 'http/0.9']: 547 return f'{domain}:{self.https_port}' 548 if alpn_proto in ['h3']: 549 return f'{domain}:{self.h3_port}' 550 return f'{domain}:{self.http_port}' 551 552 def make_data_file(self, indir: str, fname: str, fsize: int) -> str: 553 fpath = os.path.join(indir, fname) 554 s10 = "0123456789" 555 s = (101 * s10) + s10[0:3] 556 with open(fpath, 'w') as fd: 557 for i in range(int(fsize / 1024)): 558 fd.write(f"{i:09d}-{s}\n") 559 remain = int(fsize % 1024) 560 if remain != 0: 561 i = int(fsize / 1024) + 1 562 s = f"{i:09d}-{s}\n" 563 fd.write(s[0:remain]) 564 return fpath 565 566 def make_clients(self): 567 client_dir = os.path.join(self.project_dir, 'tests/http/clients') 568 p = subprocess.run(['make'], capture_output=True, text=True, 569 cwd=client_dir) 570 if p.returncode != 0: 571 pytest.exit(f"`make`in {client_dir} failed:\n{p.stderr}") 572 return False 573 return True 574
