1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import inspect 28import logging 29import os 30import subprocess 31from datetime import timedelta, datetime 32from json import JSONEncoder 33import time 34from typing import List, Union, Optional 35import copy 36 37from .curl import CurlClient, ExecResult 38from .env import Env 39 40 41log = logging.getLogger(__name__) 42 43 44class Httpd: 45 46 MODULES = [ 47 'log_config', 'logio', 'unixd', 'version', 'watchdog', 48 'authn_core', 'authn_file', 49 'authz_user', 'authz_core', 'authz_host', 50 'auth_basic', 'auth_digest', 51 'alias', 'env', 'filter', 'headers', 'mime', 'setenvif', 52 'socache_shmcb', 53 'rewrite', 'http2', 'ssl', 'proxy', 'proxy_http', 'proxy_connect', 54 'brotli', 55 'mpm_event', 56 ] 57 COMMON_MODULES_DIRS = [ 58 '/usr/lib/apache2/modules', # debian 59 '/usr/libexec/apache2/', # macos 60 ] 61 62 MOD_CURLTEST = None 63 64 def __init__(self, env: Env, proxy_auth: bool = False): 65 self.env = env 66 self._cmd = env.apachectl 67 self._apache_dir = os.path.join(env.gen_dir, 'apache') 68 self._run_dir = os.path.join(self._apache_dir, 'run') 69 self._lock_dir = os.path.join(self._apache_dir, 'locks') 70 self._docs_dir = os.path.join(self._apache_dir, 'docs') 71 self._conf_dir = os.path.join(self._apache_dir, 'conf') 72 self._conf_file = os.path.join(self._conf_dir, 'test.conf') 73 self._logs_dir = os.path.join(self._apache_dir, 'logs') 74 self._error_log = os.path.join(self._logs_dir, 'error_log') 75 self._tmp_dir = os.path.join(self._apache_dir, 'tmp') 76 self._basic_passwords = os.path.join(self._conf_dir, 'basic.passwords') 77 self._digest_passwords = os.path.join(self._conf_dir, 'digest.passwords') 78 self._mods_dir = None 79 self._auth_digest = True 80 self._proxy_auth_basic = proxy_auth 81 self._extra_configs = {} 82 self._loaded_extra_configs = None 83 assert env.apxs 84 p = subprocess.run(args=[env.apxs, '-q', 'libexecdir'], 85 capture_output=True, text=True) 86 if p.returncode != 0: 87 raise Exception(f'{env.apxs} failed to query libexecdir: {p}') 88 self._mods_dir = p.stdout.strip() 89 if self._mods_dir is None: 90 raise Exception('apache modules dir cannot be found') 91 if not os.path.exists(self._mods_dir): 92 raise Exception(f'apache modules dir does not exist: {self._mods_dir}') 93 self._process = None 94 self._rmf(self._error_log) 95 self._init_curltest() 96 97 @property 98 def docs_dir(self): 99 return self._docs_dir 100 101 def clear_logs(self): 102 self._rmf(self._error_log) 103 104 def exists(self): 105 return os.path.exists(self._cmd) 106 107 def set_extra_config(self, domain: str, lines: Optional[Union[str, List[str]]]): 108 if lines is None: 109 self._extra_configs.pop(domain, None) 110 else: 111 self._extra_configs[domain] = lines 112 113 def clear_extra_configs(self): 114 self._extra_configs = {} 115 116 def set_proxy_auth(self, active: bool): 117 self._proxy_auth_basic = active 118 119 def _run(self, args, intext=''): 120 env = os.environ.copy() 121 env['APACHE_RUN_DIR'] = self._run_dir 122 env['APACHE_RUN_USER'] = os.environ['USER'] 123 env['APACHE_LOCK_DIR'] = self._lock_dir 124 env['APACHE_CONFDIR'] = self._apache_dir 125 p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, 126 cwd=self.env.gen_dir, 127 input=intext.encode() if intext else None, 128 env=env) 129 start = datetime.now() 130 return ExecResult(args=args, exit_code=p.returncode, 131 stdout=p.stdout.decode().splitlines(), 132 stderr=p.stderr.decode().splitlines(), 133 duration=datetime.now() - start) 134 135 def _apachectl(self, cmd: str): 136 args = [self.env.apachectl, 137 "-d", self._apache_dir, 138 "-f", self._conf_file, 139 "-k", cmd] 140 return self._run(args=args) 141 142 def start(self): 143 if self._process: 144 self.stop() 145 self._write_config() 146 with open(self._error_log, 'a') as fd: 147 fd.write('start of server\n') 148 with open(os.path.join(self._apache_dir, 'xxx'), 'a') as fd: 149 fd.write('start of server\n') 150 r = self._apachectl('start') 151 if r.exit_code != 0: 152 log.error(f'failed to start httpd: {r}') 153 return False 154 self._loaded_extra_configs = copy.deepcopy(self._extra_configs) 155 return self.wait_live(timeout=timedelta(seconds=5)) 156 157 def stop(self): 158 r = self._apachectl('stop') 159 self._loaded_extra_configs = None 160 if r.exit_code == 0: 161 return self.wait_dead(timeout=timedelta(seconds=5)) 162 log.fatal(f'stopping httpd failed: {r}') 163 return r.exit_code == 0 164 165 def restart(self): 166 self.stop() 167 return self.start() 168 169 def reload(self): 170 self._write_config() 171 r = self._apachectl("graceful") 172 self._loaded_extra_configs = None 173 if r.exit_code != 0: 174 log.error(f'failed to reload httpd: {r}') 175 self._loaded_extra_configs = copy.deepcopy(self._extra_configs) 176 return self.wait_live(timeout=timedelta(seconds=5)) 177 178 def reload_if_config_changed(self): 179 if self._loaded_extra_configs == self._extra_configs: 180 return True 181 return self.reload() 182 183 def wait_dead(self, timeout: timedelta): 184 curl = CurlClient(env=self.env, run_dir=self._tmp_dir) 185 try_until = datetime.now() + timeout 186 while datetime.now() < try_until: 187 r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/') 188 if r.exit_code != 0: 189 return True 190 time.sleep(.1) 191 log.debug(f"Server still responding after {timeout}") 192 return False 193 194 def wait_live(self, timeout: timedelta): 195 curl = CurlClient(env=self.env, run_dir=self._tmp_dir, 196 timeout=timeout.total_seconds()) 197 try_until = datetime.now() + timeout 198 while datetime.now() < try_until: 199 r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/') 200 if r.exit_code == 0: 201 return True 202 time.sleep(.1) 203 log.debug(f"Server still not responding after {timeout}") 204 return False 205 206 def _rmf(self, path): 207 if os.path.exists(path): 208 return os.remove(path) 209 210 def _mkpath(self, path): 211 if not os.path.exists(path): 212 return os.makedirs(path) 213 214 def _write_config(self): 215 domain1 = self.env.domain1 216 domain1brotli = self.env.domain1brotli 217 creds1 = self.env.get_credentials(domain1) 218 assert creds1 # convince pytype this isn't None 219 domain2 = self.env.domain2 220 creds2 = self.env.get_credentials(domain2) 221 assert creds2 # convince pytype this isn't None 222 proxy_domain = self.env.proxy_domain 223 proxy_creds = self.env.get_credentials(proxy_domain) 224 assert proxy_creds # convince pytype this isn't None 225 self._mkpath(self._conf_dir) 226 self._mkpath(self._logs_dir) 227 self._mkpath(self._tmp_dir) 228 self._mkpath(os.path.join(self._docs_dir, 'two')) 229 with open(os.path.join(self._docs_dir, 'data.json'), 'w') as fd: 230 data = { 231 'server': f'{domain1}', 232 } 233 fd.write(JSONEncoder().encode(data)) 234 with open(os.path.join(self._docs_dir, 'two/data.json'), 'w') as fd: 235 data = { 236 'server': f'{domain2}', 237 } 238 fd.write(JSONEncoder().encode(data)) 239 if self._proxy_auth_basic: 240 with open(self._basic_passwords, 'w') as fd: 241 fd.write('proxy:$apr1$FQfeInbs$WQZbODJlVg60j0ogEIlTW/\n') 242 if self._auth_digest: 243 with open(self._digest_passwords, 'w') as fd: 244 fd.write('test:restricted area:57123e269fd73d71ae0656594e938e2f\n') 245 self._mkpath(os.path.join(self.docs_dir, 'restricted/digest')) 246 with open(os.path.join(self.docs_dir, 'restricted/digest/data.json'), 'w') as fd: 247 fd.write('{"area":"digest"}\n') 248 with open(self._conf_file, 'w') as fd: 249 for m in self.MODULES: 250 if os.path.exists(os.path.join(self._mods_dir, f'mod_{m}.so')): 251 fd.write(f'LoadModule {m}_module "{self._mods_dir}/mod_{m}.so"\n') 252 if Httpd.MOD_CURLTEST is not None: 253 fd.write(f'LoadModule curltest_module "{Httpd.MOD_CURLTEST}"\n') 254 conf = [ # base server config 255 f'ServerRoot "{self._apache_dir}"', 256 'DefaultRuntimeDir logs', 257 'PidFile httpd.pid', 258 f'ErrorLog {self._error_log}', 259 f'LogLevel {self._get_log_level()}', 260 'StartServers 4', 261 'ReadBufferSize 16000', 262 'H2MinWorkers 16', 263 'H2MaxWorkers 256', 264 f'Listen {self.env.http_port}', 265 f'Listen {self.env.https_port}', 266 f'Listen {self.env.proxy_port}', 267 f'Listen {self.env.proxys_port}', 268 f'TypesConfig "{self._conf_dir}/mime.types', 269 'SSLSessionCache "shmcb:ssl_gcache_data(32000)"', 270 ] 271 if 'base' in self._extra_configs: 272 conf.extend(self._extra_configs['base']) 273 conf.extend([ # plain http host for domain1 274 f'<VirtualHost *:{self.env.http_port}>', 275 f' ServerName {domain1}', 276 ' ServerAlias localhost', 277 f' DocumentRoot "{self._docs_dir}"', 278 ' Protocols h2c http/1.1', 279 ' H2Direct on', 280 ]) 281 conf.extend(self._curltest_conf(domain1)) 282 conf.extend([ 283 '</VirtualHost>', 284 '', 285 ]) 286 conf.extend([ # https host for domain1, h1 + h2 287 f'<VirtualHost *:{self.env.https_port}>', 288 f' ServerName {domain1}', 289 ' ServerAlias localhost', 290 ' Protocols h2 http/1.1', 291 ' SSLEngine on', 292 f' SSLCertificateFile {creds1.cert_file}', 293 f' SSLCertificateKeyFile {creds1.pkey_file}', 294 f' DocumentRoot "{self._docs_dir}"', 295 ]) 296 conf.extend(self._curltest_conf(domain1)) 297 if domain1 in self._extra_configs: 298 conf.extend(self._extra_configs[domain1]) 299 conf.extend([ 300 '</VirtualHost>', 301 '', 302 ]) 303 # Alternate to domain1 with BROTLI compression 304 conf.extend([ # https host for domain1, h1 + h2 305 f'<VirtualHost *:{self.env.https_port}>', 306 f' ServerName {domain1brotli}', 307 ' Protocols h2 http/1.1', 308 ' SSLEngine on', 309 f' SSLCertificateFile {creds1.cert_file}', 310 f' SSLCertificateKeyFile {creds1.pkey_file}', 311 f' DocumentRoot "{self._docs_dir}"', 312 ' SetOutputFilter BROTLI_COMPRESS', 313 ]) 314 conf.extend(self._curltest_conf(domain1)) 315 if domain1 in self._extra_configs: 316 conf.extend(self._extra_configs[domain1]) 317 conf.extend([ 318 '</VirtualHost>', 319 '', 320 ]) 321 conf.extend([ # plain http host for domain2 322 f'<VirtualHost *:{self.env.http_port}>', 323 f' ServerName {domain2}', 324 ' ServerAlias localhost', 325 f' DocumentRoot "{self._docs_dir}"', 326 ' Protocols h2c http/1.1', 327 ]) 328 conf.extend(self._curltest_conf(domain2)) 329 conf.extend([ 330 '</VirtualHost>', 331 '', 332 ]) 333 conf.extend([ # https host for domain2, no h2 334 f'<VirtualHost *:{self.env.https_port}>', 335 f' ServerName {domain2}', 336 ' Protocols http/1.1', 337 ' SSLEngine on', 338 f' SSLCertificateFile {creds2.cert_file}', 339 f' SSLCertificateKeyFile {creds2.pkey_file}', 340 f' DocumentRoot "{self._docs_dir}/two"', 341 ]) 342 conf.extend(self._curltest_conf(domain2)) 343 if domain2 in self._extra_configs: 344 conf.extend(self._extra_configs[domain2]) 345 conf.extend([ 346 '</VirtualHost>', 347 '', 348 ]) 349 conf.extend([ # http forward proxy 350 f'<VirtualHost *:{self.env.proxy_port}>', 351 f' ServerName {proxy_domain}', 352 ' Protocols h2c http/1.1', 353 ' ProxyRequests On', 354 ' H2ProxyRequests On', 355 ' ProxyVia On', 356 f' AllowCONNECT {self.env.http_port} {self.env.https_port}', 357 ]) 358 conf.extend(self._get_proxy_conf()) 359 conf.extend([ 360 '</VirtualHost>', 361 '', 362 ]) 363 conf.extend([ # https forward proxy 364 f'<VirtualHost *:{self.env.proxys_port}>', 365 f' ServerName {proxy_domain}', 366 ' Protocols h2 http/1.1', 367 ' SSLEngine on', 368 f' SSLCertificateFile {proxy_creds.cert_file}', 369 f' SSLCertificateKeyFile {proxy_creds.pkey_file}', 370 ' ProxyRequests On', 371 ' H2ProxyRequests On', 372 ' ProxyVia On', 373 f' AllowCONNECT {self.env.http_port} {self.env.https_port}', 374 ]) 375 conf.extend(self._get_proxy_conf()) 376 conf.extend([ 377 '</VirtualHost>', 378 '', 379 ]) 380 381 fd.write("\n".join(conf)) 382 with open(os.path.join(self._conf_dir, 'mime.types'), 'w') as fd: 383 fd.write("\n".join([ 384 'text/html html', 385 'application/json json', 386 '' 387 ])) 388 389 def _get_proxy_conf(self): 390 if self._proxy_auth_basic: 391 return [ 392 ' <Proxy "*">', 393 ' AuthType Basic', 394 ' AuthName "Restricted Proxy"', 395 ' AuthBasicProvider file', 396 f' AuthUserFile "{self._basic_passwords}"', 397 ' Require user proxy', 398 ' </Proxy>', 399 ] 400 else: 401 return [ 402 ' <Proxy "*">', 403 ' Require ip 127.0.0.1', 404 ' </Proxy>', 405 ] 406 407 def _get_log_level(self): 408 if self.env.verbose > 3: 409 return 'trace2' 410 if self.env.verbose > 2: 411 return 'trace1' 412 if self.env.verbose > 1: 413 return 'debug' 414 return 'info' 415 416 def _curltest_conf(self, servername) -> List[str]: 417 lines = [] 418 if Httpd.MOD_CURLTEST is not None: 419 lines.extend([ 420 ' Redirect 302 /data.json.302 /data.json', 421 ' Redirect 301 /curltest/echo301 /curltest/echo', 422 ' Redirect 302 /curltest/echo302 /curltest/echo', 423 ' Redirect 303 /curltest/echo303 /curltest/echo', 424 ' Redirect 307 /curltest/echo307 /curltest/echo', 425 ' <Location /curltest/sslinfo>', 426 ' SSLOptions StdEnvVars', 427 ' SetHandler curltest-sslinfo', 428 ' </Location>', 429 ' <Location /curltest/echo>', 430 ' SetHandler curltest-echo', 431 ' </Location>', 432 ' <Location /curltest/put>', 433 ' SetHandler curltest-put', 434 ' </Location>', 435 ' <Location /curltest/tweak>', 436 ' SetHandler curltest-tweak', 437 ' </Location>', 438 ' Redirect 302 /tweak /curltest/tweak', 439 ' <Location /curltest/1_1>', 440 ' SetHandler curltest-1_1-required', 441 ' </Location>', 442 ' <Location /curltest/shutdown_unclean>', 443 ' SetHandler curltest-tweak', 444 ' SetEnv force-response-1.0 1', 445 ' </Location>', 446 ' SetEnvIf Request_URI "/shutdown_unclean" ssl-unclean=1', 447 ]) 448 if self._auth_digest: 449 lines.extend([ 450 f' <Directory {self.docs_dir}/restricted/digest>', 451 ' AuthType Digest', 452 ' AuthName "restricted area"', 453 f' AuthDigestDomain "https://{servername}"', 454 ' AuthBasicProvider file', 455 f' AuthUserFile "{self._digest_passwords}"', 456 ' Require valid-user', 457 ' </Directory>', 458 459 ]) 460 return lines 461 462 def _init_curltest(self): 463 if Httpd.MOD_CURLTEST is not None: 464 return 465 local_dir = os.path.dirname(inspect.getfile(Httpd)) 466 p = subprocess.run([self.env.apxs, '-c', 'mod_curltest.c'], 467 capture_output=True, 468 cwd=os.path.join(local_dir, 'mod_curltest')) 469 rv = p.returncode 470 if rv != 0: 471 log.error(f"compiling mod_curltest failed: {p.stderr}") 472 raise Exception(f"compiling mod_curltest failed: {p.stderr}") 473 Httpd.MOD_CURLTEST = os.path.join( 474 local_dir, 'mod_curltest/.libs/mod_curltest.so') 475
