1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import inspect 28import logging 29import os 30import subprocess 31from datetime import timedelta, datetime 32from json import JSONEncoder 33import time 34from typing import List, Union, Optional 35import copy 36 37from .curl import CurlClient, ExecResult 38from .env import Env 39 40 41log = logging.getLogger(__name__) 42 43 44class Httpd: 45 46 MODULES = [ 47 'log_config', 'logio', 'unixd', 'version', 'watchdog', 48 'authn_core', 'authn_file', 49 'authz_user', 'authz_core', 'authz_host', 50 'auth_basic', 'auth_digest', 51 'alias', 'env', 'filter', 'headers', 'mime', 'setenvif', 52 'socache_shmcb', 53 'rewrite', 'http2', 'ssl', 'proxy', 'proxy_http', 'proxy_connect', 54 'brotli', 55 'mpm_event', 56 ] 57 COMMON_MODULES_DIRS = [ 58 '/usr/lib/apache2/modules', # debian 59 '/usr/libexec/apache2/', # macos 60 ] 61 62 MOD_CURLTEST = None 63 64 def __init__(self, env: Env, proxy_auth: bool = False): 65 self.env = env 66 self._cmd = env.apachectl 67 self._apache_dir = os.path.join(env.gen_dir, 'apache') 68 self._run_dir = os.path.join(self._apache_dir, 'run') 69 self._lock_dir = os.path.join(self._apache_dir, 'locks') 70 self._docs_dir = os.path.join(self._apache_dir, 'docs') 71 self._conf_dir = os.path.join(self._apache_dir, 'conf') 72 self._conf_file = os.path.join(self._conf_dir, 'test.conf') 73 self._logs_dir = os.path.join(self._apache_dir, 'logs') 74 self._error_log = os.path.join(self._logs_dir, 'error_log') 75 self._tmp_dir = os.path.join(self._apache_dir, 'tmp') 76 self._basic_passwords = os.path.join(self._conf_dir, 'basic.passwords') 77 self._digest_passwords = os.path.join(self._conf_dir, 'digest.passwords') 78 self._mods_dir = None 79 self._auth_digest = True 80 self._proxy_auth_basic = proxy_auth 81 self._extra_configs = {} 82 self._loaded_extra_configs = None 83 assert env.apxs 84 p = subprocess.run(args=[env.apxs, '-q', 'libexecdir'], 85 capture_output=True, text=True) 86 if p.returncode != 0: 87 raise Exception(f'{env.apxs} failed to query libexecdir: {p}') 88 self._mods_dir = p.stdout.strip() 89 if self._mods_dir is None: 90 raise Exception(f'apache modules dir cannot be found') 91 if not os.path.exists(self._mods_dir): 92 raise Exception(f'apache modules dir does not exist: {self._mods_dir}') 93 self._process = None 94 self._rmf(self._error_log) 95 self._init_curltest() 96 97 @property 98 def docs_dir(self): 99 return self._docs_dir 100 101 def clear_logs(self): 102 self._rmf(self._error_log) 103 104 def exists(self): 105 return os.path.exists(self._cmd) 106 107 def set_extra_config(self, domain: str, lines: Optional[Union[str, List[str]]]): 108 if lines is None: 109 self._extra_configs.pop(domain, None) 110 else: 111 self._extra_configs[domain] = lines 112 113 def clear_extra_configs(self): 114 self._extra_configs = {} 115 116 def set_proxy_auth(self, active: bool): 117 self._proxy_auth_basic = active 118 119 def _run(self, args, intext=''): 120 env = {} 121 for key, val in os.environ.items(): 122 env[key] = val 123 env['APACHE_RUN_DIR'] = self._run_dir 124 env['APACHE_RUN_USER'] = os.environ['USER'] 125 env['APACHE_LOCK_DIR'] = self._lock_dir 126 env['APACHE_CONFDIR'] = self._apache_dir 127 p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, 128 cwd=self.env.gen_dir, 129 input=intext.encode() if intext else None, 130 env=env) 131 start = datetime.now() 132 return ExecResult(args=args, exit_code=p.returncode, 133 stdout=p.stdout.decode().splitlines(), 134 stderr=p.stderr.decode().splitlines(), 135 duration=datetime.now() - start) 136 137 def _apachectl(self, cmd: str): 138 args = [self.env.apachectl, 139 "-d", self._apache_dir, 140 "-f", self._conf_file, 141 "-k", cmd] 142 return self._run(args=args) 143 144 def start(self): 145 if self._process: 146 self.stop() 147 self._write_config() 148 with open(self._error_log, 'a') as fd: 149 fd.write('start of server\n') 150 with open(os.path.join(self._apache_dir, 'xxx'), 'a') as fd: 151 fd.write('start of server\n') 152 r = self._apachectl('start') 153 if r.exit_code != 0: 154 log.error(f'failed to start httpd: {r}') 155 return False 156 self._loaded_extra_configs = copy.deepcopy(self._extra_configs) 157 return self.wait_live(timeout=timedelta(seconds=5)) 158 159 def stop(self): 160 r = self._apachectl('stop') 161 self._loaded_extra_configs = None 162 if r.exit_code == 0: 163 return self.wait_dead(timeout=timedelta(seconds=5)) 164 log.fatal(f'stopping httpd failed: {r}') 165 return r.exit_code == 0 166 167 def restart(self): 168 self.stop() 169 return self.start() 170 171 def reload(self): 172 self._write_config() 173 r = self._apachectl("graceful") 174 self._loaded_extra_configs = None 175 if r.exit_code != 0: 176 log.error(f'failed to reload httpd: {r}') 177 self._loaded_extra_configs = copy.deepcopy(self._extra_configs) 178 return self.wait_live(timeout=timedelta(seconds=5)) 179 180 def reload_if_config_changed(self): 181 if self._loaded_extra_configs == self._extra_configs: 182 return True 183 return self.reload() 184 185 def wait_dead(self, timeout: timedelta): 186 curl = CurlClient(env=self.env, run_dir=self._tmp_dir) 187 try_until = datetime.now() + timeout 188 while datetime.now() < try_until: 189 r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/') 190 if r.exit_code != 0: 191 return True 192 time.sleep(.1) 193 log.debug(f"Server still responding after {timeout}") 194 return False 195 196 def wait_live(self, timeout: timedelta): 197 curl = CurlClient(env=self.env, run_dir=self._tmp_dir, 198 timeout=timeout.total_seconds()) 199 try_until = datetime.now() + timeout 200 while datetime.now() < try_until: 201 r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/') 202 if r.exit_code == 0: 203 return True 204 time.sleep(.1) 205 log.debug(f"Server still not responding after {timeout}") 206 return False 207 208 def _rmf(self, path): 209 if os.path.exists(path): 210 return os.remove(path) 211 212 def _mkpath(self, path): 213 if not os.path.exists(path): 214 return os.makedirs(path) 215 216 def _write_config(self): 217 domain1 = self.env.domain1 218 domain1brotli = self.env.domain1brotli 219 creds1 = self.env.get_credentials(domain1) 220 domain2 = self.env.domain2 221 creds2 = self.env.get_credentials(domain2) 222 proxy_domain = self.env.proxy_domain 223 proxy_creds = self.env.get_credentials(proxy_domain) 224 self._mkpath(self._conf_dir) 225 self._mkpath(self._logs_dir) 226 self._mkpath(self._tmp_dir) 227 self._mkpath(os.path.join(self._docs_dir, 'two')) 228 with open(os.path.join(self._docs_dir, 'data.json'), 'w') as fd: 229 data = { 230 'server': f'{domain1}', 231 } 232 fd.write(JSONEncoder().encode(data)) 233 with open(os.path.join(self._docs_dir, 'two/data.json'), 'w') as fd: 234 data = { 235 'server': f'{domain2}', 236 } 237 fd.write(JSONEncoder().encode(data)) 238 if self._proxy_auth_basic: 239 with open(self._basic_passwords, 'w') as fd: 240 fd.write('proxy:$apr1$FQfeInbs$WQZbODJlVg60j0ogEIlTW/\n') 241 if self._auth_digest: 242 with open(self._digest_passwords, 'w') as fd: 243 fd.write('test:restricted area:57123e269fd73d71ae0656594e938e2f\n') 244 self._mkpath(os.path.join(self.docs_dir, 'restricted/digest')) 245 with open(os.path.join(self.docs_dir, 'restricted/digest/data.json'), 'w') as fd: 246 fd.write('{"area":"digest"}\n') 247 with open(self._conf_file, 'w') as fd: 248 for m in self.MODULES: 249 if os.path.exists(os.path.join(self._mods_dir, f'mod_{m}.so')): 250 fd.write(f'LoadModule {m}_module "{self._mods_dir}/mod_{m}.so"\n') 251 if Httpd.MOD_CURLTEST is not None: 252 fd.write(f'LoadModule curltest_module \"{Httpd.MOD_CURLTEST}\"\n') 253 conf = [ # base server config 254 f'ServerRoot "{self._apache_dir}"', 255 f'DefaultRuntimeDir logs', 256 f'PidFile httpd.pid', 257 f'ErrorLog {self._error_log}', 258 f'LogLevel {self._get_log_level()}', 259 f'StartServers 4', 260 f'ReadBufferSize 16000', 261 f'H2MinWorkers 16', 262 f'H2MaxWorkers 256', 263 f'Listen {self.env.http_port}', 264 f'Listen {self.env.https_port}', 265 f'Listen {self.env.proxy_port}', 266 f'Listen {self.env.proxys_port}', 267 f'TypesConfig "{self._conf_dir}/mime.types', 268 f'SSLSessionCache "shmcb:ssl_gcache_data(32000)"', 269 ] 270 if 'base' in self._extra_configs: 271 conf.extend(self._extra_configs['base']) 272 conf.extend([ # plain http host for domain1 273 f'<VirtualHost *:{self.env.http_port}>', 274 f' ServerName {domain1}', 275 f' ServerAlias localhost', 276 f' DocumentRoot "{self._docs_dir}"', 277 f' Protocols h2c http/1.1', 278 f' H2Direct on', 279 ]) 280 conf.extend(self._curltest_conf(domain1)) 281 conf.extend([ 282 f'</VirtualHost>', 283 f'', 284 ]) 285 conf.extend([ # https host for domain1, h1 + h2 286 f'<VirtualHost *:{self.env.https_port}>', 287 f' ServerName {domain1}', 288 f' ServerAlias localhost', 289 f' Protocols h2 http/1.1', 290 f' SSLEngine on', 291 f' SSLCertificateFile {creds1.cert_file}', 292 f' SSLCertificateKeyFile {creds1.pkey_file}', 293 f' DocumentRoot "{self._docs_dir}"', 294 ]) 295 conf.extend(self._curltest_conf(domain1)) 296 if domain1 in self._extra_configs: 297 conf.extend(self._extra_configs[domain1]) 298 conf.extend([ 299 f'</VirtualHost>', 300 f'', 301 ]) 302 # Alternate to domain1 with BROTLI compression 303 conf.extend([ # https host for domain1, h1 + h2 304 f'<VirtualHost *:{self.env.https_port}>', 305 f' ServerName {domain1brotli}', 306 f' Protocols h2 http/1.1', 307 f' SSLEngine on', 308 f' SSLCertificateFile {creds1.cert_file}', 309 f' SSLCertificateKeyFile {creds1.pkey_file}', 310 f' DocumentRoot "{self._docs_dir}"', 311 f' SetOutputFilter BROTLI_COMPRESS', 312 ]) 313 conf.extend(self._curltest_conf(domain1)) 314 if domain1 in self._extra_configs: 315 conf.extend(self._extra_configs[domain1]) 316 conf.extend([ 317 f'</VirtualHost>', 318 f'', 319 ]) 320 conf.extend([ # plain http host for domain2 321 f'<VirtualHost *:{self.env.http_port}>', 322 f' ServerName {domain2}', 323 f' ServerAlias localhost', 324 f' DocumentRoot "{self._docs_dir}"', 325 f' Protocols h2c http/1.1', 326 ]) 327 conf.extend(self._curltest_conf(domain2)) 328 conf.extend([ 329 f'</VirtualHost>', 330 f'', 331 ]) 332 conf.extend([ # https host for domain2, no h2 333 f'<VirtualHost *:{self.env.https_port}>', 334 f' ServerName {domain2}', 335 f' Protocols http/1.1', 336 f' SSLEngine on', 337 f' SSLCertificateFile {creds2.cert_file}', 338 f' SSLCertificateKeyFile {creds2.pkey_file}', 339 f' DocumentRoot "{self._docs_dir}/two"', 340 ]) 341 conf.extend(self._curltest_conf(domain2)) 342 if domain2 in self._extra_configs: 343 conf.extend(self._extra_configs[domain2]) 344 conf.extend([ 345 f'</VirtualHost>', 346 f'', 347 ]) 348 conf.extend([ # http forward proxy 349 f'<VirtualHost *:{self.env.proxy_port}>', 350 f' ServerName {proxy_domain}', 351 f' Protocols h2c http/1.1', 352 f' ProxyRequests On', 353 f' H2ProxyRequests On', 354 f' ProxyVia On', 355 f' AllowCONNECT {self.env.http_port} {self.env.https_port}', 356 ]) 357 conf.extend(self._get_proxy_conf()) 358 conf.extend([ 359 f'</VirtualHost>', 360 f'', 361 ]) 362 conf.extend([ # https forward proxy 363 f'<VirtualHost *:{self.env.proxys_port}>', 364 f' ServerName {proxy_domain}', 365 f' Protocols h2 http/1.1', 366 f' SSLEngine on', 367 f' SSLCertificateFile {proxy_creds.cert_file}', 368 f' SSLCertificateKeyFile {proxy_creds.pkey_file}', 369 f' ProxyRequests On', 370 f' H2ProxyRequests On', 371 f' ProxyVia On', 372 f' AllowCONNECT {self.env.http_port} {self.env.https_port}', 373 ]) 374 conf.extend(self._get_proxy_conf()) 375 conf.extend([ 376 f'</VirtualHost>', 377 f'', 378 ]) 379 380 fd.write("\n".join(conf)) 381 with open(os.path.join(self._conf_dir, 'mime.types'), 'w') as fd: 382 fd.write("\n".join([ 383 'text/html html', 384 'application/json json', 385 '' 386 ])) 387 388 def _get_proxy_conf(self): 389 if self._proxy_auth_basic: 390 return [ 391 f' <Proxy "*">', 392 f' AuthType Basic', 393 f' AuthName "Restricted Proxy"', 394 f' AuthBasicProvider file', 395 f' AuthUserFile "{self._basic_passwords}"', 396 f' Require user proxy', 397 f' </Proxy>', 398 ] 399 else: 400 return [ 401 f' <Proxy "*">', 402 f' Require ip 127.0.0.1', 403 f' </Proxy>', 404 ] 405 406 def _get_log_level(self): 407 if self.env.verbose > 3: 408 return 'trace2' 409 if self.env.verbose > 2: 410 return 'trace1' 411 if self.env.verbose > 1: 412 return 'debug' 413 return 'info' 414 415 def _curltest_conf(self, servername) -> List[str]: 416 lines = [] 417 if Httpd.MOD_CURLTEST is not None: 418 lines.extend([ 419 f' Redirect 302 /data.json.302 /data.json', 420 f' Redirect 301 /curltest/echo301 /curltest/echo', 421 f' Redirect 302 /curltest/echo302 /curltest/echo', 422 f' Redirect 303 /curltest/echo303 /curltest/echo', 423 f' Redirect 307 /curltest/echo307 /curltest/echo', 424 f' <Location /curltest/sslinfo>', 425 f' SSLOptions StdEnvVars', 426 f' SetHandler curltest-sslinfo', 427 f' </Location>', 428 f' <Location /curltest/echo>', 429 f' SetHandler curltest-echo', 430 f' </Location>', 431 f' <Location /curltest/put>', 432 f' SetHandler curltest-put', 433 f' </Location>', 434 f' <Location /curltest/tweak>', 435 f' SetHandler curltest-tweak', 436 f' </Location>', 437 f' Redirect 302 /tweak /curltest/tweak', 438 f' <Location /curltest/1_1>', 439 f' SetHandler curltest-1_1-required', 440 f' </Location>', 441 f' <Location /curltest/shutdown_unclean>', 442 f' SetHandler curltest-tweak', 443 f' SetEnv force-response-1.0 1', 444 f' </Location>', 445 f' SetEnvIf Request_URI "/shutdown_unclean" ssl-unclean=1', 446 ]) 447 if self._auth_digest: 448 lines.extend([ 449 f' <Directory {self.docs_dir}/restricted/digest>', 450 f' AuthType Digest', 451 f' AuthName "restricted area"', 452 f' AuthDigestDomain "https://{servername}"', 453 f' AuthBasicProvider file', 454 f' AuthUserFile "{self._digest_passwords}"', 455 f' Require valid-user', 456 f' </Directory>', 457 458 ]) 459 return lines 460 461 def _init_curltest(self): 462 if Httpd.MOD_CURLTEST is not None: 463 return 464 local_dir = os.path.dirname(inspect.getfile(Httpd)) 465 p = subprocess.run([self.env.apxs, '-c', 'mod_curltest.c'], 466 capture_output=True, 467 cwd=os.path.join(local_dir, 'mod_curltest')) 468 rv = p.returncode 469 if rv != 0: 470 log.error(f"compiling mod_curltest failed: {p.stderr}") 471 raise Exception(f"compiling mod_curltest failed: {p.stderr}") 472 Httpd.MOD_CURLTEST = os.path.join( 473 local_dir, 'mod_curltest/.libs/mod_curltest.so') 474
