1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import inspect 28import logging 29import os 30import subprocess 31from datetime import timedelta, datetime 32from json import JSONEncoder 33import time 34from typing import List, Union, Optional 35 36from .curl import CurlClient, ExecResult 37from .env import Env 38 39 40log = logging.getLogger(__name__) 41 42 43class Httpd: 44 45 MODULES = [ 46 'log_config', 'logio', 'unixd', 'version', 'watchdog', 47 'authn_core', 'authn_file', 48 'authz_user', 'authz_core', 'authz_host', 49 'auth_basic', 'auth_digest', 50 'alias', 'env', 'filter', 'headers', 'mime', 'setenvif', 51 'socache_shmcb', 52 'rewrite', 'http2', 'ssl', 'proxy', 'proxy_http', 'proxy_connect', 53 'brotli', 54 'mpm_event', 55 ] 56 COMMON_MODULES_DIRS = [ 57 '/usr/lib/apache2/modules', # debian 58 '/usr/libexec/apache2/', # macos 59 ] 60 61 MOD_CURLTEST = None 62 63 def __init__(self, env: Env, proxy_auth: bool = False): 64 self.env = env 65 self._cmd = env.apachectl 66 self._apache_dir = os.path.join(env.gen_dir, 'apache') 67 self._run_dir = os.path.join(self._apache_dir, 'run') 68 self._lock_dir = os.path.join(self._apache_dir, 'locks') 69 self._docs_dir = os.path.join(self._apache_dir, 'docs') 70 self._conf_dir = os.path.join(self._apache_dir, 'conf') 71 self._conf_file = os.path.join(self._conf_dir, 'test.conf') 72 self._logs_dir = os.path.join(self._apache_dir, 'logs') 73 self._error_log = os.path.join(self._logs_dir, 'error_log') 74 self._tmp_dir = os.path.join(self._apache_dir, 'tmp') 75 self._basic_passwords = os.path.join(self._conf_dir, 'basic.passwords') 76 self._digest_passwords = os.path.join(self._conf_dir, 'digest.passwords') 77 self._mods_dir = None 78 self._auth_digest = True 79 self._proxy_auth_basic = proxy_auth 80 self._extra_configs = {} 81 assert env.apxs 82 p = subprocess.run(args=[env.apxs, '-q', 'libexecdir'], 83 capture_output=True, text=True) 84 if p.returncode != 0: 85 raise Exception(f'{env.apxs} failed to query libexecdir: {p}') 86 self._mods_dir = p.stdout.strip() 87 if self._mods_dir is None: 88 raise Exception(f'apache modules dir cannot be found') 89 if not os.path.exists(self._mods_dir): 90 raise Exception(f'apache modules dir does not exist: {self._mods_dir}') 91 self._process = None 92 self._rmf(self._error_log) 93 self._init_curltest() 94 95 @property 96 def docs_dir(self): 97 return self._docs_dir 98 99 def clear_logs(self): 100 self._rmf(self._error_log) 101 102 def exists(self): 103 return os.path.exists(self._cmd) 104 105 def set_extra_config(self, domain: str, lines: Optional[Union[str, List[str]]]): 106 if lines is None: 107 self._extra_configs.pop(domain, None) 108 else: 109 self._extra_configs[domain] = lines 110 111 def clear_extra_configs(self): 112 self._extra_configs = {} 113 114 def set_proxy_auth(self, active: bool): 115 self._proxy_auth_basic = active 116 117 def _run(self, args, intext=''): 118 env = {} 119 for key, val in os.environ.items(): 120 env[key] = val 121 env['APACHE_RUN_DIR'] = self._run_dir 122 env['APACHE_RUN_USER'] = os.environ['USER'] 123 env['APACHE_LOCK_DIR'] = self._lock_dir 124 env['APACHE_CONFDIR'] = self._apache_dir 125 p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, 126 cwd=self.env.gen_dir, 127 input=intext.encode() if intext else None, 128 env=env) 129 start = datetime.now() 130 return ExecResult(args=args, exit_code=p.returncode, 131 stdout=p.stdout.decode().splitlines(), 132 stderr=p.stderr.decode().splitlines(), 133 duration=datetime.now() - start) 134 135 def _apachectl(self, cmd: str): 136 args = [self.env.apachectl, 137 "-d", self._apache_dir, 138 "-f", self._conf_file, 139 "-k", cmd] 140 return self._run(args=args) 141 142 def start(self): 143 if self._process: 144 self.stop() 145 self._write_config() 146 with open(self._error_log, 'a') as fd: 147 fd.write('start of server\n') 148 with open(os.path.join(self._apache_dir, 'xxx'), 'a') as fd: 149 fd.write('start of server\n') 150 r = self._apachectl('start') 151 if r.exit_code != 0: 152 log.error(f'failed to start httpd: {r}') 153 return False 154 return self.wait_live(timeout=timedelta(seconds=5)) 155 156 def stop(self): 157 r = self._apachectl('stop') 158 if r.exit_code == 0: 159 return self.wait_dead(timeout=timedelta(seconds=5)) 160 log.fatal(f'stopping httpd failed: {r}') 161 return r.exit_code == 0 162 163 def restart(self): 164 self.stop() 165 return self.start() 166 167 def reload(self): 168 self._write_config() 169 r = self._apachectl("graceful") 170 if r.exit_code != 0: 171 log.error(f'failed to reload httpd: {r}') 172 return self.wait_live(timeout=timedelta(seconds=5)) 173 174 def wait_dead(self, timeout: timedelta): 175 curl = CurlClient(env=self.env, run_dir=self._tmp_dir) 176 try_until = datetime.now() + timeout 177 while datetime.now() < try_until: 178 r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/') 179 if r.exit_code != 0: 180 return True 181 time.sleep(.1) 182 log.debug(f"Server still responding after {timeout}") 183 return False 184 185 def wait_live(self, timeout: timedelta): 186 curl = CurlClient(env=self.env, run_dir=self._tmp_dir, 187 timeout=timeout.total_seconds()) 188 try_until = datetime.now() + timeout 189 while datetime.now() < try_until: 190 r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/') 191 if r.exit_code == 0: 192 return True 193 time.sleep(.1) 194 log.debug(f"Server still not responding after {timeout}") 195 return False 196 197 def _rmf(self, path): 198 if os.path.exists(path): 199 return os.remove(path) 200 201 def _mkpath(self, path): 202 if not os.path.exists(path): 203 return os.makedirs(path) 204 205 def _write_config(self): 206 domain1 = self.env.domain1 207 domain1brotli = self.env.domain1brotli 208 creds1 = self.env.get_credentials(domain1) 209 domain2 = self.env.domain2 210 creds2 = self.env.get_credentials(domain2) 211 proxy_domain = self.env.proxy_domain 212 proxy_creds = self.env.get_credentials(proxy_domain) 213 self._mkpath(self._conf_dir) 214 self._mkpath(self._logs_dir) 215 self._mkpath(self._tmp_dir) 216 self._mkpath(os.path.join(self._docs_dir, 'two')) 217 with open(os.path.join(self._docs_dir, 'data.json'), 'w') as fd: 218 data = { 219 'server': f'{domain1}', 220 } 221 fd.write(JSONEncoder().encode(data)) 222 with open(os.path.join(self._docs_dir, 'two/data.json'), 'w') as fd: 223 data = { 224 'server': f'{domain2}', 225 } 226 fd.write(JSONEncoder().encode(data)) 227 if self._proxy_auth_basic: 228 with open(self._basic_passwords, 'w') as fd: 229 fd.write('proxy:$apr1$FQfeInbs$WQZbODJlVg60j0ogEIlTW/\n') 230 if self._auth_digest: 231 with open(self._digest_passwords, 'w') as fd: 232 fd.write('test:restricted area:57123e269fd73d71ae0656594e938e2f\n') 233 self._mkpath(os.path.join(self.docs_dir, 'restricted/digest')) 234 with open(os.path.join(self.docs_dir, 'restricted/digest/data.json'), 'w') as fd: 235 fd.write('{"area":"digest"}\n') 236 with open(self._conf_file, 'w') as fd: 237 for m in self.MODULES: 238 if os.path.exists(os.path.join(self._mods_dir, f'mod_{m}.so')): 239 fd.write(f'LoadModule {m}_module "{self._mods_dir}/mod_{m}.so"\n') 240 if Httpd.MOD_CURLTEST is not None: 241 fd.write(f'LoadModule curltest_module \"{Httpd.MOD_CURLTEST}\"\n') 242 conf = [ # base server config 243 f'ServerRoot "{self._apache_dir}"', 244 f'DefaultRuntimeDir logs', 245 f'PidFile httpd.pid', 246 f'ErrorLog {self._error_log}', 247 f'LogLevel {self._get_log_level()}', 248 f'StartServers 4', 249 f'H2MinWorkers 16', 250 f'H2MaxWorkers 256', 251 f'H2Direct on', 252 f'Listen {self.env.http_port}', 253 f'Listen {self.env.https_port}', 254 f'Listen {self.env.proxy_port}', 255 f'Listen {self.env.proxys_port}', 256 f'TypesConfig "{self._conf_dir}/mime.types', 257 f'SSLSessionCache "shmcb:ssl_gcache_data(32000)"', 258 ] 259 if 'base' in self._extra_configs: 260 conf.extend(self._extra_configs['base']) 261 conf.extend([ # plain http host for domain1 262 f'<VirtualHost *:{self.env.http_port}>', 263 f' ServerName {domain1}', 264 f' ServerAlias localhost', 265 f' DocumentRoot "{self._docs_dir}"', 266 f' Protocols h2c http/1.1', 267 ]) 268 conf.extend(self._curltest_conf(domain1)) 269 conf.extend([ 270 f'</VirtualHost>', 271 f'', 272 ]) 273 conf.extend([ # https host for domain1, h1 + h2 274 f'<VirtualHost *:{self.env.https_port}>', 275 f' ServerName {domain1}', 276 f' ServerAlias localhost', 277 f' Protocols h2 http/1.1', 278 f' SSLEngine on', 279 f' SSLCertificateFile {creds1.cert_file}', 280 f' SSLCertificateKeyFile {creds1.pkey_file}', 281 f' DocumentRoot "{self._docs_dir}"', 282 ]) 283 conf.extend(self._curltest_conf(domain1)) 284 if domain1 in self._extra_configs: 285 conf.extend(self._extra_configs[domain1]) 286 conf.extend([ 287 f'</VirtualHost>', 288 f'', 289 ]) 290 # Alternate to domain1 with BROTLI compression 291 conf.extend([ # https host for domain1, h1 + h2 292 f'<VirtualHost *:{self.env.https_port}>', 293 f' ServerName {domain1brotli}', 294 f' Protocols h2 http/1.1', 295 f' SSLEngine on', 296 f' SSLCertificateFile {creds1.cert_file}', 297 f' SSLCertificateKeyFile {creds1.pkey_file}', 298 f' DocumentRoot "{self._docs_dir}"', 299 f' SetOutputFilter BROTLI_COMPRESS', 300 ]) 301 conf.extend(self._curltest_conf(domain1)) 302 if domain1 in self._extra_configs: 303 conf.extend(self._extra_configs[domain1]) 304 conf.extend([ 305 f'</VirtualHost>', 306 f'', 307 ]) 308 conf.extend([ # https host for domain2, no h2 309 f'<VirtualHost *:{self.env.https_port}>', 310 f' ServerName {domain2}', 311 f' Protocols http/1.1', 312 f' SSLEngine on', 313 f' SSLCertificateFile {creds2.cert_file}', 314 f' SSLCertificateKeyFile {creds2.pkey_file}', 315 f' DocumentRoot "{self._docs_dir}/two"', 316 ]) 317 conf.extend(self._curltest_conf(domain2)) 318 if domain2 in self._extra_configs: 319 conf.extend(self._extra_configs[domain2]) 320 conf.extend([ 321 f'</VirtualHost>', 322 f'', 323 ]) 324 conf.extend([ # http forward proxy 325 f'<VirtualHost *:{self.env.proxy_port}>', 326 f' ServerName {proxy_domain}', 327 f' Protocols h2c http/1.1', 328 f' ProxyRequests On', 329 f' H2ProxyRequests On', 330 f' ProxyVia On', 331 f' AllowCONNECT {self.env.http_port} {self.env.https_port}', 332 ]) 333 conf.extend(self._get_proxy_conf()) 334 conf.extend([ 335 f'</VirtualHost>', 336 f'', 337 ]) 338 conf.extend([ # https forward proxy 339 f'<VirtualHost *:{self.env.proxys_port}>', 340 f' ServerName {proxy_domain}', 341 f' Protocols h2 http/1.1', 342 f' SSLEngine on', 343 f' SSLCertificateFile {proxy_creds.cert_file}', 344 f' SSLCertificateKeyFile {proxy_creds.pkey_file}', 345 f' ProxyRequests On', 346 f' H2ProxyRequests On', 347 f' ProxyVia On', 348 f' AllowCONNECT {self.env.http_port} {self.env.https_port}', 349 ]) 350 conf.extend(self._get_proxy_conf()) 351 conf.extend([ 352 f'</VirtualHost>', 353 f'', 354 ]) 355 356 fd.write("\n".join(conf)) 357 with open(os.path.join(self._conf_dir, 'mime.types'), 'w') as fd: 358 fd.write("\n".join([ 359 'text/html html', 360 'application/json json', 361 '' 362 ])) 363 364 def _get_proxy_conf(self): 365 if self._proxy_auth_basic: 366 return [ 367 f' <Proxy "*">', 368 f' AuthType Basic', 369 f' AuthName "Restricted Proxy"', 370 f' AuthBasicProvider file', 371 f' AuthUserFile "{self._basic_passwords}"', 372 f' Require user proxy', 373 f' </Proxy>', 374 ] 375 else: 376 return [ 377 f' <Proxy "*">', 378 f' Require ip 127.0.0.1', 379 f' </Proxy>', 380 ] 381 382 def _get_log_level(self): 383 if self.env.verbose > 3: 384 return 'trace2' 385 if self.env.verbose > 2: 386 return 'trace1' 387 if self.env.verbose > 1: 388 return 'debug' 389 return 'info' 390 391 def _curltest_conf(self, servername) -> List[str]: 392 lines = [] 393 if Httpd.MOD_CURLTEST is not None: 394 lines.extend([ 395 f' Redirect 302 /data.json.302 /data.json', 396 f' Redirect 301 /curltest/echo301 /curltest/echo', 397 f' Redirect 302 /curltest/echo302 /curltest/echo', 398 f' Redirect 303 /curltest/echo303 /curltest/echo', 399 f' Redirect 307 /curltest/echo307 /curltest/echo', 400 f' <Location /curltest/sslinfo>', 401 f' SSLOptions StdEnvVars', 402 f' SetHandler curltest-sslinfo', 403 f' </Location>', 404 f' <Location /curltest/echo>', 405 f' SetHandler curltest-echo', 406 f' </Location>', 407 f' <Location /curltest/put>', 408 f' SetHandler curltest-put', 409 f' </Location>', 410 f' <Location /curltest/tweak>', 411 f' SetHandler curltest-tweak', 412 f' </Location>', 413 f' Redirect 302 /tweak /curltest/tweak', 414 f' <Location /curltest/1_1>', 415 f' SetHandler curltest-1_1-required', 416 f' </Location>', 417 f' <Location /curltest/shutdown_unclean>', 418 f' SetHandler curltest-tweak', 419 f' SetEnv force-response-1.0 1', 420 f' </Location>', 421 f' SetEnvIf Request_URI "/shutdown_unclean" ssl-unclean=1', 422 ]) 423 if self._auth_digest: 424 lines.extend([ 425 f' <Directory {self.docs_dir}/restricted/digest>', 426 f' AuthType Digest', 427 f' AuthName "restricted area"', 428 f' AuthDigestDomain "https://{servername}"', 429 f' AuthBasicProvider file', 430 f' AuthUserFile "{self._digest_passwords}"', 431 f' Require valid-user', 432 f' </Directory>', 433 434 ]) 435 return lines 436 437 def _init_curltest(self): 438 if Httpd.MOD_CURLTEST is not None: 439 return 440 local_dir = os.path.dirname(inspect.getfile(Httpd)) 441 p = subprocess.run([self.env.apxs, '-c', 'mod_curltest.c'], 442 capture_output=True, 443 cwd=os.path.join(local_dir, 'mod_curltest')) 444 rv = p.returncode 445 if rv != 0: 446 log.error(f"compiling mod_curltest failed: {p.stderr}") 447 raise Exception(f"compiling mod_curltest failed: {p.stderr}") 448 Httpd.MOD_CURLTEST = os.path.join( 449 local_dir, 'mod_curltest/.libs/mod_curltest.so') 450
