1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import difflib 28import filecmp 29import logging 30import math 31import os 32import re 33from datetime import timedelta 34import pytest 35 36from testenv import Env, CurlClient, LocalClient 37 38 39log = logging.getLogger(__name__) 40 41 42class TestDownload: 43 44 @pytest.fixture(autouse=True, scope='class') 45 def _class_scope(self, env, httpd, nghttpx): 46 if env.have_h3(): 47 nghttpx.start_if_needed() 48 httpd.clear_extra_configs() 49 httpd.reload() 50 51 @pytest.fixture(autouse=True, scope='class') 52 def _class_scope(self, env, httpd): 53 indir = httpd.docs_dir 54 env.make_data_file(indir=indir, fname="data-10k", fsize=10*1024) 55 env.make_data_file(indir=indir, fname="data-100k", fsize=100*1024) 56 env.make_data_file(indir=indir, fname="data-1m", fsize=1024*1024) 57 env.make_data_file(indir=indir, fname="data-10m", fsize=10*1024*1024) 58 env.make_data_file(indir=indir, fname="data-50m", fsize=50*1024*1024) 59 60 # download 1 file 61 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 62 def test_02_01_download_1(self, env: Env, httpd, nghttpx, repeat, proto): 63 if proto == 'h3' and not env.have_h3(): 64 pytest.skip("h3 not supported") 65 curl = CurlClient(env=env) 66 url = f'https://{env.authority_for(env.domain1, proto)}/data.json' 67 r = curl.http_download(urls=[url], alpn_proto=proto) 68 r.check_response(http_status=200) 69 70 # download 2 files 71 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 72 def test_02_02_download_2(self, env: Env, httpd, nghttpx, repeat, proto): 73 if proto == 'h3' and not env.have_h3(): 74 pytest.skip("h3 not supported") 75 curl = CurlClient(env=env) 76 url = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-1]' 77 r = curl.http_download(urls=[url], alpn_proto=proto) 78 r.check_response(http_status=200, count=2) 79 80 # download 100 files sequentially 81 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 82 def test_02_03_download_sequential(self, env: Env, 83 httpd, nghttpx, repeat, proto): 84 if proto == 'h3' and not env.have_h3(): 85 pytest.skip("h3 not supported") 86 count = 10 87 curl = CurlClient(env=env) 88 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 89 r = curl.http_download(urls=[urln], alpn_proto=proto) 90 r.check_response(http_status=200, count=count, connect_count=1) 91 92 # download 100 files parallel 93 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 94 def test_02_04_download_parallel(self, env: Env, 95 httpd, nghttpx, repeat, proto): 96 if proto == 'h3' and not env.have_h3(): 97 pytest.skip("h3 not supported") 98 count = 10 99 max_parallel = 5 100 curl = CurlClient(env=env) 101 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 102 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 103 '--parallel', '--parallel-max', f'{max_parallel}' 104 ]) 105 r.check_response(http_status=200, count=count) 106 if proto == 'http/1.1': 107 # http/1.1 parallel transfers will open multiple connections 108 assert r.total_connects > 1, r.dump_logs() 109 else: 110 # http2 parallel transfers will use one connection (common limit is 100) 111 assert r.total_connects == 1, r.dump_logs() 112 113 # download 500 files sequential 114 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 115 def test_02_05_download_many_sequential(self, env: Env, 116 httpd, nghttpx, repeat, proto): 117 if proto == 'h3' and not env.have_h3(): 118 pytest.skip("h3 not supported") 119 if proto == 'h3' and env.curl_uses_lib('msh3'): 120 pytest.skip("msh3 shaky here") 121 count = 200 122 curl = CurlClient(env=env) 123 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 124 r = curl.http_download(urls=[urln], alpn_proto=proto) 125 r.check_response(http_status=200, count=count) 126 if proto == 'http/1.1': 127 # http/1.1 parallel transfers will open multiple connections 128 assert r.total_connects > 1, r.dump_logs() 129 else: 130 # http2 parallel transfers will use one connection (common limit is 100) 131 assert r.total_connects == 1, r.dump_logs() 132 133 # download 500 files parallel 134 @pytest.mark.parametrize("proto", ['h2', 'h3']) 135 def test_02_06_download_many_parallel(self, env: Env, 136 httpd, nghttpx, repeat, proto): 137 if proto == 'h3' and not env.have_h3(): 138 pytest.skip("h3 not supported") 139 count = 200 140 max_parallel = 50 141 curl = CurlClient(env=env) 142 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[000-{count-1}]' 143 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 144 '--parallel', '--parallel-max', f'{max_parallel}' 145 ]) 146 r.check_response(http_status=200, count=count, connect_count=1) 147 148 # download files parallel, check connection reuse/multiplex 149 @pytest.mark.parametrize("proto", ['h2', 'h3']) 150 def test_02_07_download_reuse(self, env: Env, 151 httpd, nghttpx, repeat, proto): 152 if proto == 'h3' and not env.have_h3(): 153 pytest.skip("h3 not supported") 154 count = 200 155 curl = CurlClient(env=env) 156 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 157 r = curl.http_download(urls=[urln], alpn_proto=proto, 158 with_stats=True, extra_args=[ 159 '--parallel', '--parallel-max', '200' 160 ]) 161 r.check_response(http_status=200, count=count) 162 # should have used at most 2 connections only (test servers allow 100 req/conn) 163 # it may be just 1 on slow systems where request are answered faster than 164 # curl can exhaust the capacity or if curl runs with address-sanitizer speed 165 assert r.total_connects <= 2, "h2 should use fewer connections here" 166 167 # download files parallel with http/1.1, check connection not reused 168 @pytest.mark.parametrize("proto", ['http/1.1']) 169 def test_02_07b_download_reuse(self, env: Env, 170 httpd, nghttpx, repeat, proto): 171 count = 6 172 curl = CurlClient(env=env) 173 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 174 r = curl.http_download(urls=[urln], alpn_proto=proto, 175 with_stats=True, extra_args=[ 176 '--parallel' 177 ]) 178 r.check_response(count=count, http_status=200) 179 # http/1.1 should have used count connections 180 assert r.total_connects == count, "http/1.1 should use this many connections" 181 182 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 183 def test_02_08_1MB_serial(self, env: Env, 184 httpd, nghttpx, repeat, proto): 185 if proto == 'h3' and not env.have_h3(): 186 pytest.skip("h3 not supported") 187 count = 5 188 urln = f'https://{env.authority_for(env.domain1, proto)}/data-1m?[0-{count-1}]' 189 curl = CurlClient(env=env) 190 r = curl.http_download(urls=[urln], alpn_proto=proto) 191 r.check_response(count=count, http_status=200) 192 193 @pytest.mark.parametrize("proto", ['h2', 'h3']) 194 def test_02_09_1MB_parallel(self, env: Env, 195 httpd, nghttpx, repeat, proto): 196 if proto == 'h3' and not env.have_h3(): 197 pytest.skip("h3 not supported") 198 count = 5 199 urln = f'https://{env.authority_for(env.domain1, proto)}/data-1m?[0-{count-1}]' 200 curl = CurlClient(env=env) 201 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 202 '--parallel' 203 ]) 204 r.check_response(count=count, http_status=200) 205 206 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 207 @pytest.mark.skipif(condition=Env().ci_run, reason="not suitable for CI runs") 208 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 209 def test_02_10_10MB_serial(self, env: Env, 210 httpd, nghttpx, repeat, proto): 211 if proto == 'h3' and not env.have_h3(): 212 pytest.skip("h3 not supported") 213 count = 3 214 urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]' 215 curl = CurlClient(env=env) 216 r = curl.http_download(urls=[urln], alpn_proto=proto) 217 r.check_response(count=count, http_status=200) 218 219 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 220 @pytest.mark.skipif(condition=Env().ci_run, reason="not suitable for CI runs") 221 @pytest.mark.parametrize("proto", ['h2', 'h3']) 222 def test_02_11_10MB_parallel(self, env: Env, 223 httpd, nghttpx, repeat, proto): 224 if proto == 'h3' and not env.have_h3(): 225 pytest.skip("h3 not supported") 226 if proto == 'h3' and env.curl_uses_lib('msh3'): 227 pytest.skip("msh3 stalls here") 228 count = 3 229 urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]' 230 curl = CurlClient(env=env) 231 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 232 '--parallel' 233 ]) 234 r.check_response(count=count, http_status=200) 235 236 @pytest.mark.parametrize("proto", ['h2', 'h3']) 237 def test_02_12_head_serial_https(self, env: Env, 238 httpd, nghttpx, repeat, proto): 239 if proto == 'h3' and not env.have_h3(): 240 pytest.skip("h3 not supported") 241 count = 5 242 urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]' 243 curl = CurlClient(env=env) 244 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 245 '--head' 246 ]) 247 r.check_response(count=count, http_status=200) 248 249 @pytest.mark.parametrize("proto", ['h2']) 250 def test_02_13_head_serial_h2c(self, env: Env, 251 httpd, nghttpx, repeat, proto): 252 if proto == 'h3' and not env.have_h3(): 253 pytest.skip("h3 not supported") 254 count = 5 255 urln = f'http://{env.domain1}:{env.http_port}/data-10m?[0-{count-1}]' 256 curl = CurlClient(env=env) 257 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 258 '--head', '--http2-prior-knowledge', '--fail-early' 259 ]) 260 r.check_response(count=count, http_status=200) 261 262 @pytest.mark.parametrize("proto", ['h2', 'h3']) 263 def test_02_14_not_found(self, env: Env, httpd, nghttpx, repeat, proto): 264 if proto == 'h3' and not env.have_h3(): 265 pytest.skip("h3 not supported") 266 if proto == 'h3' and env.curl_uses_lib('msh3'): 267 pytest.skip("msh3 stalls here") 268 count = 5 269 urln = f'https://{env.authority_for(env.domain1, proto)}/not-found?[0-{count-1}]' 270 curl = CurlClient(env=env) 271 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 272 '--parallel' 273 ]) 274 r.check_stats(count=count, http_status=404, exitcode=0, 275 remote_port=env.port_for(alpn_proto=proto), 276 remote_ip='127.0.0.1') 277 278 @pytest.mark.parametrize("proto", ['h2', 'h3']) 279 def test_02_15_fail_not_found(self, env: Env, httpd, nghttpx, repeat, proto): 280 if proto == 'h3' and not env.have_h3(): 281 pytest.skip("h3 not supported") 282 if proto == 'h3' and env.curl_uses_lib('msh3'): 283 pytest.skip("msh3 stalls here") 284 count = 5 285 urln = f'https://{env.authority_for(env.domain1, proto)}/not-found?[0-{count-1}]' 286 curl = CurlClient(env=env) 287 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 288 '--fail' 289 ]) 290 r.check_stats(count=count, http_status=404, exitcode=22, 291 remote_port=env.port_for(alpn_proto=proto), 292 remote_ip='127.0.0.1') 293 294 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 295 def test_02_20_h2_small_frames(self, env: Env, httpd, repeat): 296 # Test case to reproduce content corruption as observed in 297 # https://github.com/curl/curl/issues/10525 298 # To reliably reproduce, we need an Apache httpd that supports 299 # setting smaller frame sizes. This is not released yet, we 300 # test if it works and back out if not. 301 httpd.set_extra_config(env.domain1, lines=[ 302 'H2MaxDataFrameLen 1024', 303 ]) 304 assert httpd.stop() 305 if not httpd.start(): 306 # no, not supported, bail out 307 httpd.set_extra_config(env.domain1, lines=None) 308 assert httpd.start() 309 pytest.skip('H2MaxDataFrameLen not supported') 310 # ok, make 100 downloads with 2 parallel running and they 311 # are expected to stumble into the issue when using `lib/http2.c` 312 # from curl 7.88.0 313 count = 5 314 urln = f'https://{env.authority_for(env.domain1, "h2")}/data-1m?[0-{count-1}]' 315 curl = CurlClient(env=env) 316 r = curl.http_download(urls=[urln], alpn_proto="h2", extra_args=[ 317 '--parallel', '--parallel-max', '2' 318 ]) 319 r.check_response(count=count, http_status=200) 320 srcfile = os.path.join(httpd.docs_dir, 'data-1m') 321 self.check_downloads(curl, srcfile, count) 322 # restore httpd defaults 323 httpd.set_extra_config(env.domain1, lines=None) 324 assert httpd.stop() 325 assert httpd.start() 326 327 # download via lib client, 1 at a time, pause/resume at different offsets 328 @pytest.mark.parametrize("pause_offset", [0, 10*1024, 100*1023, 640000]) 329 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 330 def test_02_21_lib_serial(self, env: Env, httpd, nghttpx, proto, pause_offset, repeat): 331 if proto == 'h3' and not env.have_h3(): 332 pytest.skip("h3 not supported") 333 count = 2 334 docname = 'data-10m' 335 url = f'https://localhost:{env.https_port}/{docname}' 336 client = LocalClient(name='hx-download', env=env) 337 if not client.exists(): 338 pytest.skip(f'example client not built: {client.name}') 339 r = client.run(args=[ 340 '-n', f'{count}', '-P', f'{pause_offset}', '-V', proto, url 341 ]) 342 r.check_exit_code(0) 343 srcfile = os.path.join(httpd.docs_dir, docname) 344 self.check_downloads(client, srcfile, count) 345 346 # download via lib client, several at a time, pause/resume 347 @pytest.mark.parametrize("pause_offset", [100*1023]) 348 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 349 def test_02_22_lib_parallel_resume(self, env: Env, httpd, nghttpx, proto, pause_offset, repeat): 350 if proto == 'h3' and not env.have_h3(): 351 pytest.skip("h3 not supported") 352 count = 2 353 max_parallel = 5 354 docname = 'data-10m' 355 url = f'https://localhost:{env.https_port}/{docname}' 356 client = LocalClient(name='hx-download', env=env) 357 if not client.exists(): 358 pytest.skip(f'example client not built: {client.name}') 359 r = client.run(args=[ 360 '-n', f'{count}', '-m', f'{max_parallel}', 361 '-P', f'{pause_offset}', '-V', proto, url 362 ]) 363 r.check_exit_code(0) 364 srcfile = os.path.join(httpd.docs_dir, docname) 365 self.check_downloads(client, srcfile, count) 366 367 # download, several at a time, pause and abort paused 368 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 369 def test_02_23a_lib_abort_paused(self, env: Env, httpd, nghttpx, proto, repeat): 370 if proto == 'h3' and not env.have_h3(): 371 pytest.skip("h3 not supported") 372 if proto == 'h3' and env.curl_uses_ossl_quic(): 373 pytest.skip('OpenSSL QUIC fails here') 374 if proto == 'h3' and env.ci_run and env.curl_uses_lib('quiche'): 375 pytest.skip("fails in CI, but works locally for unknown reasons") 376 count = 10 377 max_parallel = 5 378 if proto in ['h2', 'h3']: 379 pause_offset = 64 * 1024 380 else: 381 pause_offset = 12 * 1024 382 docname = 'data-1m' 383 url = f'https://localhost:{env.https_port}/{docname}' 384 client = LocalClient(name='hx-download', env=env) 385 if not client.exists(): 386 pytest.skip(f'example client not built: {client.name}') 387 r = client.run(args=[ 388 '-n', f'{count}', '-m', f'{max_parallel}', '-a', 389 '-P', f'{pause_offset}', '-V', proto, url 390 ]) 391 r.check_exit_code(0) 392 srcfile = os.path.join(httpd.docs_dir, docname) 393 # downloads should be there, but not necessarily complete 394 self.check_downloads(client, srcfile, count, complete=False) 395 396 # download, several at a time, abort after n bytes 397 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 398 def test_02_23b_lib_abort_offset(self, env: Env, httpd, nghttpx, proto, repeat): 399 if proto == 'h3' and not env.have_h3(): 400 pytest.skip("h3 not supported") 401 if proto == 'h3' and env.curl_uses_ossl_quic(): 402 pytest.skip('OpenSSL QUIC fails here') 403 if proto == 'h3' and env.ci_run and env.curl_uses_lib('quiche'): 404 pytest.skip("fails in CI, but works locally for unknown reasons") 405 count = 10 406 max_parallel = 5 407 if proto in ['h2', 'h3']: 408 abort_offset = 64 * 1024 409 else: 410 abort_offset = 12 * 1024 411 docname = 'data-1m' 412 url = f'https://localhost:{env.https_port}/{docname}' 413 client = LocalClient(name='hx-download', env=env) 414 if not client.exists(): 415 pytest.skip(f'example client not built: {client.name}') 416 r = client.run(args=[ 417 '-n', f'{count}', '-m', f'{max_parallel}', '-a', 418 '-A', f'{abort_offset}', '-V', proto, url 419 ]) 420 r.check_exit_code(0) 421 srcfile = os.path.join(httpd.docs_dir, docname) 422 # downloads should be there, but not necessarily complete 423 self.check_downloads(client, srcfile, count, complete=False) 424 425 # download, several at a time, abort after n bytes 426 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 427 def test_02_23c_lib_fail_offset(self, env: Env, httpd, nghttpx, proto, repeat): 428 if proto == 'h3' and not env.have_h3(): 429 pytest.skip("h3 not supported") 430 if proto == 'h3' and env.curl_uses_ossl_quic(): 431 pytest.skip('OpenSSL QUIC fails here') 432 if proto == 'h3' and env.ci_run and env.curl_uses_lib('quiche'): 433 pytest.skip("fails in CI, but works locally for unknown reasons") 434 count = 10 435 max_parallel = 5 436 if proto in ['h2', 'h3']: 437 fail_offset = 64 * 1024 438 else: 439 fail_offset = 12 * 1024 440 docname = 'data-1m' 441 url = f'https://localhost:{env.https_port}/{docname}' 442 client = LocalClient(name='hx-download', env=env) 443 if not client.exists(): 444 pytest.skip(f'example client not built: {client.name}') 445 r = client.run(args=[ 446 '-n', f'{count}', '-m', f'{max_parallel}', '-a', 447 '-F', f'{fail_offset}', '-V', proto, url 448 ]) 449 r.check_exit_code(0) 450 srcfile = os.path.join(httpd.docs_dir, docname) 451 # downloads should be there, but not necessarily complete 452 self.check_downloads(client, srcfile, count, complete=False) 453 454 # speed limited download 455 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 456 def test_02_24_speed_limit(self, env: Env, httpd, nghttpx, proto, repeat): 457 if proto == 'h3' and not env.have_h3(): 458 pytest.skip("h3 not supported") 459 count = 1 460 url = f'https://{env.authority_for(env.domain1, proto)}/data-1m' 461 curl = CurlClient(env=env) 462 speed_limit = 384 * 1024 463 min_duration = math.floor((1024 * 1024)/speed_limit) 464 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[ 465 '--limit-rate', f'{speed_limit}' 466 ]) 467 r.check_response(count=count, http_status=200) 468 assert r.duration > timedelta(seconds=min_duration), \ 469 f'rate limited transfer should take more than {min_duration}s, '\ 470 f'not {r.duration}' 471 472 # make extreme parallel h2 upgrades, check invalid conn reuse 473 # before protocol switch has happened 474 def test_02_25_h2_upgrade_x(self, env: Env, httpd, repeat): 475 url = f'http://localhost:{env.http_port}/data-100k' 476 client = LocalClient(name='h2-upgrade-extreme', env=env, timeout=15) 477 if not client.exists(): 478 pytest.skip(f'example client not built: {client.name}') 479 r = client.run(args=[url]) 480 assert r.exit_code == 0, f'{client.dump_logs()}' 481 482 # Special client that tests TLS session reuse in parallel transfers 483 # TODO: just uses a single connection for h2/h3. Not sure how to prevent that 484 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 485 def test_02_26_session_shared_reuse(self, env: Env, proto, httpd, nghttpx, repeat): 486 url = f'https://{env.authority_for(env.domain1, proto)}/data-100k' 487 client = LocalClient(name='tls-session-reuse', env=env) 488 if not client.exists(): 489 pytest.skip(f'example client not built: {client.name}') 490 r = client.run(args=[proto, url]) 491 r.check_exit_code(0) 492 493 # test on paused transfers, based on issue #11982 494 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 495 def test_02_27a_paused_no_cl(self, env: Env, httpd, nghttpx, proto, repeat): 496 url = f'https://{env.authority_for(env.domain1, proto)}' \ 497 '/curltest/tweak/?&chunks=6&chunk_size=8000' 498 client = LocalClient(env=env, name='h2-pausing') 499 r = client.run(args=['-V', proto, url]) 500 r.check_exit_code(0) 501 502 # test on paused transfers, based on issue #11982 503 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 504 def test_02_27b_paused_no_cl(self, env: Env, httpd, nghttpx, proto, repeat): 505 url = f'https://{env.authority_for(env.domain1, proto)}' \ 506 '/curltest/tweak/?error=502' 507 client = LocalClient(env=env, name='h2-pausing') 508 r = client.run(args=['-V', proto, url]) 509 r.check_exit_code(0) 510 511 # test on paused transfers, based on issue #11982 512 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 513 def test_02_27c_paused_no_cl(self, env: Env, httpd, nghttpx, proto, repeat): 514 url = f'https://{env.authority_for(env.domain1, proto)}' \ 515 '/curltest/tweak/?status=200&chunks=1&chunk_size=100' 516 client = LocalClient(env=env, name='h2-pausing') 517 r = client.run(args=['-V', proto, url]) 518 r.check_exit_code(0) 519 520 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 521 def test_02_28_get_compressed(self, env: Env, httpd, nghttpx, repeat, proto): 522 if proto == 'h3' and not env.have_h3(): 523 pytest.skip("h3 not supported") 524 count = 1 525 urln = f'https://{env.authority_for(env.domain1brotli, proto)}/data-100k?[0-{count-1}]' 526 curl = CurlClient(env=env) 527 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 528 '--compressed' 529 ]) 530 r.check_exit_code(code=0) 531 r.check_response(count=count, http_status=200) 532 533 def check_downloads(self, client, srcfile: str, count: int, 534 complete: bool = True): 535 for i in range(count): 536 dfile = client.download_file(i) 537 assert os.path.exists(dfile) 538 if complete and not filecmp.cmp(srcfile, dfile, shallow=False): 539 diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(), 540 b=open(dfile).readlines(), 541 fromfile=srcfile, 542 tofile=dfile, 543 n=1)) 544 assert False, f'download {dfile} differs:\n{diff}' 545 546 # download via lib client, 1 at a time, pause/resume at different offsets 547 @pytest.mark.parametrize("pause_offset", [0, 10*1024, 100*1023, 640000]) 548 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 549 def test_02_29_h2_lib_serial(self, env: Env, httpd, nghttpx, proto, pause_offset, repeat): 550 count = 2 551 docname = 'data-10m' 552 url = f'https://localhost:{env.https_port}/{docname}' 553 client = LocalClient(name='hx-download', env=env) 554 if not client.exists(): 555 pytest.skip(f'example client not built: {client.name}') 556 r = client.run(args=[ 557 '-n', f'{count}', '-P', f'{pause_offset}', '-V', proto, url 558 ]) 559 r.check_exit_code(0) 560 srcfile = os.path.join(httpd.docs_dir, docname) 561 self.check_downloads(client, srcfile, count) 562 563 # download parallel with prior knowledge 564 def test_02_30_parallel_prior_knowledge(self, env: Env, httpd): 565 count = 3 566 curl = CurlClient(env=env) 567 urln = f'http://{env.domain1}:{env.http_port}/data.json?[0-{count-1}]' 568 r = curl.http_download(urls=[urln], extra_args=[ 569 '--parallel', '--http2-prior-knowledge' 570 ]) 571 r.check_response(http_status=200, count=count) 572 assert r.total_connects == 1, r.dump_logs() 573 574 # download parallel with h2 "Upgrade:" 575 def test_02_31_parallel_upgrade(self, env: Env, httpd, nghttpx): 576 count = 3 577 curl = CurlClient(env=env) 578 urln = f'http://{env.domain1}:{env.http_port}/data.json?[0-{count-1}]' 579 r = curl.http_download(urls=[urln], extra_args=[ 580 '--parallel', '--http2' 581 ]) 582 r.check_response(http_status=200, count=count) 583 # we see 3 connections, because Apache only every serves a single 584 # request via Upgrade: and then closed the connection. 585 assert r.total_connects == 3, r.dump_logs() 586 587 # nghttpx is the only server we have that supports TLS early data 588 @pytest.mark.skipif(condition=not Env.have_nghttpx(), reason="no nghttpx") 589 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 590 def test_02_32_earlydata(self, env: Env, httpd, nghttpx, proto): 591 if not env.curl_uses_lib('gnutls'): 592 pytest.skip('TLS earlydata only implemented in GnuTLS') 593 if proto == 'h3' and not env.have_h3(): 594 pytest.skip("h3 not supported") 595 count = 2 596 docname = 'data-10k' 597 # we want this test to always connect to nghttpx, since it is 598 # the only server we have that supports TLS earlydata 599 port = env.port_for(proto) 600 if proto != 'h3': 601 port = env.nghttpx_https_port 602 url = f'https://{env.domain1}:{port}/{docname}' 603 client = LocalClient(name='hx-download', env=env) 604 if not client.exists(): 605 pytest.skip(f'example client not built: {client.name}') 606 r = client.run(args=[ 607 '-n', f'{count}', 608 '-e', # use TLS earlydata 609 '-f', # forbid reuse of connections 610 '-r', f'{env.domain1}:{port}:127.0.0.1', 611 '-V', proto, url 612 ]) 613 r.check_exit_code(0) 614 srcfile = os.path.join(httpd.docs_dir, docname) 615 self.check_downloads(client, srcfile, count) 616 # check that TLS earlydata worked as expected 617 earlydata = {} 618 reused_session = False 619 for line in r.trace_lines: 620 m = re.match(r'^\[t-(\d+)] EarlyData: (-?\d+)', line) 621 if m: 622 earlydata[int(m.group(1))] = int(m.group(2)) 623 continue 624 m = re.match(r'\[1-1] \* SSL reusing session.*', line) 625 if m: 626 reused_session = True 627 assert reused_session, 'session was not reused for 2nd transfer' 628 assert earlydata[0] == 0, f'{earlydata}' 629 if proto == 'http/1.1': 630 assert earlydata[1] == 69, f'{earlydata}' 631 elif proto == 'h2': 632 assert earlydata[1] == 107, f'{earlydata}' 633 elif proto == 'h3': 634 # not implemented 635 assert earlydata[1] == 0, f'{earlydata}' 636 637 @pytest.mark.parametrize("proto", ['http/1.1', 'h2']) 638 @pytest.mark.parametrize("max_host_conns", [0, 1, 5]) 639 def test_02_33_max_host_conns(self, env: Env, httpd, nghttpx, proto, max_host_conns): 640 if proto == 'h3' and not env.have_h3(): 641 pytest.skip("h3 not supported") 642 count = 100 643 max_parallel = 100 644 docname = 'data-10k' 645 port = env.port_for(proto) 646 url = f'https://{env.domain1}:{port}/{docname}' 647 client = LocalClient(name='hx-download', env=env) 648 if not client.exists(): 649 pytest.skip(f'example client not built: {client.name}') 650 r = client.run(args=[ 651 '-n', f'{count}', 652 '-m', f'{max_parallel}', 653 '-x', # always use a fresh connection 654 '-M', str(max_host_conns), # limit conns per host 655 '-r', f'{env.domain1}:{port}:127.0.0.1', 656 '-V', proto, url 657 ]) 658 r.check_exit_code(0) 659 srcfile = os.path.join(httpd.docs_dir, docname) 660 self.check_downloads(client, srcfile, count) 661
