1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import difflib 28import filecmp 29import logging 30import os 31import re 32import pytest 33from typing import List 34 35from testenv import Env, CurlClient, LocalClient 36 37 38log = logging.getLogger(__name__) 39 40 41class TestUpload: 42 43 @pytest.fixture(autouse=True, scope='class') 44 def _class_scope(self, env, httpd, nghttpx): 45 if env.have_h3(): 46 nghttpx.start_if_needed() 47 env.make_data_file(indir=env.gen_dir, fname="data-10k", fsize=10*1024) 48 env.make_data_file(indir=env.gen_dir, fname="data-63k", fsize=63*1024) 49 env.make_data_file(indir=env.gen_dir, fname="data-64k", fsize=64*1024) 50 env.make_data_file(indir=env.gen_dir, fname="data-100k", fsize=100*1024) 51 env.make_data_file(indir=env.gen_dir, fname="data-1m+", fsize=(1024*1024)+1) 52 env.make_data_file(indir=env.gen_dir, fname="data-10m", fsize=10*1024*1024) 53 httpd.clear_extra_configs() 54 httpd.reload() 55 56 # upload small data, check that this is what was echoed 57 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 58 def test_07_01_upload_1_small(self, env: Env, httpd, nghttpx, repeat, proto): 59 if proto == 'h3' and not env.have_h3(): 60 pytest.skip("h3 not supported") 61 if proto == 'h3' and env.curl_uses_lib('msh3'): 62 pytest.skip("msh3 fails here") 63 data = '0123456789' 64 curl = CurlClient(env=env) 65 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 66 r = curl.http_upload(urls=[url], data=data, alpn_proto=proto) 67 r.check_stats(count=1, http_status=200, exitcode=0) 68 respdata = open(curl.response_file(0)).readlines() 69 assert respdata == [data] 70 71 # upload large data, check that this is what was echoed 72 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 73 def test_07_02_upload_1_large(self, env: Env, httpd, nghttpx, repeat, proto): 74 if proto == 'h3' and not env.have_h3(): 75 pytest.skip("h3 not supported") 76 if proto == 'h3' and env.curl_uses_lib('msh3'): 77 pytest.skip("msh3 fails here") 78 fdata = os.path.join(env.gen_dir, 'data-100k') 79 curl = CurlClient(env=env) 80 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 81 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto) 82 r.check_stats(count=1, http_status=200, exitcode=0) 83 indata = open(fdata).readlines() 84 respdata = open(curl.response_file(0)).readlines() 85 assert respdata == indata 86 87 # upload data sequentially, check that they were echoed 88 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 89 def test_07_10_upload_sequential(self, env: Env, httpd, nghttpx, repeat, proto): 90 if proto == 'h3' and not env.have_h3(): 91 pytest.skip("h3 not supported") 92 if proto == 'h3' and env.curl_uses_lib('msh3'): 93 pytest.skip("msh3 stalls here") 94 count = 20 95 data = '0123456789' 96 curl = CurlClient(env=env) 97 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]' 98 r = curl.http_upload(urls=[url], data=data, alpn_proto=proto) 99 r.check_stats(count=count, http_status=200, exitcode=0) 100 for i in range(count): 101 respdata = open(curl.response_file(i)).readlines() 102 assert respdata == [data] 103 104 # upload data parallel, check that they were echoed 105 @pytest.mark.parametrize("proto", ['h2', 'h3']) 106 def test_07_11_upload_parallel(self, env: Env, httpd, nghttpx, repeat, proto): 107 if proto == 'h3' and not env.have_h3(): 108 pytest.skip("h3 not supported") 109 if proto == 'h3' and env.curl_uses_lib('msh3'): 110 pytest.skip("msh3 stalls here") 111 # limit since we use a separate connection in h1 112 count = 20 113 data = '0123456789' 114 curl = CurlClient(env=env) 115 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]' 116 r = curl.http_upload(urls=[url], data=data, alpn_proto=proto, 117 extra_args=['--parallel']) 118 r.check_stats(count=count, http_status=200, exitcode=0) 119 for i in range(count): 120 respdata = open(curl.response_file(i)).readlines() 121 assert respdata == [data] 122 123 # upload large data sequentially, check that this is what was echoed 124 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 125 def test_07_12_upload_seq_large(self, env: Env, httpd, nghttpx, repeat, proto): 126 if proto == 'h3' and not env.have_h3(): 127 pytest.skip("h3 not supported") 128 if proto == 'h3' and env.curl_uses_lib('msh3'): 129 pytest.skip("msh3 stalls here") 130 fdata = os.path.join(env.gen_dir, 'data-100k') 131 count = 10 132 curl = CurlClient(env=env) 133 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]' 134 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto) 135 r.check_response(count=count, http_status=200) 136 indata = open(fdata).readlines() 137 r.check_stats(count=count, http_status=200, exitcode=0) 138 for i in range(count): 139 respdata = open(curl.response_file(i)).readlines() 140 assert respdata == indata 141 142 # upload very large data sequentially, check that this is what was echoed 143 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 144 def test_07_13_upload_seq_large(self, env: Env, httpd, nghttpx, repeat, proto): 145 if proto == 'h3' and not env.have_h3(): 146 pytest.skip("h3 not supported") 147 if proto == 'h3' and env.curl_uses_lib('msh3'): 148 pytest.skip("msh3 stalls here") 149 fdata = os.path.join(env.gen_dir, 'data-10m') 150 count = 2 151 curl = CurlClient(env=env) 152 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]' 153 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto) 154 r.check_stats(count=count, http_status=200, exitcode=0) 155 indata = open(fdata).readlines() 156 for i in range(count): 157 respdata = open(curl.response_file(i)).readlines() 158 assert respdata == indata 159 160 # upload from stdin, issue #14870 161 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 162 @pytest.mark.parametrize("indata", [ 163 '', '1', '123\n456andsomething\n\n' 164 ]) 165 def test_07_14_upload_stdin(self, env: Env, httpd, nghttpx, proto, indata): 166 if proto == 'h3' and not env.have_h3(): 167 pytest.skip("h3 not supported") 168 if proto == 'h3' and env.curl_uses_lib('msh3'): 169 pytest.skip("msh3 stalls here") 170 count = 1 171 curl = CurlClient(env=env) 172 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?id=[0-{count-1}]' 173 r = curl.http_put(urls=[url], data=indata, alpn_proto=proto) 174 r.check_stats(count=count, http_status=200, exitcode=0) 175 for i in range(count): 176 respdata = open(curl.response_file(i)).readlines() 177 assert respdata == [f'{len(indata)}'] 178 179 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 180 def test_07_15_hx_put(self, env: Env, httpd, nghttpx, proto): 181 if proto == 'h3' and not env.have_h3(): 182 pytest.skip("h3 not supported") 183 count = 2 184 upload_size = 128*1024 185 url = f'https://localhost:{env.https_port}/curltest/put?id=[0-{count-1}]' 186 client = LocalClient(name='hx-upload', env=env) 187 if not client.exists(): 188 pytest.skip(f'example client not built: {client.name}') 189 r = client.run(args=[ 190 '-n', f'{count}', '-S', f'{upload_size}', '-V', proto, url 191 ]) 192 r.check_exit_code(0) 193 self.check_downloads(client, [f"{upload_size}"], count) 194 195 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 196 def test_07_16_hx_put_reuse(self, env: Env, httpd, nghttpx, proto): 197 if proto == 'h3' and not env.have_h3(): 198 pytest.skip("h3 not supported") 199 count = 2 200 upload_size = 128*1024 201 url = f'https://localhost:{env.https_port}/curltest/put?id=[0-{count-1}]' 202 client = LocalClient(name='hx-upload', env=env) 203 if not client.exists(): 204 pytest.skip(f'example client not built: {client.name}') 205 r = client.run(args=[ 206 '-n', f'{count}', '-S', f'{upload_size}', '-R', '-V', proto, url 207 ]) 208 r.check_exit_code(0) 209 self.check_downloads(client, [f"{upload_size}"], count) 210 211 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 212 def test_07_17_hx_post_reuse(self, env: Env, httpd, nghttpx, proto): 213 if proto == 'h3' and not env.have_h3(): 214 pytest.skip("h3 not supported") 215 count = 2 216 upload_size = 128*1024 217 url = f'https://localhost:{env.https_port}/curltest/echo?id=[0-{count-1}]' 218 client = LocalClient(name='hx-upload', env=env) 219 if not client.exists(): 220 pytest.skip(f'example client not built: {client.name}') 221 r = client.run(args=[ 222 '-n', f'{count}', '-M', 'POST', '-S', f'{upload_size}', '-R', '-V', proto, url 223 ]) 224 r.check_exit_code(0) 225 self.check_downloads(client, ["x" * upload_size], count) 226 227 # upload data parallel, check that they were echoed 228 @pytest.mark.parametrize("proto", ['h2', 'h3']) 229 def test_07_20_upload_parallel(self, env: Env, httpd, nghttpx, repeat, proto): 230 if proto == 'h3' and not env.have_h3(): 231 pytest.skip("h3 not supported") 232 if proto == 'h3' and env.curl_uses_lib('msh3'): 233 pytest.skip("msh3 stalls here") 234 # limit since we use a separate connection in h1 235 count = 10 236 data = '0123456789' 237 curl = CurlClient(env=env) 238 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]' 239 r = curl.http_upload(urls=[url], data=data, alpn_proto=proto, 240 extra_args=['--parallel']) 241 r.check_stats(count=count, http_status=200, exitcode=0) 242 for i in range(count): 243 respdata = open(curl.response_file(i)).readlines() 244 assert respdata == [data] 245 246 # upload large data parallel, check that this is what was echoed 247 @pytest.mark.parametrize("proto", ['h2', 'h3']) 248 def test_07_21_upload_parallel_large(self, env: Env, httpd, nghttpx, repeat, proto): 249 if proto == 'h3' and not env.have_h3(): 250 pytest.skip("h3 not supported") 251 if proto == 'h3' and env.curl_uses_lib('msh3'): 252 pytest.skip("msh3 stalls here") 253 fdata = os.path.join(env.gen_dir, 'data-100k') 254 # limit since we use a separate connection in h1 255 count = 10 256 curl = CurlClient(env=env) 257 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-{count-1}]' 258 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto, 259 extra_args=['--parallel']) 260 r.check_response(count=count, http_status=200) 261 self.check_download(count, fdata, curl) 262 263 # upload large data parallel to a URL that denies uploads 264 @pytest.mark.parametrize("proto", ['h2', 'h3']) 265 def test_07_22_upload_parallel_fail(self, env: Env, httpd, nghttpx, repeat, proto): 266 if proto == 'h3' and not env.have_h3(): 267 pytest.skip("h3 not supported") 268 if proto == 'h3' and env.curl_uses_lib('msh3'): 269 pytest.skip("msh3 stalls here") 270 fdata = os.path.join(env.gen_dir, 'data-10m') 271 count = 20 272 curl = CurlClient(env=env) 273 url = f'https://{env.authority_for(env.domain1, proto)}'\ 274 f'/curltest/tweak?status=400&delay=5ms&chunks=1&body_error=reset&id=[0-{count-1}]' 275 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto, 276 extra_args=['--parallel']) 277 exp_exit = 92 if proto == 'h2' else 95 278 r.check_stats(count=count, exitcode=exp_exit) 279 280 # PUT 100k 281 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 282 def test_07_30_put_100k(self, env: Env, httpd, nghttpx, repeat, proto): 283 if proto == 'h3' and not env.have_h3(): 284 pytest.skip("h3 not supported") 285 if proto == 'h3' and env.curl_uses_lib('msh3'): 286 pytest.skip("msh3 fails here") 287 fdata = os.path.join(env.gen_dir, 'data-100k') 288 count = 1 289 curl = CurlClient(env=env) 290 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?id=[0-{count-1}]' 291 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto, 292 extra_args=['--parallel']) 293 r.check_stats(count=count, http_status=200, exitcode=0) 294 exp_data = [f'{os.path.getsize(fdata)}'] 295 r.check_response(count=count, http_status=200) 296 for i in range(count): 297 respdata = open(curl.response_file(i)).readlines() 298 assert respdata == exp_data 299 300 # PUT 10m 301 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 302 def test_07_31_put_10m(self, env: Env, httpd, nghttpx, repeat, proto): 303 if proto == 'h3' and not env.have_h3(): 304 pytest.skip("h3 not supported") 305 if proto == 'h3' and env.curl_uses_lib('msh3'): 306 pytest.skip("msh3 fails here") 307 fdata = os.path.join(env.gen_dir, 'data-10m') 308 count = 1 309 curl = CurlClient(env=env) 310 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?id=[0-{count-1}]&chunk_delay=2ms' 311 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto, 312 extra_args=['--parallel']) 313 r.check_stats(count=count, http_status=200, exitcode=0) 314 exp_data = [f'{os.path.getsize(fdata)}'] 315 r.check_response(count=count, http_status=200) 316 for i in range(count): 317 respdata = open(curl.response_file(i)).readlines() 318 assert respdata == exp_data 319 320 # issue #10591 321 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 322 def test_07_32_issue_10591(self, env: Env, httpd, nghttpx, repeat, proto): 323 if proto == 'h3' and not env.have_h3(): 324 pytest.skip("h3 not supported") 325 if proto == 'h3' and env.curl_uses_lib('msh3'): 326 pytest.skip("msh3 fails here") 327 fdata = os.path.join(env.gen_dir, 'data-10m') 328 count = 1 329 curl = CurlClient(env=env) 330 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?id=[0-{count-1}]' 331 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto) 332 r.check_stats(count=count, http_status=200, exitcode=0) 333 334 # issue #11157, upload that is 404'ed by server, needs to terminate 335 # correctly and not time out on sending 336 def test_07_33_issue_11157a(self, env: Env, httpd, nghttpx, repeat): 337 proto = 'h2' 338 fdata = os.path.join(env.gen_dir, 'data-10m') 339 # send a POST to our PUT handler which will send immediately a 404 back 340 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put' 341 curl = CurlClient(env=env) 342 r = curl.run_direct(with_stats=True, args=[ 343 '--resolve', f'{env.authority_for(env.domain1, proto)}:127.0.0.1', 344 '--cacert', env.ca.cert_file, 345 '--request', 'POST', 346 '--max-time', '5', '-v', 347 '--url', url, 348 '--form', 'idList=12345678', 349 '--form', 'pos=top', 350 '--form', 'name=mr_test', 351 '--form', f'fileSource=@{fdata};type=application/pdf', 352 ]) 353 assert r.exit_code == 0, f'{r}' 354 r.check_stats(1, 404) 355 356 # issue #11157, send upload that is slowly read in 357 def test_07_33_issue_11157b(self, env: Env, httpd, nghttpx, repeat): 358 proto = 'h2' 359 fdata = os.path.join(env.gen_dir, 'data-10m') 360 # tell our test PUT handler to read the upload more slowly, so 361 # that the send buffering and transfer loop needs to wait 362 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?chunk_delay=2ms' 363 curl = CurlClient(env=env) 364 r = curl.run_direct(with_stats=True, args=[ 365 '--verbose', '--trace-config', 'ids,time', 366 '--resolve', f'{env.authority_for(env.domain1, proto)}:127.0.0.1', 367 '--cacert', env.ca.cert_file, 368 '--request', 'PUT', 369 '--max-time', '10', '-v', 370 '--url', url, 371 '--form', 'idList=12345678', 372 '--form', 'pos=top', 373 '--form', 'name=mr_test', 374 '--form', f'fileSource=@{fdata};type=application/pdf', 375 ]) 376 assert r.exit_code == 0, r.dump_logs() 377 r.check_stats(1, 200) 378 379 def test_07_34_issue_11194(self, env: Env, httpd, nghttpx, repeat): 380 proto = 'h2' 381 # tell our test PUT handler to read the upload more slowly, so 382 # that the send buffering and transfer loop needs to wait 383 fdata = os.path.join(env.gen_dir, 'data-100k') 384 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put' 385 curl = CurlClient(env=env) 386 r = curl.run_direct(with_stats=True, args=[ 387 '--verbose', '--trace-config', 'ids,time', 388 '--resolve', f'{env.authority_for(env.domain1, proto)}:127.0.0.1', 389 '--cacert', env.ca.cert_file, 390 '--request', 'PUT', 391 '--digest', '--user', 'test:test', 392 '--data-binary', f'@{fdata}', 393 '--url', url, 394 ]) 395 assert r.exit_code == 0, r.dump_logs() 396 r.check_stats(1, 200) 397 398 # upload large data on a h1 to h2 upgrade 399 def test_07_35_h1_h2_upgrade_upload(self, env: Env, httpd, nghttpx, repeat): 400 fdata = os.path.join(env.gen_dir, 'data-100k') 401 curl = CurlClient(env=env) 402 url = f'http://{env.domain1}:{env.http_port}/curltest/echo?id=[0-0]' 403 r = curl.http_upload(urls=[url], data=f'@{fdata}', extra_args=[ 404 '--http2' 405 ]) 406 r.check_response(count=1, http_status=200) 407 # apache does not Upgrade on request with a body 408 assert r.stats[0]['http_version'] == '1.1', f'{r}' 409 indata = open(fdata).readlines() 410 respdata = open(curl.response_file(0)).readlines() 411 assert respdata == indata 412 413 # upload to a 301,302,303 response 414 @pytest.mark.parametrize("redir", ['301', '302', '303']) 415 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 416 def test_07_36_upload_30x(self, env: Env, httpd, nghttpx, repeat, redir, proto): 417 if proto == 'h3' and not env.have_h3(): 418 pytest.skip("h3 not supported") 419 if proto == 'h3' and env.curl_uses_lib('msh3'): 420 pytest.skip("msh3 fails here") 421 data = '0123456789' * 10 422 curl = CurlClient(env=env) 423 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo{redir}?id=[0-0]' 424 r = curl.http_upload(urls=[url], data=data, alpn_proto=proto, extra_args=[ 425 '-L', '--trace-config', 'http/2,http/3' 426 ]) 427 r.check_response(count=1, http_status=200) 428 respdata = open(curl.response_file(0)).readlines() 429 assert respdata == [] # was transformed to a GET 430 431 # upload to a 307 response 432 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 433 def test_07_37_upload_307(self, env: Env, httpd, nghttpx, repeat, proto): 434 if proto == 'h3' and not env.have_h3(): 435 pytest.skip("h3 not supported") 436 if proto == 'h3' and env.curl_uses_lib('msh3'): 437 pytest.skip("msh3 fails here") 438 data = '0123456789' * 10 439 curl = CurlClient(env=env) 440 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo307?id=[0-0]' 441 r = curl.http_upload(urls=[url], data=data, alpn_proto=proto, extra_args=[ 442 '-L', '--trace-config', 'http/2,http/3' 443 ]) 444 r.check_response(count=1, http_status=200) 445 respdata = open(curl.response_file(0)).readlines() 446 assert respdata == [data] # was POST again 447 448 # POST form data, yet another code path in transfer 449 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 450 def test_07_38_form_small(self, env: Env, httpd, nghttpx, repeat, proto): 451 if proto == 'h3' and not env.have_h3(): 452 pytest.skip("h3 not supported") 453 if proto == 'h3' and env.curl_uses_lib('msh3'): 454 pytest.skip("msh3 fails here") 455 curl = CurlClient(env=env) 456 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 457 r = curl.http_form(urls=[url], alpn_proto=proto, form={ 458 'name1': 'value1', 459 }) 460 r.check_stats(count=1, http_status=200, exitcode=0) 461 462 # POST data urlencoded, small enough to be sent with request headers 463 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 464 def test_07_39_post_urlenc_small(self, env: Env, httpd, nghttpx, repeat, proto): 465 if proto == 'h3' and not env.have_h3(): 466 pytest.skip("h3 not supported") 467 if proto == 'h3' and env.curl_uses_lib('msh3'): 468 pytest.skip("msh3 fails here") 469 fdata = os.path.join(env.gen_dir, 'data-63k') 470 curl = CurlClient(env=env) 471 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 472 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto, extra_args=[ 473 '--trace-config', 'http/2,http/3' 474 ]) 475 r.check_stats(count=1, http_status=200, exitcode=0) 476 indata = open(fdata).readlines() 477 respdata = open(curl.response_file(0)).readlines() 478 assert respdata == indata 479 480 # POST data urlencoded, large enough to be sent separate from request headers 481 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 482 def test_07_40_post_urlenc_large(self, env: Env, httpd, nghttpx, repeat, proto): 483 if proto == 'h3' and not env.have_h3(): 484 pytest.skip("h3 not supported") 485 if proto == 'h3' and env.curl_uses_lib('msh3'): 486 pytest.skip("msh3 fails here") 487 fdata = os.path.join(env.gen_dir, 'data-64k') 488 curl = CurlClient(env=env) 489 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 490 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto, extra_args=[ 491 '--trace-config', 'http/2,http/3' 492 ]) 493 r.check_stats(count=1, http_status=200, exitcode=0) 494 indata = open(fdata).readlines() 495 respdata = open(curl.response_file(0)).readlines() 496 assert respdata == indata 497 498 # POST data urlencoded, small enough to be sent with request headers 499 # and request headers are so large that the first send is larger 500 # than our default upload buffer length (64KB). 501 # Unfixed, this will fail when run with CURL_DBG_SOCK_WBLOCK=80 most 502 # of the time 503 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 504 def test_07_41_post_urlenc_small(self, env: Env, httpd, nghttpx, repeat, proto): 505 if proto == 'h3' and not env.have_h3(): 506 pytest.skip("h3 not supported") 507 if proto == 'h3' and env.curl_uses_lib('msh3'): 508 pytest.skip("msh3 fails here") 509 if proto == 'h3' and env.curl_uses_lib('quiche'): 510 pytest.skip("quiche has CWND issues with large requests") 511 fdata = os.path.join(env.gen_dir, 'data-63k') 512 curl = CurlClient(env=env) 513 extra_args = ['--trace-config', 'http/2,http/3'] 514 # add enough headers so that the first send chunk is > 64KB 515 for i in range(63): 516 extra_args.extend(['-H', f'x{i:02d}: {"y"*1019}']) 517 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 518 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto, extra_args=extra_args) 519 r.check_stats(count=1, http_status=200, exitcode=0) 520 indata = open(fdata).readlines() 521 respdata = open(curl.response_file(0)).readlines() 522 assert respdata == indata 523 524 def check_download(self, count, srcfile, curl): 525 for i in range(count): 526 dfile = curl.download_file(i) 527 assert os.path.exists(dfile) 528 if not filecmp.cmp(srcfile, dfile, shallow=False): 529 diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(), 530 b=open(dfile).readlines(), 531 fromfile=srcfile, 532 tofile=dfile, 533 n=1)) 534 assert False, f'download {dfile} differs:\n{diff}' 535 536 # upload data, pause, let connection die with an incomplete response 537 # issues #11769 #13260 538 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 539 def test_07_42a_upload_disconnect(self, env: Env, httpd, nghttpx, repeat, proto): 540 if proto == 'h3' and not env.have_h3(): 541 pytest.skip("h3 not supported") 542 if proto == 'h3' and env.curl_uses_lib('msh3'): 543 pytest.skip("msh3 fails here") 544 client = LocalClient(name='upload-pausing', env=env, timeout=60) 545 if not client.exists(): 546 pytest.skip(f'example client not built: {client.name}') 547 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]&die_after=0' 548 r = client.run(['-V', proto, url]) 549 if r.exit_code == 18: # PARTIAL_FILE is always ok 550 pass 551 elif proto == 'h2': 552 r.check_exit_code(92) # CURLE_HTTP2_STREAM also ok 553 elif proto == 'h3': 554 r.check_exit_code(95) # CURLE_HTTP3 also ok 555 else: 556 r.check_exit_code(18) # will fail as it should 557 558 # upload data, pause, let connection die without any response at all 559 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 560 def test_07_42b_upload_disconnect(self, env: Env, httpd, nghttpx, repeat, proto): 561 if proto == 'h3' and not env.have_h3(): 562 pytest.skip("h3 not supported") 563 if proto == 'h3' and env.curl_uses_lib('msh3'): 564 pytest.skip("msh3 fails here") 565 client = LocalClient(name='upload-pausing', env=env, timeout=60) 566 if not client.exists(): 567 pytest.skip(f'example client not built: {client.name}') 568 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]&just_die=1' 569 r = client.run(['-V', proto, url]) 570 exp_code = 52 # GOT_NOTHING 571 if proto == 'h2' or proto == 'h3': 572 exp_code = 0 # we get a 500 from the server 573 r.check_exit_code(exp_code) # GOT_NOTHING 574 575 # upload data, pause, let connection die after 100 continue 576 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 577 def test_07_42c_upload_disconnect(self, env: Env, httpd, nghttpx, repeat, proto): 578 if proto == 'h3' and not env.have_h3(): 579 pytest.skip("h3 not supported") 580 if proto == 'h3' and env.curl_uses_lib('msh3'): 581 pytest.skip("msh3 fails here") 582 client = LocalClient(name='upload-pausing', env=env, timeout=60) 583 if not client.exists(): 584 pytest.skip(f'example client not built: {client.name}') 585 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]&die_after_100=1' 586 r = client.run(['-V', proto, url]) 587 exp_code = 52 # GOT_NOTHING 588 if proto == 'h2' or proto == 'h3': 589 exp_code = 0 # we get a 500 from the server 590 r.check_exit_code(exp_code) # GOT_NOTHING 591 592 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 593 def test_07_43_upload_denied(self, env: Env, httpd, nghttpx, repeat, proto): 594 if proto == 'h3' and not env.have_h3(): 595 pytest.skip("h3 not supported") 596 if proto == 'h3' and env.curl_uses_lib('msh3'): 597 pytest.skip("msh3 fails here") 598 fdata = os.path.join(env.gen_dir, 'data-10m') 599 count = 1 600 max_upload = 128 * 1024 601 curl = CurlClient(env=env) 602 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?'\ 603 f'id=[0-{count-1}]&max_upload={max_upload}' 604 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto, 605 extra_args=['--trace-config', 'all']) 606 r.check_stats(count=count, http_status=413, exitcode=0) 607 608 # speed limited on put handler 609 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 610 def test_07_50_put_speed_limit(self, env: Env, httpd, nghttpx, proto, repeat): 611 if proto == 'h3' and not env.have_h3(): 612 pytest.skip("h3 not supported") 613 count = 1 614 fdata = os.path.join(env.gen_dir, 'data-100k') 615 up_len = 100 * 1024 616 speed_limit = 50 * 1024 617 curl = CurlClient(env=env) 618 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?id=[0-0]' 619 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto, 620 with_headers=True, extra_args=[ 621 '--limit-rate', f'{speed_limit}' 622 ]) 623 r.check_response(count=count, http_status=200) 624 assert r.responses[0]['header']['received-length'] == f'{up_len}', f'{r.responses[0]}' 625 up_speed = r.stats[0]['speed_upload'] 626 assert (speed_limit * 0.5) <= up_speed <= (speed_limit * 1.5), f'{r.stats[0]}' 627 628 # speed limited on echo handler 629 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 630 def test_07_51_echo_speed_limit(self, env: Env, httpd, nghttpx, proto, repeat): 631 if proto == 'h3' and not env.have_h3(): 632 pytest.skip("h3 not supported") 633 count = 1 634 fdata = os.path.join(env.gen_dir, 'data-100k') 635 speed_limit = 50 * 1024 636 curl = CurlClient(env=env) 637 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/echo?id=[0-0]' 638 r = curl.http_upload(urls=[url], data=f'@{fdata}', alpn_proto=proto, 639 with_headers=True, extra_args=[ 640 '--limit-rate', f'{speed_limit}' 641 ]) 642 r.check_response(count=count, http_status=200) 643 up_speed = r.stats[0]['speed_upload'] 644 assert (speed_limit * 0.5) <= up_speed <= (speed_limit * 1.5), f'{r.stats[0]}' 645 646 # upload larger data, triggering "Expect: 100-continue" code paths 647 @pytest.mark.parametrize("proto", ['http/1.1']) 648 def test_07_60_upload_exp100(self, env: Env, httpd, nghttpx, repeat, proto): 649 fdata = os.path.join(env.gen_dir, 'data-1m+') 650 read_delay = 1 651 curl = CurlClient(env=env) 652 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?id=[0-0]'\ 653 f'&read_delay={read_delay}s' 654 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto, extra_args=[ 655 '--expect100-timeout', f'{read_delay+1}' 656 ]) 657 r.check_stats(count=1, http_status=200, exitcode=0) 658 659 # upload larger data, triggering "Expect: 100-continue" code paths 660 @pytest.mark.parametrize("proto", ['http/1.1']) 661 def test_07_61_upload_exp100_timeout(self, env: Env, httpd, nghttpx, repeat, proto): 662 fdata = os.path.join(env.gen_dir, 'data-1m+') 663 read_delay = 2 664 curl = CurlClient(env=env) 665 url = f'https://{env.authority_for(env.domain1, proto)}/curltest/put?id=[0-0]'\ 666 f'&read_delay={read_delay}s' 667 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto, extra_args=[ 668 '--expect100-timeout', f'{read_delay-1}' 669 ]) 670 r.check_stats(count=1, http_status=200, exitcode=0) 671 672 # nghttpx is the only server we have that supports TLS early data and 673 # has a limit of 16k it announces 674 @pytest.mark.skipif(condition=not Env.have_nghttpx(), reason="no nghttpx") 675 @pytest.mark.parametrize("proto,upload_size,exp_early", [ 676 ['http/1.1', 100, 203], # headers+body 677 ['http/1.1', 10*1024, 10345], # headers+body 678 ['http/1.1', 32*1024, 16384], # headers+body, limited by server max 679 ['h2', 10*1024, 10378], # headers+body 680 ['h2', 32*1024, 16384], # headers+body, limited by server max 681 ['h3', 1024, 0], # earlydata not supported 682 ]) 683 def test_07_70_put_earlydata(self, env: Env, httpd, nghttpx, proto, upload_size, exp_early): 684 if not env.curl_uses_lib('gnutls'): 685 pytest.skip('TLS earlydata only implemented in GnuTLS') 686 if proto == 'h3' and not env.have_h3(): 687 pytest.skip("h3 not supported") 688 count = 2 689 # we want this test to always connect to nghttpx, since it is 690 # the only server we have that supports TLS earlydata 691 port = env.port_for(proto) 692 if proto != 'h3': 693 port = env.nghttpx_https_port 694 url = f'https://{env.domain1}:{port}/curltest/put?id=[0-{count-1}]' 695 client = LocalClient(name='hx-upload', env=env) 696 if not client.exists(): 697 pytest.skip(f'example client not built: {client.name}') 698 r = client.run(args=[ 699 '-n', f'{count}', 700 '-e', # use TLS earlydata 701 '-f', # forbid reuse of connections 702 '-l', # announce upload length, no 'Expect: 100' 703 '-S', f'{upload_size}', 704 '-r', f'{env.domain1}:{port}:127.0.0.1', 705 '-V', proto, url 706 ]) 707 r.check_exit_code(0) 708 self.check_downloads(client, [f"{upload_size}"], count) 709 earlydata = {} 710 for line in r.trace_lines: 711 m = re.match(r'^\[t-(\d+)] EarlyData: (\d+)', line) 712 if m: 713 earlydata[int(m.group(1))] = int(m.group(2)) 714 assert earlydata[0] == 0, f'{earlydata}' 715 assert earlydata[1] == exp_early, f'{earlydata}' 716 717 def check_downloads(self, client, source: List[str], count: int, 718 complete: bool = True): 719 for i in range(count): 720 dfile = client.download_file(i) 721 assert os.path.exists(dfile) 722 if complete: 723 diff = "".join(difflib.unified_diff(a=source, 724 b=open(dfile).readlines(), 725 fromfile='-', 726 tofile=dfile, 727 n=1)) 728 assert not diff, f'download {dfile} differs:\n{diff}' 729
