1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import difflib 28import filecmp 29import logging 30import os 31import re 32import pytest 33 34from testenv import Env, CurlClient, Caddy, LocalClient 35 36 37log = logging.getLogger(__name__) 38 39 40@pytest.mark.skipif(condition=not Env.has_caddy(), reason="missing caddy") 41@pytest.mark.skipif(condition=not Env.have_ssl_curl(), reason="curl without SSL") 42class TestCaddy: 43 44 @pytest.fixture(autouse=True, scope='class') 45 def caddy(self, env): 46 caddy = Caddy(env=env) 47 assert caddy.start() 48 yield caddy 49 caddy.stop() 50 51 def _make_docs_file(self, docs_dir: str, fname: str, fsize: int): 52 fpath = os.path.join(docs_dir, fname) 53 data1k = 1024*'x' 54 flen = 0 55 with open(fpath, 'w') as fd: 56 while flen < fsize: 57 fd.write(data1k) 58 flen += len(data1k) 59 return flen 60 61 @pytest.fixture(autouse=True, scope='class') 62 def _class_scope(self, env, caddy): 63 self._make_docs_file(docs_dir=caddy.docs_dir, fname='data10k.data', fsize=10*1024) 64 self._make_docs_file(docs_dir=caddy.docs_dir, fname='data1.data', fsize=1024*1024) 65 self._make_docs_file(docs_dir=caddy.docs_dir, fname='data5.data', fsize=5*1024*1024) 66 self._make_docs_file(docs_dir=caddy.docs_dir, fname='data10.data', fsize=10*1024*1024) 67 self._make_docs_file(docs_dir=caddy.docs_dir, fname='data100.data', fsize=100*1024*1024) 68 env.make_data_file(indir=env.gen_dir, fname="data-10m", fsize=10*1024*1024) 69 70 # download 1 file 71 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 72 def test_08_01_download_1(self, env: Env, caddy: Caddy, repeat, proto): 73 if proto == 'h3' and not env.have_h3_curl(): 74 pytest.skip("h3 not supported in curl") 75 if proto == 'h3' and env.curl_uses_lib('msh3'): 76 pytest.skip("msh3 itself crashes") 77 curl = CurlClient(env=env) 78 url = f'https://{env.domain1}:{caddy.port}/data.json' 79 r = curl.http_download(urls=[url], alpn_proto=proto) 80 r.check_response(count=1, http_status=200) 81 82 # download 1MB files sequentially 83 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 84 def test_08_02_download_1mb_sequential(self, env: Env, caddy: Caddy, 85 repeat, proto): 86 if proto == 'h3' and not env.have_h3_curl(): 87 pytest.skip("h3 not supported in curl") 88 if proto == 'h3' and env.curl_uses_lib('msh3'): 89 pytest.skip("msh3 itself crashes") 90 count = 50 91 curl = CurlClient(env=env) 92 urln = f'https://{env.domain1}:{caddy.port}/data1.data?[0-{count-1}]' 93 r = curl.http_download(urls=[urln], alpn_proto=proto) 94 r.check_response(count=count, http_status=200, connect_count=1) 95 96 # download 1MB files parallel 97 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 98 def test_08_03_download_1mb_parallel(self, env: Env, caddy: Caddy, 99 repeat, proto): 100 if proto == 'h3' and not env.have_h3_curl(): 101 pytest.skip("h3 not supported in curl") 102 if proto == 'h3' and env.curl_uses_lib('msh3'): 103 pytest.skip("msh3 itself crashes") 104 count = 20 105 curl = CurlClient(env=env) 106 urln = f'https://{env.domain1}:{caddy.port}/data1.data?[0-{count-1}]' 107 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 108 '--parallel' 109 ]) 110 r.check_response(count=count, http_status=200) 111 if proto == 'http/1.1': 112 # http/1.1 parallel transfers will open multiple connections 113 assert r.total_connects > 1, r.dump_logs() 114 else: 115 assert r.total_connects == 1, r.dump_logs() 116 117 # download 5MB files sequentially 118 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 119 @pytest.mark.skipif(condition=Env().ci_run, reason="not suitable for CI runs") 120 @pytest.mark.parametrize("proto", ['h2', 'h3']) 121 def test_08_04a_download_10mb_sequential(self, env: Env, caddy: Caddy, 122 repeat, proto): 123 if proto == 'h3' and not env.have_h3_curl(): 124 pytest.skip("h3 not supported in curl") 125 if proto == 'h3' and env.curl_uses_lib('msh3'): 126 pytest.skip("msh3 itself crashes") 127 count = 40 128 curl = CurlClient(env=env) 129 urln = f'https://{env.domain1}:{caddy.port}/data5.data?[0-{count-1}]' 130 r = curl.http_download(urls=[urln], alpn_proto=proto) 131 r.check_response(count=count, http_status=200, connect_count=1) 132 133 # download 10MB files sequentially 134 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 135 @pytest.mark.skipif(condition=Env().ci_run, reason="not suitable for CI runs") 136 @pytest.mark.parametrize("proto", ['h2', 'h3']) 137 def test_08_04b_download_10mb_sequential(self, env: Env, caddy: Caddy, 138 repeat, proto): 139 if proto == 'h3' and not env.have_h3_curl(): 140 pytest.skip("h3 not supported in curl") 141 if proto == 'h3' and env.curl_uses_lib('msh3'): 142 pytest.skip("msh3 itself crashes") 143 count = 20 144 curl = CurlClient(env=env) 145 urln = f'https://{env.domain1}:{caddy.port}/data10.data?[0-{count-1}]' 146 r = curl.http_download(urls=[urln], alpn_proto=proto) 147 r.check_response(count=count, http_status=200, connect_count=1) 148 149 # download 10MB files parallel 150 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 151 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 152 @pytest.mark.skipif(condition=Env().ci_run, reason="not suitable for CI runs") 153 def test_08_05_download_1mb_parallel(self, env: Env, caddy: Caddy, 154 repeat, proto): 155 if proto == 'h3' and not env.have_h3_curl(): 156 pytest.skip("h3 not supported in curl") 157 if proto == 'h3' and env.curl_uses_lib('msh3'): 158 pytest.skip("msh3 itself crashes") 159 if proto == 'http/1.1' and env.curl_uses_lib('mbedtls'): 160 pytest.skip("mbedtls 3.6.0 fails on 50 connections with: "\ 161 "ssl_handshake returned: (-0x7F00) SSL - Memory allocation failed") 162 count = 50 163 curl = CurlClient(env=env) 164 urln = f'https://{env.domain1}:{caddy.port}/data10.data?[0-{count-1}]' 165 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 166 '--parallel' 167 ]) 168 r.check_response(count=count, http_status=200) 169 if proto == 'http/1.1': 170 # http/1.1 parallel transfers will open multiple connections 171 assert r.total_connects > 1, r.dump_logs() 172 else: 173 assert r.total_connects == 1, r.dump_logs() 174 175 # post data parallel, check that they were echoed 176 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 177 def test_08_06_post_parallel(self, env: Env, httpd, caddy, repeat, proto): 178 if proto == 'h3' and not env.have_h3(): 179 pytest.skip("h3 not supported") 180 if proto == 'h3' and env.curl_uses_lib('msh3'): 181 pytest.skip("msh3 stalls here") 182 # limit since we use a separate connection in h1 183 count = 20 184 data = '0123456789' 185 curl = CurlClient(env=env) 186 url = f'https://{env.domain2}:{caddy.port}/curltest/echo?id=[0-{count-1}]' 187 r = curl.http_upload(urls=[url], data=data, alpn_proto=proto, 188 extra_args=['--parallel']) 189 r.check_stats(count=count, http_status=200, exitcode=0) 190 for i in range(count): 191 respdata = open(curl.response_file(i)).readlines() 192 assert respdata == [data] 193 194 # put large file, check that they length were echoed 195 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 196 def test_08_07_put_large(self, env: Env, httpd, caddy, repeat, proto): 197 if proto == 'h3' and not env.have_h3(): 198 pytest.skip("h3 not supported") 199 if proto == 'h3' and env.curl_uses_lib('msh3'): 200 pytest.skip("msh3 stalls here") 201 # limit since we use a separate connection in h1< 202 count = 1 203 fdata = os.path.join(env.gen_dir, 'data-10m') 204 curl = CurlClient(env=env) 205 url = f'https://{env.domain2}:{caddy.port}/curltest/put?id=[0-{count-1}]' 206 r = curl.http_put(urls=[url], fdata=fdata, alpn_proto=proto) 207 exp_data = [f'{os.path.getsize(fdata)}'] 208 r.check_response(count=count, http_status=200) 209 for i in range(count): 210 respdata = open(curl.response_file(i)).readlines() 211 assert respdata == exp_data 212 213 @pytest.mark.parametrize("proto", ['http/1.1', 'h2']) 214 def test_08_08_earlydata(self, env: Env, httpd, caddy, proto): 215 count = 2 216 docname = 'data10k.data' 217 url = f'https://{env.domain1}:{caddy.port}/{docname}' 218 client = LocalClient(name='hx-download', env=env) 219 if not client.exists(): 220 pytest.skip(f'example client not built: {client.name}') 221 r = client.run(args=[ 222 '-n', f'{count}', 223 '-e', # use TLS earlydata 224 '-f', # forbid reuse of connections 225 '-r', f'{env.domain1}:{caddy.port}:127.0.0.1', 226 '-V', proto, url 227 ]) 228 r.check_exit_code(0) 229 srcfile = os.path.join(caddy.docs_dir, docname) 230 self.check_downloads(client, srcfile, count) 231 earlydata = {} 232 for line in r.trace_lines: 233 m = re.match(r'^\[t-(\d+)] EarlyData: (\d+)', line) 234 if m: 235 earlydata[int(m.group(1))] = int(m.group(2)) 236 # Caddy does not support early data 237 assert earlydata[0] == 0, f'{earlydata}' 238 assert earlydata[1] == 0, f'{earlydata}' 239 240 def check_downloads(self, client, srcfile: str, count: int, 241 complete: bool = True): 242 for i in range(count): 243 dfile = client.download_file(i) 244 assert os.path.exists(dfile) 245 if complete and not filecmp.cmp(srcfile, dfile, shallow=False): 246 diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(), 247 b=open(dfile).readlines(), 248 fromfile=srcfile, 249 tofile=dfile, 250 n=1)) 251 assert False, f'download {dfile} differs:\n{diff}' 252
