1 /*
2 * Copyright 2016-2023 The OpenSSL Project Authors. All Rights Reserved.
3 *
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
8 */
9
10 #include "../testutil.h"
11 #include "output.h"
12 #include "tu_local.h"
13
14 #include <string.h>
15 #include <assert.h>
16
17 #include "internal/nelem.h"
18 #include <openssl/bio.h>
19
20 #include "platform.h" /* From libapps */
21
22 #if defined(_WIN32) && !defined(__BORLANDC__)
23 # define strdup _strdup
24 #endif
25
26
27 /*
28 * Declares the structures needed to register each test case function.
29 */
30 typedef struct test_info {
31 const char *test_case_name;
32 int (*test_fn) (void);
33 int (*param_test_fn)(int idx);
34 int num;
35
36 /* flags */
37 int subtest:1;
38 } TEST_INFO;
39
40 static TEST_INFO all_tests[1024];
41 static int num_tests = 0;
42 static int show_list = 0;
43 static int single_test = -1;
44 static int single_iter = -1;
45 static int level = 0;
46 static int seed = 0;
47 static int rand_order = 0;
48
49 /*
50 * A parameterised test runs a loop of test cases.
51 * |num_test_cases| counts the total number of non-subtest test cases
52 * across all tests.
53 */
54 static int num_test_cases = 0;
55
56 static int process_shared_options(void);
57
58
add_test(const char * test_case_name,int (* test_fn)(void))59 void add_test(const char *test_case_name, int (*test_fn) (void))
60 {
61 assert(num_tests != OSSL_NELEM(all_tests));
62 all_tests[num_tests].test_case_name = test_case_name;
63 all_tests[num_tests].test_fn = test_fn;
64 all_tests[num_tests].num = -1;
65 ++num_tests;
66 ++num_test_cases;
67 }
68
add_all_tests(const char * test_case_name,int (* test_fn)(int idx),int num,int subtest)69 void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
70 int num, int subtest)
71 {
72 assert(num_tests != OSSL_NELEM(all_tests));
73 all_tests[num_tests].test_case_name = test_case_name;
74 all_tests[num_tests].param_test_fn = test_fn;
75 all_tests[num_tests].num = num;
76 all_tests[num_tests].subtest = subtest;
77 ++num_tests;
78 if (subtest)
79 ++num_test_cases;
80 else
81 num_test_cases += num;
82 }
83
gcd(int a,int b)84 static int gcd(int a, int b)
85 {
86 while (b != 0) {
87 int t = b;
88 b = a % b;
89 a = t;
90 }
91 return a;
92 }
93
set_seed(int s)94 static void set_seed(int s)
95 {
96 seed = s;
97 if (seed <= 0)
98 seed = (int)time(NULL);
99 test_random_seed(seed);
100 }
101
102
setup_test_framework(int argc,char * argv[])103 int setup_test_framework(int argc, char *argv[])
104 {
105 char *test_rand_order = getenv("OPENSSL_TEST_RAND_ORDER");
106 char *test_rand_seed = getenv("OPENSSL_TEST_RAND_SEED");
107 char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");
108
109 if (TAP_levels != NULL)
110 level = 4 * atoi(TAP_levels);
111 test_adjust_streams_tap_level(level);
112 if (test_rand_order != NULL) {
113 rand_order = 1;
114 set_seed(atoi(test_rand_order));
115 } else if (test_rand_seed != NULL) {
116 set_seed(atoi(test_rand_seed));
117 } else {
118 set_seed(0);
119 }
120
121 #if defined(OPENSSL_SYS_VMS) && defined(__DECC)
122 argv = copy_argv(&argc, argv);
123 #elif defined(_WIN32)
124 /*
125 * Replace argv[] with UTF-8 encoded strings.
126 */
127 win32_utf8argv(&argc, &argv);
128 #endif
129
130 if (!opt_init(argc, argv, test_get_options()))
131 return 0;
132 return 1;
133 }
134
135
136 /*
137 * This can only be called after setup() has run, since num_tests and
138 * all_tests[] are setup at this point
139 */
check_single_test_params(char * name,char * testname,char * itname)140 static int check_single_test_params(char *name, char *testname, char *itname)
141 {
142 if (name != NULL) {
143 int i;
144 for (i = 0; i < num_tests; ++i) {
145 if (strcmp(name, all_tests[i].test_case_name) == 0) {
146 single_test = 1 + i;
147 break;
148 }
149 }
150 if (i >= num_tests)
151 single_test = atoi(name);
152 }
153
154
155 /* if only iteration is specified, assume we want the first test */
156 if (single_test == -1 && single_iter != -1)
157 single_test = 1;
158
159 if (single_test != -1) {
160 if (single_test < 1 || single_test > num_tests) {
161 test_printf_stderr("Invalid -%s value "
162 "(Value must be a valid test name OR a value between %d..%d)\n",
163 testname, 1, num_tests);
164 return 0;
165 }
166 }
167 if (single_iter != -1) {
168 if (all_tests[single_test - 1].num == -1) {
169 test_printf_stderr("-%s option is not valid for test %d:%s\n",
170 itname,
171 single_test,
172 all_tests[single_test - 1].test_case_name);
173 return 0;
174 } else if (single_iter < 1
175 || single_iter > all_tests[single_test - 1].num) {
176 test_printf_stderr("Invalid -%s value for test %d:%s\t"
177 "(Value must be in the range %d..%d)\n",
178 itname, single_test,
179 all_tests[single_test - 1].test_case_name,
180 1, all_tests[single_test - 1].num);
181 return 0;
182 }
183 }
184 return 1;
185 }
186
process_shared_options(void)187 static int process_shared_options(void)
188 {
189 OPTION_CHOICE_DEFAULT o;
190 int value;
191 int ret = -1;
192 char *flag_test = "";
193 char *flag_iter = "";
194 char *testname = NULL;
195
196 opt_begin();
197 while ((o = opt_next()) != OPT_EOF) {
198 switch (o) {
199 /* Ignore any test options at this level */
200 default:
201 break;
202 case OPT_ERR:
203 return ret;
204 case OPT_TEST_HELP:
205 opt_help(test_get_options());
206 return 0;
207 case OPT_TEST_LIST:
208 show_list = 1;
209 break;
210 case OPT_TEST_SINGLE:
211 flag_test = opt_flag();
212 testname = opt_arg();
213 break;
214 case OPT_TEST_ITERATION:
215 flag_iter = opt_flag();
216 if (!opt_int(opt_arg(), &single_iter))
217 goto end;
218 break;
219 case OPT_TEST_INDENT:
220 if (!opt_int(opt_arg(), &value))
221 goto end;
222 level = 4 * value;
223 test_adjust_streams_tap_level(level);
224 break;
225 case OPT_TEST_SEED:
226 if (!opt_int(opt_arg(), &value))
227 goto end;
228 set_seed(value);
229 break;
230 }
231 }
232 if (!check_single_test_params(testname, flag_test, flag_iter))
233 goto end;
234 ret = 1;
235 end:
236 return ret;
237 }
238
239
pulldown_test_framework(int ret)240 int pulldown_test_framework(int ret)
241 {
242 set_test_title(NULL);
243 return ret;
244 }
245
finalize(int success)246 static void finalize(int success)
247 {
248 if (success)
249 ERR_clear_error();
250 else
251 ERR_print_errors_cb(openssl_error_cb, NULL);
252 }
253
254 static char *test_title = NULL;
255
set_test_title(const char * title)256 void set_test_title(const char *title)
257 {
258 free(test_title);
259 test_title = title == NULL ? NULL : strdup(title);
260 }
261
test_verdict(int verdict,const char * description,...)262 PRINTF_FORMAT(2, 3) static void test_verdict(int verdict,
263 const char *description, ...)
264 {
265 va_list ap;
266
267 test_flush_stdout();
268 test_flush_stderr();
269
270 if (verdict == 0) {
271 if (rand_order)
272 test_printf_tapout("# OPENSSL_TEST_RAND_ORDER=%d\n", seed);
273 else
274 test_printf_tapout("# OPENSSL_TEST_RAND_SEED=%d\n", seed);
275 }
276 test_printf_tapout("%s ", verdict != 0 ? "ok" : "not ok");
277 va_start(ap, description);
278 test_vprintf_tapout(description, ap);
279 va_end(ap);
280 if (verdict == TEST_SKIP_CODE)
281 test_printf_tapout(" # skipped");
282 test_printf_tapout("\n");
283 test_flush_tapout();
284 }
285
run_tests(const char * test_prog_name)286 int run_tests(const char *test_prog_name)
287 {
288 int num_failed = 0;
289 int verdict = 1;
290 int ii, i, jj, j, jstep;
291 int test_case_count = 0;
292 int subtest_case_count = 0;
293 int permute[OSSL_NELEM(all_tests)];
294
295 i = process_shared_options();
296 if (i == 0)
297 return EXIT_SUCCESS;
298 if (i == -1)
299 return EXIT_FAILURE;
300
301 if (num_tests < 1) {
302 test_printf_tapout("1..0 # Skipped: %s\n", test_prog_name);
303 } else if (show_list == 0 && single_test == -1) {
304 if (level > 0) {
305 test_printf_stdout("Subtest: %s\n", test_prog_name);
306 test_flush_stdout();
307 }
308 test_printf_tapout("1..%d\n", num_test_cases);
309 }
310
311 test_flush_tapout();
312
313 for (i = 0; i < num_tests; i++)
314 permute[i] = i;
315 if (rand_order != 0)
316 for (i = num_tests - 1; i >= 1; i--) {
317 j = test_random() % (1 + i);
318 ii = permute[j];
319 permute[j] = permute[i];
320 permute[i] = ii;
321 }
322
323 for (ii = 0; ii != num_tests; ++ii) {
324 i = permute[ii];
325
326 if (single_test != -1 && ((i+1) != single_test)) {
327 continue;
328 }
329 else if (show_list) {
330 if (all_tests[i].num != -1) {
331 test_printf_tapout("%d - %s (%d..%d)\n", ii + 1,
332 all_tests[i].test_case_name, 1,
333 all_tests[i].num);
334 } else {
335 test_printf_tapout("%d - %s\n", ii + 1,
336 all_tests[i].test_case_name);
337 }
338 test_flush_tapout();
339 } else if (all_tests[i].num == -1) {
340 set_test_title(all_tests[i].test_case_name);
341 ERR_clear_error();
342 verdict = all_tests[i].test_fn();
343 finalize(verdict != 0);
344 test_verdict(verdict, "%d - %s", test_case_count + 1, test_title);
345 if (verdict == 0)
346 num_failed++;
347 test_case_count++;
348 } else {
349 verdict = TEST_SKIP_CODE;
350 set_test_title(all_tests[i].test_case_name);
351 if (all_tests[i].subtest) {
352 level += 4;
353 test_adjust_streams_tap_level(level);
354 if (single_iter == -1) {
355 test_printf_stdout("Subtest: %s\n", test_title);
356 test_printf_tapout("%d..%d\n", 1, all_tests[i].num);
357 test_flush_stdout();
358 test_flush_tapout();
359 }
360 }
361
362 j = -1;
363 if (rand_order == 0 || all_tests[i].num < 3)
364 jstep = 1;
365 else
366 do
367 jstep = test_random() % all_tests[i].num;
368 while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1);
369
370 for (jj = 0; jj < all_tests[i].num; jj++) {
371 int v;
372
373 j = (j + jstep) % all_tests[i].num;
374 if (single_iter != -1 && ((jj + 1) != single_iter))
375 continue;
376 ERR_clear_error();
377 v = all_tests[i].param_test_fn(j);
378
379 if (v == 0) {
380 verdict = 0;
381 } else if (v != TEST_SKIP_CODE && verdict != 0) {
382 verdict = 1;
383 }
384
385 finalize(v != 0);
386
387 if (all_tests[i].subtest)
388 test_verdict(v, "%d - iteration %d",
389 subtest_case_count + 1, j + 1);
390 else
391 test_verdict(v, "%d - %s - iteration %d",
392 test_case_count + subtest_case_count + 1,
393 test_title, j + 1);
394 subtest_case_count++;
395 }
396
397 if (all_tests[i].subtest) {
398 level -= 4;
399 test_adjust_streams_tap_level(level);
400 }
401 if (verdict == 0)
402 ++num_failed;
403 if (all_tests[i].num == -1 || all_tests[i].subtest)
404 test_verdict(verdict, "%d - %s", test_case_count + 1,
405 all_tests[i].test_case_name);
406 test_case_count++;
407 }
408 }
409 if (num_failed != 0)
410 return EXIT_FAILURE;
411 return EXIT_SUCCESS;
412 }
413
414 /*
415 * Glue an array of strings together and return it as an allocated string.
416 * Optionally return the whole length of this string in |out_len|
417 */
glue_strings(const char * list[],size_t * out_len)418 char *glue_strings(const char *list[], size_t *out_len)
419 {
420 size_t len = 0;
421 char *p, *ret;
422 int i;
423
424 for (i = 0; list[i] != NULL; i++)
425 len += strlen(list[i]);
426
427 if (out_len != NULL)
428 *out_len = len;
429
430 if (!TEST_ptr(ret = p = OPENSSL_malloc(len + 1)))
431 return NULL;
432
433 for (i = 0; list[i] != NULL; i++)
434 p += strlen(strcpy(p, list[i]));
435
436 return ret;
437 }
438
test_mk_file_path(const char * dir,const char * file)439 char *test_mk_file_path(const char *dir, const char *file)
440 {
441 # ifndef OPENSSL_SYS_VMS
442 const char *sep = "/";
443 # else
444 const char *sep = "";
445 char *dir_end;
446 char dir_end_sep;
447 # endif
448 size_t dirlen = dir != NULL ? strlen(dir) : 0;
449 size_t len = dirlen + strlen(sep) + strlen(file) + 1;
450 char *full_file = OPENSSL_zalloc(len);
451
452 if (full_file != NULL) {
453 if (dir != NULL && dirlen > 0) {
454 OPENSSL_strlcpy(full_file, dir, len);
455 # ifdef OPENSSL_SYS_VMS
456 /*
457 * If |file| contains a directory spec, we need to do some
458 * careful merging.
459 * "vol:[dir.dir]" + "[.certs]sm2-root.crt" should become
460 * "vol:[dir.dir.certs]sm2-root.crt"
461 */
462 dir_end = &full_file[strlen(full_file) - 1];
463 dir_end_sep = *dir_end;
464 if ((dir_end_sep == ']' || dir_end_sep == '>')
465 && (file[0] == '[' || file[0] == '<')) {
466 file++;
467 if (file[0] == '.')
468 *dir_end = '\0';
469 else
470 *dir_end = '.';
471 }
472 #else
473 OPENSSL_strlcat(full_file, sep, len);
474 #endif
475 }
476 OPENSSL_strlcat(full_file, file, len);
477 }
478
479 return full_file;
480 }
481
