1 /*
2 * Copyright 2022-2024 The OpenSSL Project Authors. All Rights Reserved.
3 *
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
8 */
9
10 #include "internal/quic_txp.h"
11 #include "internal/quic_fifd.h"
12 #include "internal/quic_stream_map.h"
13 #include "internal/quic_error.h"
14 #include "internal/common.h"
15 #include <openssl/err.h>
16
17 #define MIN_CRYPTO_HDR_SIZE 3
18
19 #define MIN_FRAME_SIZE_HANDSHAKE_DONE 1
20 #define MIN_FRAME_SIZE_MAX_DATA 2
21 #define MIN_FRAME_SIZE_ACK 5
22 #define MIN_FRAME_SIZE_CRYPTO (MIN_CRYPTO_HDR_SIZE + 1)
23 #define MIN_FRAME_SIZE_STREAM 3 /* minimum useful size (for non-FIN) */
24 #define MIN_FRAME_SIZE_MAX_STREAMS_BIDI 2
25 #define MIN_FRAME_SIZE_MAX_STREAMS_UNI 2
26
27 /*
28 * Packet Archetypes
29 * =================
30 */
31
32 /* Generate normal packets containing most frame types, subject to EL. */
33 #define TX_PACKETISER_ARCHETYPE_NORMAL 0
34
35 /*
36 * A probe packet is different in that:
37 * - It bypasses CC, but *is* counted as in flight for purposes of CC;
38 * - It must be ACK-eliciting.
39 */
40 #define TX_PACKETISER_ARCHETYPE_PROBE 1
41
42 /*
43 * An ACK-only packet is different in that:
44 * - It bypasses CC, and is considered a 'non-inflight' packet;
45 * - It may not contain anything other than an ACK frame, not even padding.
46 */
47 #define TX_PACKETISER_ARCHETYPE_ACK_ONLY 2
48
49 #define TX_PACKETISER_ARCHETYPE_NUM 3
50
51 struct ossl_quic_tx_packetiser_st {
52 OSSL_QUIC_TX_PACKETISER_ARGS args;
53
54 /*
55 * Opaque initial token blob provided by caller. TXP frees using the
56 * callback when it is no longer needed.
57 */
58 const unsigned char *initial_token;
59 size_t initial_token_len;
60 ossl_quic_initial_token_free_fn *initial_token_free_cb;
61 void *initial_token_free_cb_arg;
62
63 /* Subcomponents of the TXP that we own. */
64 QUIC_FIFD fifd; /* QUIC Frame-in-Flight Dispatcher */
65
66 /* Internal state. */
67 uint64_t next_pn[QUIC_PN_SPACE_NUM]; /* Next PN to use in given PN space. */
68 OSSL_TIME last_tx_time; /* Last time a packet was generated, or 0. */
69
70 /* Internal state - frame (re)generation flags. */
71 unsigned int want_handshake_done : 1;
72 unsigned int want_max_data : 1;
73 unsigned int want_max_streams_bidi : 1;
74 unsigned int want_max_streams_uni : 1;
75
76 /* Internal state - frame (re)generation flags - per PN space. */
77 unsigned int want_ack : QUIC_PN_SPACE_NUM;
78 unsigned int force_ack_eliciting : QUIC_PN_SPACE_NUM;
79
80 /*
81 * Internal state - connection close terminal state.
82 * Once this is set, it is not unset unlike other want_ flags - we keep
83 * sending it in every packet.
84 */
85 unsigned int want_conn_close : 1;
86
87 /* Has the handshake been completed? */
88 unsigned int handshake_complete : 1;
89
90 OSSL_QUIC_FRAME_CONN_CLOSE conn_close_frame;
91
92 /*
93 * Counts of the number of bytes received and sent while in the closing
94 * state.
95 */
96 uint64_t closing_bytes_recv;
97 uint64_t closing_bytes_xmit;
98
99 /* Internal state - packet assembly. */
100 struct txp_el {
101 unsigned char *scratch; /* scratch buffer for packet assembly */
102 size_t scratch_len; /* number of bytes allocated for scratch */
103 OSSL_QTX_IOVEC *iovec; /* scratch iovec array for use with QTX */
104 size_t alloc_iovec; /* size of iovec array */
105 } el[QUIC_ENC_LEVEL_NUM];
106
107 /* Message callback related arguments */
108 ossl_msg_cb msg_callback;
109 void *msg_callback_arg;
110 SSL *msg_callback_ssl;
111
112 /* Callbacks. */
113 void (*ack_tx_cb)(const OSSL_QUIC_FRAME_ACK *ack,
114 uint32_t pn_space,
115 void *arg);
116 void *ack_tx_cb_arg;
117 };
118
119 /*
120 * The TX helper records state used while generating frames into packets. It
121 * enables serialization into the packet to be done "transactionally" where
122 * serialization of a frame can be rolled back if it fails midway (e.g. if it
123 * does not fit).
124 */
125 struct tx_helper {
126 OSSL_QUIC_TX_PACKETISER *txp;
127 /*
128 * The Maximum Packet Payload Length in bytes. This is the amount of
129 * space we have to generate frames into.
130 */
131 size_t max_ppl;
132 /*
133 * Number of bytes we have generated so far.
134 */
135 size_t bytes_appended;
136 /*
137 * Number of scratch bytes in txp->scratch we have used so far. Some iovecs
138 * will reference this scratch buffer. When we need to use more of it (e.g.
139 * when we need to put frame headers somewhere), we append to the scratch
140 * buffer, resizing if necessary, and increase this accordingly.
141 */
142 size_t scratch_bytes;
143 /*
144 * Bytes reserved in the MaxPPL budget. We keep this number of bytes spare
145 * until reserve_allowed is set to 1. Currently this is always at most 1, as
146 * a PING frame takes up one byte and this mechanism is only used to ensure
147 * we can encode a PING frame if we have been asked to ensure a packet is
148 * ACK-eliciting and we are unusure if we are going to add any other
149 * ACK-eliciting frames before we reach our MaxPPL budget.
150 */
151 size_t reserve;
152 /*
153 * Number of iovecs we have currently appended. This is the number of
154 * entries valid in txp->iovec.
155 */
156 size_t num_iovec;
157 /* The EL this TX helper is being used for. */
158 uint32_t enc_level;
159 /*
160 * Whether we are allowed to make use of the reserve bytes in our MaxPPL
161 * budget. This is used to ensure we have room to append a PING frame later
162 * if we need to. Once we know we will not need to append a PING frame, this
163 * is set to 1.
164 */
165 unsigned int reserve_allowed : 1;
166 /*
167 * Set to 1 if we have appended a STREAM frame with an implicit length. If
168 * this happens we should never append another frame after that frame as it
169 * cannot be validly encoded. This is just a safety check.
170 */
171 unsigned int done_implicit : 1;
172 struct {
173 /*
174 * The fields in this structure are valid if active is set, which means
175 * that a serialization transaction is currently in progress.
176 */
177 unsigned char *data;
178 WPACKET wpkt;
179 unsigned int active : 1;
180 } txn;
181 };
182
183 static void tx_helper_rollback(struct tx_helper *h);
184 static int txp_el_ensure_iovec(struct txp_el *el, size_t num);
185
186 /* Initialises the TX helper. */
tx_helper_init(struct tx_helper * h,OSSL_QUIC_TX_PACKETISER * txp,uint32_t enc_level,size_t max_ppl,size_t reserve)187 static int tx_helper_init(struct tx_helper *h, OSSL_QUIC_TX_PACKETISER *txp,
188 uint32_t enc_level, size_t max_ppl, size_t reserve)
189 {
190 if (reserve > max_ppl)
191 return 0;
192
193 h->txp = txp;
194 h->enc_level = enc_level;
195 h->max_ppl = max_ppl;
196 h->reserve = reserve;
197 h->num_iovec = 0;
198 h->bytes_appended = 0;
199 h->scratch_bytes = 0;
200 h->reserve_allowed = 0;
201 h->done_implicit = 0;
202 h->txn.data = NULL;
203 h->txn.active = 0;
204
205 if (max_ppl > h->txp->el[enc_level].scratch_len) {
206 unsigned char *scratch;
207
208 scratch = OPENSSL_realloc(h->txp->el[enc_level].scratch, max_ppl);
209 if (scratch == NULL)
210 return 0;
211
212 h->txp->el[enc_level].scratch = scratch;
213 h->txp->el[enc_level].scratch_len = max_ppl;
214 }
215
216 return 1;
217 }
218
tx_helper_cleanup(struct tx_helper * h)219 static void tx_helper_cleanup(struct tx_helper *h)
220 {
221 if (h->txn.active)
222 tx_helper_rollback(h);
223
224 h->txp = NULL;
225 }
226
tx_helper_unrestrict(struct tx_helper * h)227 static void tx_helper_unrestrict(struct tx_helper *h)
228 {
229 h->reserve_allowed = 1;
230 }
231
232 /*
233 * Append an extent of memory to the iovec list. The memory must remain
234 * allocated until we finish generating the packet and call the QTX.
235 *
236 * In general, the buffers passed to this function will be from one of two
237 * ranges:
238 *
239 * - Application data contained in stream buffers managed elsewhere
240 * in the QUIC stack; or
241 *
242 * - Control frame data appended into txp->scratch using tx_helper_begin and
243 * tx_helper_commit.
244 *
245 */
tx_helper_append_iovec(struct tx_helper * h,const unsigned char * buf,size_t buf_len)246 static int tx_helper_append_iovec(struct tx_helper *h,
247 const unsigned char *buf,
248 size_t buf_len)
249 {
250 struct txp_el *el = &h->txp->el[h->enc_level];
251
252 if (buf_len == 0)
253 return 1;
254
255 if (!ossl_assert(!h->done_implicit))
256 return 0;
257
258 if (!txp_el_ensure_iovec(el, h->num_iovec + 1))
259 return 0;
260
261 el->iovec[h->num_iovec].buf = buf;
262 el->iovec[h->num_iovec].buf_len = buf_len;
263
264 ++h->num_iovec;
265 h->bytes_appended += buf_len;
266 return 1;
267 }
268
269 /*
270 * How many more bytes of space do we have left in our plaintext packet payload?
271 */
tx_helper_get_space_left(struct tx_helper * h)272 static size_t tx_helper_get_space_left(struct tx_helper *h)
273 {
274 return h->max_ppl
275 - (h->reserve_allowed ? 0 : h->reserve) - h->bytes_appended;
276 }
277
278 /*
279 * Begin a control frame serialization transaction. This allows the
280 * serialization of the control frame to be backed out if it turns out it won't
281 * fit. Write the control frame to the returned WPACKET. Ensure you always
282 * call tx_helper_rollback or tx_helper_commit (or tx_helper_cleanup). Returns
283 * NULL on failure.
284 */
tx_helper_begin(struct tx_helper * h)285 static WPACKET *tx_helper_begin(struct tx_helper *h)
286 {
287 size_t space_left, len;
288 unsigned char *data;
289 struct txp_el *el = &h->txp->el[h->enc_level];
290
291 if (!ossl_assert(!h->txn.active))
292 return NULL;
293
294 if (!ossl_assert(!h->done_implicit))
295 return NULL;
296
297 data = (unsigned char *)el->scratch + h->scratch_bytes;
298 len = el->scratch_len - h->scratch_bytes;
299
300 space_left = tx_helper_get_space_left(h);
301 if (!ossl_assert(space_left <= len))
302 return NULL;
303
304 if (!WPACKET_init_static_len(&h->txn.wpkt, data, len, 0))
305 return NULL;
306
307 if (!WPACKET_set_max_size(&h->txn.wpkt, space_left)) {
308 WPACKET_cleanup(&h->txn.wpkt);
309 return NULL;
310 }
311
312 h->txn.data = data;
313 h->txn.active = 1;
314 return &h->txn.wpkt;
315 }
316
tx_helper_end(struct tx_helper * h,int success)317 static void tx_helper_end(struct tx_helper *h, int success)
318 {
319 if (success)
320 WPACKET_finish(&h->txn.wpkt);
321 else
322 WPACKET_cleanup(&h->txn.wpkt);
323
324 h->txn.active = 0;
325 h->txn.data = NULL;
326 }
327
328 /* Abort a control frame serialization transaction. */
tx_helper_rollback(struct tx_helper * h)329 static void tx_helper_rollback(struct tx_helper *h)
330 {
331 if (!h->txn.active)
332 return;
333
334 tx_helper_end(h, 0);
335 }
336
337 /* Commit a control frame. */
tx_helper_commit(struct tx_helper * h)338 static int tx_helper_commit(struct tx_helper *h)
339 {
340 size_t l = 0;
341
342 if (!h->txn.active)
343 return 0;
344
345 if (!WPACKET_get_total_written(&h->txn.wpkt, &l)) {
346 tx_helper_end(h, 0);
347 return 0;
348 }
349
350 if (!tx_helper_append_iovec(h, h->txn.data, l)) {
351 tx_helper_end(h, 0);
352 return 0;
353 }
354
355 if (h->txp->msg_callback != NULL && l > 0) {
356 uint64_t ftype;
357 int ctype = SSL3_RT_QUIC_FRAME_FULL;
358 PACKET pkt;
359
360 if (!PACKET_buf_init(&pkt, h->txn.data, l)
361 || !ossl_quic_wire_peek_frame_header(&pkt, &ftype, NULL)) {
362 tx_helper_end(h, 0);
363 return 0;
364 }
365
366 if (ftype == OSSL_QUIC_FRAME_TYPE_PADDING)
367 ctype = SSL3_RT_QUIC_FRAME_PADDING;
368 else if (OSSL_QUIC_FRAME_TYPE_IS_STREAM(ftype)
369 || ftype == OSSL_QUIC_FRAME_TYPE_CRYPTO)
370 ctype = SSL3_RT_QUIC_FRAME_HEADER;
371
372 h->txp->msg_callback(1, OSSL_QUIC1_VERSION, ctype, h->txn.data, l,
373 h->txp->msg_callback_ssl,
374 h->txp->msg_callback_arg);
375 }
376
377 h->scratch_bytes += l;
378 tx_helper_end(h, 1);
379 return 1;
380 }
381
382 struct archetype_data {
383 unsigned int allow_ack : 1;
384 unsigned int allow_ping : 1;
385 unsigned int allow_crypto : 1;
386 unsigned int allow_handshake_done : 1;
387 unsigned int allow_path_challenge : 1;
388 unsigned int allow_path_response : 1;
389 unsigned int allow_new_conn_id : 1;
390 unsigned int allow_retire_conn_id : 1;
391 unsigned int allow_stream_rel : 1;
392 unsigned int allow_conn_fc : 1;
393 unsigned int allow_conn_close : 1;
394 unsigned int allow_cfq_other : 1;
395 unsigned int allow_new_token : 1;
396 unsigned int allow_force_ack_eliciting : 1;
397 unsigned int allow_padding : 1;
398 unsigned int require_ack_eliciting : 1;
399 unsigned int bypass_cc : 1;
400 };
401
402 struct txp_pkt_geom {
403 size_t cmpl, cmppl, hwm, pkt_overhead;
404 uint32_t archetype;
405 struct archetype_data adata;
406 };
407
408 struct txp_pkt {
409 struct tx_helper h;
410 int h_valid;
411 QUIC_TXPIM_PKT *tpkt;
412 QUIC_STREAM *stream_head;
413 QUIC_PKT_HDR phdr;
414 struct txp_pkt_geom geom;
415 int force_pad;
416 };
417
418 static QUIC_SSTREAM *get_sstream_by_id(uint64_t stream_id, uint32_t pn_space,
419 void *arg);
420 static void on_regen_notify(uint64_t frame_type, uint64_t stream_id,
421 QUIC_TXPIM_PKT *pkt, void *arg);
422 static void on_confirm_notify(uint64_t frame_type, uint64_t stream_id,
423 QUIC_TXPIM_PKT *pkt, void *arg);
424 static void on_sstream_updated(uint64_t stream_id, void *arg);
425 static int sstream_is_pending(QUIC_SSTREAM *sstream);
426 static int txp_should_try_staging(OSSL_QUIC_TX_PACKETISER *txp,
427 uint32_t enc_level,
428 uint32_t archetype,
429 uint64_t cc_limit,
430 uint32_t *conn_close_enc_level);
431 static size_t txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER *txp);
432 static int txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER *txp,
433 size_t pl,
434 uint32_t enc_level,
435 size_t hdr_len,
436 size_t *r);
437 static size_t txp_get_mdpl(OSSL_QUIC_TX_PACKETISER *txp);
438 static int txp_generate_for_el(OSSL_QUIC_TX_PACKETISER *txp,
439 struct txp_pkt *pkt,
440 int chosen_for_conn_close);
441 static int txp_pkt_init(struct txp_pkt *pkt, OSSL_QUIC_TX_PACKETISER *txp,
442 uint32_t enc_level, uint32_t archetype,
443 size_t running_total);
444 static void txp_pkt_cleanup(struct txp_pkt *pkt, OSSL_QUIC_TX_PACKETISER *txp);
445 static int txp_pkt_postgen_update_pkt_overhead(struct txp_pkt *pkt,
446 OSSL_QUIC_TX_PACKETISER *txp);
447 static int txp_pkt_append_padding(struct txp_pkt *pkt,
448 OSSL_QUIC_TX_PACKETISER *txp, size_t num_bytes);
449 static int txp_pkt_commit(OSSL_QUIC_TX_PACKETISER *txp, struct txp_pkt *pkt,
450 uint32_t archetype, int *txpim_pkt_reffed);
451 static uint32_t txp_determine_archetype(OSSL_QUIC_TX_PACKETISER *txp,
452 uint64_t cc_limit);
453
ossl_quic_tx_packetiser_new(const OSSL_QUIC_TX_PACKETISER_ARGS * args)454 OSSL_QUIC_TX_PACKETISER *ossl_quic_tx_packetiser_new(const OSSL_QUIC_TX_PACKETISER_ARGS *args)
455 {
456 OSSL_QUIC_TX_PACKETISER *txp;
457
458 if (args == NULL
459 || args->qtx == NULL
460 || args->txpim == NULL
461 || args->cfq == NULL
462 || args->ackm == NULL
463 || args->qsm == NULL
464 || args->conn_txfc == NULL
465 || args->conn_rxfc == NULL
466 || args->max_streams_bidi_rxfc == NULL
467 || args->max_streams_uni_rxfc == NULL) {
468 ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER);
469 return NULL;
470 }
471
472 txp = OPENSSL_zalloc(sizeof(*txp));
473 if (txp == NULL)
474 return NULL;
475
476 txp->args = *args;
477 txp->last_tx_time = ossl_time_zero();
478
479 if (!ossl_quic_fifd_init(&txp->fifd,
480 txp->args.cfq, txp->args.ackm, txp->args.txpim,
481 get_sstream_by_id, txp,
482 on_regen_notify, txp,
483 on_confirm_notify, txp,
484 on_sstream_updated, txp,
485 args->get_qlog_cb,
486 args->get_qlog_cb_arg)) {
487 OPENSSL_free(txp);
488 return NULL;
489 }
490
491 return txp;
492 }
493
ossl_quic_tx_packetiser_free(OSSL_QUIC_TX_PACKETISER * txp)494 void ossl_quic_tx_packetiser_free(OSSL_QUIC_TX_PACKETISER *txp)
495 {
496 uint32_t enc_level;
497
498 if (txp == NULL)
499 return;
500
501 ossl_quic_tx_packetiser_set_initial_token(txp, NULL, 0, NULL, NULL);
502 ossl_quic_fifd_cleanup(&txp->fifd);
503 OPENSSL_free(txp->conn_close_frame.reason);
504
505 for (enc_level = QUIC_ENC_LEVEL_INITIAL;
506 enc_level < QUIC_ENC_LEVEL_NUM;
507 ++enc_level) {
508 OPENSSL_free(txp->el[enc_level].iovec);
509 OPENSSL_free(txp->el[enc_level].scratch);
510 }
511
512 OPENSSL_free(txp);
513 }
514
515 /*
516 * Determine if an Initial packet token length is reasonable based on the
517 * current MDPL, returning 1 if it is OK.
518 *
519 * The real PMTU to the peer could differ from our (pessimistic) understanding
520 * of the PMTU, therefore it is possible we could receive an Initial token from
521 * a server in a Retry packet which is bigger than the MDPL. In this case it is
522 * impossible for us ever to make forward progress and we need to error out
523 * and fail the connection attempt.
524 *
525 * The specific boundary condition is complex: for example, after the size of
526 * the Initial token, there are the Initial packet header overheads and then
527 * encryption/AEAD tag overheads. After that, the minimum room for frame data in
528 * order to guarantee forward progress must be guaranteed. For example, a crypto
529 * stream needs to always be able to serialize at least one byte in a CRYPTO
530 * frame in order to make forward progress. Because the offset field of a CRYPTO
531 * frame uses a variable-length integer, the number of bytes needed to ensure
532 * this also varies.
533 *
534 * Rather than trying to get this boundary condition check actually right,
535 * require a reasonable amount of slack to avoid pathological behaviours. (After
536 * all, transmitting a CRYPTO stream one byte at a time is probably not
537 * desirable anyway.)
538 *
539 * We choose 160 bytes as the required margin, which is double the rough
540 * estimation of the minimum we would require to guarantee forward progress
541 * under worst case packet overheads.
542 */
543 #define TXP_REQUIRED_TOKEN_MARGIN 160
544
txp_check_token_len(size_t token_len,size_t mdpl)545 static int txp_check_token_len(size_t token_len, size_t mdpl)
546 {
547 if (token_len == 0)
548 return 1;
549
550 if (token_len >= mdpl)
551 return 0;
552
553 if (TXP_REQUIRED_TOKEN_MARGIN >= mdpl)
554 /* (should not be possible because MDPL must be at least 1200) */
555 return 0;
556
557 if (token_len > mdpl - TXP_REQUIRED_TOKEN_MARGIN)
558 return 0;
559
560 return 1;
561 }
562
ossl_quic_tx_packetiser_set_initial_token(OSSL_QUIC_TX_PACKETISER * txp,const unsigned char * token,size_t token_len,ossl_quic_initial_token_free_fn * free_cb,void * free_cb_arg)563 int ossl_quic_tx_packetiser_set_initial_token(OSSL_QUIC_TX_PACKETISER *txp,
564 const unsigned char *token,
565 size_t token_len,
566 ossl_quic_initial_token_free_fn *free_cb,
567 void *free_cb_arg)
568 {
569 if (!txp_check_token_len(token_len, txp_get_mdpl(txp)))
570 return 0;
571
572 if (txp->initial_token != NULL && txp->initial_token_free_cb != NULL)
573 txp->initial_token_free_cb(txp->initial_token, txp->initial_token_len,
574 txp->initial_token_free_cb_arg);
575
576 txp->initial_token = token;
577 txp->initial_token_len = token_len;
578 txp->initial_token_free_cb = free_cb;
579 txp->initial_token_free_cb_arg = free_cb_arg;
580 return 1;
581 }
582
ossl_quic_tx_packetiser_set_cur_dcid(OSSL_QUIC_TX_PACKETISER * txp,const QUIC_CONN_ID * dcid)583 int ossl_quic_tx_packetiser_set_cur_dcid(OSSL_QUIC_TX_PACKETISER *txp,
584 const QUIC_CONN_ID *dcid)
585 {
586 if (dcid == NULL) {
587 ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER);
588 return 0;
589 }
590
591 txp->args.cur_dcid = *dcid;
592 return 1;
593 }
594
ossl_quic_tx_packetiser_set_cur_scid(OSSL_QUIC_TX_PACKETISER * txp,const QUIC_CONN_ID * scid)595 int ossl_quic_tx_packetiser_set_cur_scid(OSSL_QUIC_TX_PACKETISER *txp,
596 const QUIC_CONN_ID *scid)
597 {
598 if (scid == NULL) {
599 ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER);
600 return 0;
601 }
602
603 txp->args.cur_scid = *scid;
604 return 1;
605 }
606
607 /* Change the destination L4 address the TXP uses to send datagrams. */
ossl_quic_tx_packetiser_set_peer(OSSL_QUIC_TX_PACKETISER * txp,const BIO_ADDR * peer)608 int ossl_quic_tx_packetiser_set_peer(OSSL_QUIC_TX_PACKETISER *txp,
609 const BIO_ADDR *peer)
610 {
611 if (peer == NULL) {
612 BIO_ADDR_clear(&txp->args.peer);
613 return 1;
614 }
615
616 txp->args.peer = *peer;
617 return 1;
618 }
619
ossl_quic_tx_packetiser_set_ack_tx_cb(OSSL_QUIC_TX_PACKETISER * txp,void (* cb)(const OSSL_QUIC_FRAME_ACK * ack,uint32_t pn_space,void * arg),void * cb_arg)620 void ossl_quic_tx_packetiser_set_ack_tx_cb(OSSL_QUIC_TX_PACKETISER *txp,
621 void (*cb)(const OSSL_QUIC_FRAME_ACK *ack,
622 uint32_t pn_space,
623 void *arg),
624 void *cb_arg)
625 {
626 txp->ack_tx_cb = cb;
627 txp->ack_tx_cb_arg = cb_arg;
628 }
629
ossl_quic_tx_packetiser_set_qlog_cb(OSSL_QUIC_TX_PACKETISER * txp,QLOG * (* get_qlog_cb)(void * arg),void * get_qlog_cb_arg)630 void ossl_quic_tx_packetiser_set_qlog_cb(OSSL_QUIC_TX_PACKETISER *txp,
631 QLOG *(*get_qlog_cb)(void *arg),
632 void *get_qlog_cb_arg)
633 {
634 ossl_quic_fifd_set_qlog_cb(&txp->fifd, get_qlog_cb, get_qlog_cb_arg);
635
636 }
637
ossl_quic_tx_packetiser_discard_enc_level(OSSL_QUIC_TX_PACKETISER * txp,uint32_t enc_level)638 int ossl_quic_tx_packetiser_discard_enc_level(OSSL_QUIC_TX_PACKETISER *txp,
639 uint32_t enc_level)
640 {
641 if (enc_level >= QUIC_ENC_LEVEL_NUM) {
642 ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT);
643 return 0;
644 }
645
646 if (enc_level != QUIC_ENC_LEVEL_0RTT)
647 txp->args.crypto[ossl_quic_enc_level_to_pn_space(enc_level)] = NULL;
648
649 return 1;
650 }
651
ossl_quic_tx_packetiser_notify_handshake_complete(OSSL_QUIC_TX_PACKETISER * txp)652 void ossl_quic_tx_packetiser_notify_handshake_complete(OSSL_QUIC_TX_PACKETISER *txp)
653 {
654 txp->handshake_complete = 1;
655 }
656
ossl_quic_tx_packetiser_schedule_handshake_done(OSSL_QUIC_TX_PACKETISER * txp)657 void ossl_quic_tx_packetiser_schedule_handshake_done(OSSL_QUIC_TX_PACKETISER *txp)
658 {
659 txp->want_handshake_done = 1;
660 }
661
ossl_quic_tx_packetiser_schedule_ack_eliciting(OSSL_QUIC_TX_PACKETISER * txp,uint32_t pn_space)662 void ossl_quic_tx_packetiser_schedule_ack_eliciting(OSSL_QUIC_TX_PACKETISER *txp,
663 uint32_t pn_space)
664 {
665 txp->force_ack_eliciting |= (1UL << pn_space);
666 }
667
ossl_quic_tx_packetiser_schedule_ack(OSSL_QUIC_TX_PACKETISER * txp,uint32_t pn_space)668 void ossl_quic_tx_packetiser_schedule_ack(OSSL_QUIC_TX_PACKETISER *txp,
669 uint32_t pn_space)
670 {
671 txp->want_ack |= (1UL << pn_space);
672 }
673
674 #define TXP_ERR_INTERNAL 0 /* Internal (e.g. alloc) error */
675 #define TXP_ERR_SUCCESS 1 /* Success */
676 #define TXP_ERR_SPACE 2 /* Not enough room for another packet */
677 #define TXP_ERR_INPUT 3 /* Invalid/malformed input */
678
679 /*
680 * Generates a datagram by polling the various ELs to determine if they want to
681 * generate any frames, and generating a datagram which coalesces packets for
682 * any ELs which do.
683 */
ossl_quic_tx_packetiser_generate(OSSL_QUIC_TX_PACKETISER * txp,QUIC_TXP_STATUS * status)684 int ossl_quic_tx_packetiser_generate(OSSL_QUIC_TX_PACKETISER *txp,
685 QUIC_TXP_STATUS *status)
686 {
687 /*
688 * Called to generate one or more datagrams, each containing one or more
689 * packets.
690 *
691 * There are some tricky things to note here:
692 *
693 * - The TXP is only concerned with generating encrypted packets;
694 * other packets use a different path.
695 *
696 * - Any datagram containing an Initial packet must have a payload length
697 * (DPL) of at least 1200 bytes. This padding need not necessarily be
698 * found in the Initial packet.
699 *
700 * - It is desirable to be able to coalesce an Initial packet
701 * with a Handshake packet. Since, before generating the Handshake
702 * packet, we do not know how long it will be, we cannot know the
703 * correct amount of padding to ensure a DPL of at least 1200 bytes.
704 * Thus this padding must added to the Handshake packet (or whatever
705 * packet is the last in the datagram).
706 *
707 * - However, at the time that we generate the Initial packet,
708 * we do not actually know for sure that we will be followed
709 * in the datagram by another packet. For example, suppose we have
710 * some queued data (e.g. crypto stream data for the HANDSHAKE EL)
711 * it looks like we will want to send on the HANDSHAKE EL.
712 * We could assume padding will be placed in the Handshake packet
713 * subsequently and avoid adding any padding to the Initial packet
714 * (which would leave no room for the Handshake packet in the
715 * datagram).
716 *
717 * However, this is not actually a safe assumption. Suppose that we
718 * are using a link with a MDPL of 1200 bytes, the minimum allowed by
719 * QUIC. Suppose that the Initial packet consumes 1195 bytes in total.
720 * Since it is not possible to fit a Handshake packet in just 5 bytes,
721 * upon trying to add a Handshake packet after generating the Initial
722 * packet, we will discover we have no room to fit it! This is not a
723 * problem in itself as another datagram can be sent subsequently, but
724 * it is a problem because we were counting to use that packet to hold
725 * the essential padding. But if we have already finished encrypting
726 * the Initial packet, we cannot go and add padding to it anymore.
727 * This leaves us stuck.
728 *
729 * Because of this, we have to plan multiple packets simultaneously, such
730 * that we can start generating a Handshake (or 0-RTT or 1-RTT, or so on)
731 * packet while still having the option to go back and add padding to the
732 * Initial packet if it turns out to be needed.
733 *
734 * Trying to predict ahead of time (e.g. during Initial packet generation)
735 * whether we will successfully generate a subsequent packet is fraught with
736 * error as it relies on a large number of variables:
737 *
738 * - Do we have room to fit a packet header? (Consider that due to
739 * variable-length integer encoding this is highly variable and can even
740 * depend on payload length due to a variable-length Length field.)
741 *
742 * - Can we fit even a single one of the frames we want to put in this
743 * packet in the packet? (Each frame type has a bespoke encoding. While
744 * our encodings of some frame types are adaptive based on the available
745 * room - e.g. STREAM frames - ultimately all frame types have some
746 * absolute minimum number of bytes to be successfully encoded. For
747 * example, if after an Initial packet there is enough room to encode
748 * only one byte of frame data, it is quite likely we can't send any of
749 * the frames we wanted to send.) While this is not strictly a problem
750 * because we could just fill the packet with padding frames, this is a
751 * pointless packet and is wasteful.
752 *
753 * Thus we adopt a multi-phase architecture:
754 *
755 * 1. Archetype Selection: Determine desired packet archetype.
756 *
757 * 2. Packet Staging: Generation of packet information and packet payload
758 * data (frame data) into staging areas.
759 *
760 * 3. Packet Adjustment: Adjustment of staged packets, adding padding to
761 * the staged packets if needed.
762 *
763 * 4. Commit: The packets are sent to the QTX and recorded as having been
764 * sent to the FIFM.
765 *
766 */
767 int res = 0, rc;
768 uint32_t archetype, enc_level;
769 uint32_t conn_close_enc_level = QUIC_ENC_LEVEL_NUM;
770 struct txp_pkt pkt[QUIC_ENC_LEVEL_NUM];
771 size_t pkts_done = 0;
772 uint64_t cc_limit = txp->args.cc_method->get_tx_allowance(txp->args.cc_data);
773 int need_padding = 0, txpim_pkt_reffed;
774
775 for (enc_level = QUIC_ENC_LEVEL_INITIAL;
776 enc_level < QUIC_ENC_LEVEL_NUM;
777 ++enc_level)
778 pkt[enc_level].h_valid = 0;
779
780 memset(status, 0, sizeof(*status));
781
782 /*
783 * Should not be needed, but a sanity check in case anyone else has been
784 * using the QTX.
785 */
786 ossl_qtx_finish_dgram(txp->args.qtx);
787
788 /* 1. Archetype Selection */
789 archetype = txp_determine_archetype(txp, cc_limit);
790
791 /* 2. Packet Staging */
792 for (enc_level = QUIC_ENC_LEVEL_INITIAL;
793 enc_level < QUIC_ENC_LEVEL_NUM;
794 ++enc_level) {
795 size_t running_total = (enc_level > QUIC_ENC_LEVEL_INITIAL)
796 ? pkt[enc_level - 1].geom.hwm : 0;
797
798 pkt[enc_level].geom.hwm = running_total;
799
800 if (!txp_should_try_staging(txp, enc_level, archetype, cc_limit,
801 &conn_close_enc_level))
802 continue;
803
804 if (!txp_pkt_init(&pkt[enc_level], txp, enc_level, archetype,
805 running_total))
806 /*
807 * If this fails this is not a fatal error - it means the geometry
808 * planning determined there was not enough space for another
809 * packet. So just proceed with what we've already planned for.
810 */
811 break;
812
813 rc = txp_generate_for_el(txp, &pkt[enc_level],
814 conn_close_enc_level == enc_level);
815 if (rc != TXP_ERR_SUCCESS)
816 goto out;
817
818 if (pkt[enc_level].force_pad)
819 /*
820 * txp_generate_for_el emitted a frame which forces packet padding.
821 */
822 need_padding = 1;
823
824 pkt[enc_level].geom.hwm = running_total
825 + pkt[enc_level].h.bytes_appended
826 + pkt[enc_level].geom.pkt_overhead;
827 }
828
829 /* 3. Packet Adjustment */
830 if (pkt[QUIC_ENC_LEVEL_INITIAL].h_valid
831 && pkt[QUIC_ENC_LEVEL_INITIAL].h.bytes_appended > 0)
832 /*
833 * We have an Initial packet in this datagram, so we need to make sure
834 * the total size of the datagram is adequate.
835 */
836 need_padding = 1;
837
838 if (need_padding) {
839 size_t total_dgram_size = 0;
840 const size_t min_dpl = QUIC_MIN_INITIAL_DGRAM_LEN;
841 uint32_t pad_el = QUIC_ENC_LEVEL_NUM;
842
843 for (enc_level = QUIC_ENC_LEVEL_INITIAL;
844 enc_level < QUIC_ENC_LEVEL_NUM;
845 ++enc_level)
846 if (pkt[enc_level].h_valid && pkt[enc_level].h.bytes_appended > 0) {
847 if (pad_el == QUIC_ENC_LEVEL_NUM
848 /*
849 * We might not be able to add padding, for example if we
850 * are using the ACK_ONLY archetype.
851 */
852 && pkt[enc_level].geom.adata.allow_padding
853 && !pkt[enc_level].h.done_implicit)
854 pad_el = enc_level;
855
856 txp_pkt_postgen_update_pkt_overhead(&pkt[enc_level], txp);
857 total_dgram_size += pkt[enc_level].geom.pkt_overhead
858 + pkt[enc_level].h.bytes_appended;
859 }
860
861 if (pad_el != QUIC_ENC_LEVEL_NUM && total_dgram_size < min_dpl) {
862 size_t deficit = min_dpl - total_dgram_size;
863
864 if (!txp_pkt_append_padding(&pkt[pad_el], txp, deficit))
865 goto out;
866
867 total_dgram_size += deficit;
868
869 /*
870 * Padding frames make a packet ineligible for being a non-inflight
871 * packet.
872 */
873 pkt[pad_el].tpkt->ackm_pkt.is_inflight = 1;
874 }
875
876 /*
877 * If we have failed to make a datagram of adequate size, for example
878 * because we have a padding requirement but are using the ACK_ONLY
879 * archetype (because we are CC limited), which precludes us from
880 * sending padding, give up on generating the datagram - there is
881 * nothing we can do.
882 */
883 if (total_dgram_size < min_dpl) {
884 res = 1;
885 goto out;
886 }
887 }
888
889 /* 4. Commit */
890 for (enc_level = QUIC_ENC_LEVEL_INITIAL;
891 enc_level < QUIC_ENC_LEVEL_NUM;
892 ++enc_level) {
893
894 if (!pkt[enc_level].h_valid)
895 /* Did not attempt to generate a packet for this EL. */
896 continue;
897
898 if (pkt[enc_level].h.bytes_appended == 0)
899 /* Nothing was generated for this EL, so skip. */
900 continue;
901
902 rc = txp_pkt_commit(txp, &pkt[enc_level], archetype,
903 &txpim_pkt_reffed);
904 if (rc) {
905 status->sent_ack_eliciting
906 = status->sent_ack_eliciting
907 || pkt[enc_level].tpkt->ackm_pkt.is_ack_eliciting;
908
909 if (enc_level == QUIC_ENC_LEVEL_HANDSHAKE)
910 status->sent_handshake
911 = (pkt[enc_level].h_valid
912 && pkt[enc_level].h.bytes_appended > 0);
913 }
914
915 if (txpim_pkt_reffed)
916 pkt[enc_level].tpkt = NULL; /* don't free */
917
918 if (!rc)
919 goto out;
920
921 ++pkts_done;
922 }
923
924 /* Flush & Cleanup */
925 res = 1;
926 out:
927 ossl_qtx_finish_dgram(txp->args.qtx);
928
929 for (enc_level = QUIC_ENC_LEVEL_INITIAL;
930 enc_level < QUIC_ENC_LEVEL_NUM;
931 ++enc_level)
932 txp_pkt_cleanup(&pkt[enc_level], txp);
933
934 status->sent_pkt = pkts_done;
935
936 return res;
937 }
938
939 static const struct archetype_data archetypes[QUIC_ENC_LEVEL_NUM][TX_PACKETISER_ARCHETYPE_NUM] = {
940 /* EL 0(INITIAL) */
941 {
942 /* EL 0(INITIAL) - Archetype 0(NORMAL) */
943 {
944 /*allow_ack =*/ 1,
945 /*allow_ping =*/ 1,
946 /*allow_crypto =*/ 1,
947 /*allow_handshake_done =*/ 0,
948 /*allow_path_challenge =*/ 0,
949 /*allow_path_response =*/ 0,
950 /*allow_new_conn_id =*/ 0,
951 /*allow_retire_conn_id =*/ 0,
952 /*allow_stream_rel =*/ 0,
953 /*allow_conn_fc =*/ 0,
954 /*allow_conn_close =*/ 1,
955 /*allow_cfq_other =*/ 0,
956 /*allow_new_token =*/ 0,
957 /*allow_force_ack_eliciting =*/ 1,
958 /*allow_padding =*/ 1,
959 /*require_ack_eliciting =*/ 0,
960 /*bypass_cc =*/ 0,
961 },
962 /* EL 0(INITIAL) - Archetype 1(PROBE) */
963 {
964 /*allow_ack =*/ 1,
965 /*allow_ping =*/ 1,
966 /*allow_crypto =*/ 1,
967 /*allow_handshake_done =*/ 0,
968 /*allow_path_challenge =*/ 0,
969 /*allow_path_response =*/ 0,
970 /*allow_new_conn_id =*/ 0,
971 /*allow_retire_conn_id =*/ 0,
972 /*allow_stream_rel =*/ 0,
973 /*allow_conn_fc =*/ 0,
974 /*allow_conn_close =*/ 1,
975 /*allow_cfq_other =*/ 0,
976 /*allow_new_token =*/ 0,
977 /*allow_force_ack_eliciting =*/ 1,
978 /*allow_padding =*/ 1,
979 /*require_ack_eliciting =*/ 1,
980 /*bypass_cc =*/ 1,
981 },
982 /* EL 0(INITIAL) - Archetype 2(ACK_ONLY) */
983 {
984 /*allow_ack =*/ 1,
985 /*allow_ping =*/ 0,
986 /*allow_crypto =*/ 0,
987 /*allow_handshake_done =*/ 0,
988 /*allow_path_challenge =*/ 0,
989 /*allow_path_response =*/ 0,
990 /*allow_new_conn_id =*/ 0,
991 /*allow_retire_conn_id =*/ 0,
992 /*allow_stream_rel =*/ 0,
993 /*allow_conn_fc =*/ 0,
994 /*allow_conn_close =*/ 0,
995 /*allow_cfq_other =*/ 0,
996 /*allow_new_token =*/ 0,
997 /*allow_force_ack_eliciting =*/ 1,
998 /*allow_padding =*/ 0,
999 /*require_ack_eliciting =*/ 0,
1000 /*bypass_cc =*/ 1,
1001 },
1002 },
1003 /* EL 1(HANDSHAKE) */
1004 {
1005 /* EL 1(HANDSHAKE) - Archetype 0(NORMAL) */
1006 {
1007 /*allow_ack =*/ 1,
1008 /*allow_ping =*/ 1,
1009 /*allow_crypto =*/ 1,
1010 /*allow_handshake_done =*/ 0,
1011 /*allow_path_challenge =*/ 0,
1012 /*allow_path_response =*/ 0,
1013 /*allow_new_conn_id =*/ 0,
1014 /*allow_retire_conn_id =*/ 0,
1015 /*allow_stream_rel =*/ 0,
1016 /*allow_conn_fc =*/ 0,
1017 /*allow_conn_close =*/ 1,
1018 /*allow_cfq_other =*/ 0,
1019 /*allow_new_token =*/ 0,
1020 /*allow_force_ack_eliciting =*/ 1,
1021 /*allow_padding =*/ 1,
1022 /*require_ack_eliciting =*/ 0,
1023 /*bypass_cc =*/ 0,
1024 },
1025 /* EL 1(HANDSHAKE) - Archetype 1(PROBE) */
1026 {
1027 /*allow_ack =*/ 1,
1028 /*allow_ping =*/ 1,
1029 /*allow_crypto =*/ 1,
1030 /*allow_handshake_done =*/ 0,
1031 /*allow_path_challenge =*/ 0,
1032 /*allow_path_response =*/ 0,
1033 /*allow_new_conn_id =*/ 0,
1034 /*allow_retire_conn_id =*/ 0,
1035 /*allow_stream_rel =*/ 0,
1036 /*allow_conn_fc =*/ 0,
1037 /*allow_conn_close =*/ 1,
1038 /*allow_cfq_other =*/ 0,
1039 /*allow_new_token =*/ 0,
1040 /*allow_force_ack_eliciting =*/ 1,
1041 /*allow_padding =*/ 1,
1042 /*require_ack_eliciting =*/ 1,
1043 /*bypass_cc =*/ 1,
1044 },
1045 /* EL 1(HANDSHAKE) - Archetype 2(ACK_ONLY) */
1046 {
1047 /*allow_ack =*/ 1,
1048 /*allow_ping =*/ 0,
1049 /*allow_crypto =*/ 0,
1050 /*allow_handshake_done =*/ 0,
1051 /*allow_path_challenge =*/ 0,
1052 /*allow_path_response =*/ 0,
1053 /*allow_new_conn_id =*/ 0,
1054 /*allow_retire_conn_id =*/ 0,
1055 /*allow_stream_rel =*/ 0,
1056 /*allow_conn_fc =*/ 0,
1057 /*allow_conn_close =*/ 0,
1058 /*allow_cfq_other =*/ 0,
1059 /*allow_new_token =*/ 0,
1060 /*allow_force_ack_eliciting =*/ 1,
1061 /*allow_padding =*/ 0,
1062 /*require_ack_eliciting =*/ 0,
1063 /*bypass_cc =*/ 1,
1064 },
1065 },
1066 /* EL 2(0RTT) */
1067 {
1068 /* EL 2(0RTT) - Archetype 0(NORMAL) */
1069 {
1070 /*allow_ack =*/ 0,
1071 /*allow_ping =*/ 1,
1072 /*allow_crypto =*/ 0,
1073 /*allow_handshake_done =*/ 0,
1074 /*allow_path_challenge =*/ 0,
1075 /*allow_path_response =*/ 0,
1076 /*allow_new_conn_id =*/ 1,
1077 /*allow_retire_conn_id =*/ 1,
1078 /*allow_stream_rel =*/ 1,
1079 /*allow_conn_fc =*/ 1,
1080 /*allow_conn_close =*/ 1,
1081 /*allow_cfq_other =*/ 0,
1082 /*allow_new_token =*/ 0,
1083 /*allow_force_ack_eliciting =*/ 0,
1084 /*allow_padding =*/ 1,
1085 /*require_ack_eliciting =*/ 0,
1086 /*bypass_cc =*/ 0,
1087 },
1088 /* EL 2(0RTT) - Archetype 1(PROBE) */
1089 {
1090 /*allow_ack =*/ 0,
1091 /*allow_ping =*/ 1,
1092 /*allow_crypto =*/ 0,
1093 /*allow_handshake_done =*/ 0,
1094 /*allow_path_challenge =*/ 0,
1095 /*allow_path_response =*/ 0,
1096 /*allow_new_conn_id =*/ 1,
1097 /*allow_retire_conn_id =*/ 1,
1098 /*allow_stream_rel =*/ 1,
1099 /*allow_conn_fc =*/ 1,
1100 /*allow_conn_close =*/ 1,
1101 /*allow_cfq_other =*/ 0,
1102 /*allow_new_token =*/ 0,
1103 /*allow_force_ack_eliciting =*/ 0,
1104 /*allow_padding =*/ 1,
1105 /*require_ack_eliciting =*/ 1,
1106 /*bypass_cc =*/ 1,
1107 },
1108 /* EL 2(0RTT) - Archetype 2(ACK_ONLY) */
1109 {
1110 /*allow_ack =*/ 0,
1111 /*allow_ping =*/ 0,
1112 /*allow_crypto =*/ 0,
1113 /*allow_handshake_done =*/ 0,
1114 /*allow_path_challenge =*/ 0,
1115 /*allow_path_response =*/ 0,
1116 /*allow_new_conn_id =*/ 0,
1117 /*allow_retire_conn_id =*/ 0,
1118 /*allow_stream_rel =*/ 0,
1119 /*allow_conn_fc =*/ 0,
1120 /*allow_conn_close =*/ 0,
1121 /*allow_cfq_other =*/ 0,
1122 /*allow_new_token =*/ 0,
1123 /*allow_force_ack_eliciting =*/ 0,
1124 /*allow_padding =*/ 0,
1125 /*require_ack_eliciting =*/ 0,
1126 /*bypass_cc =*/ 1,
1127 },
1128 },
1129 /* EL 3(1RTT) */
1130 {
1131 /* EL 3(1RTT) - Archetype 0(NORMAL) */
1132 {
1133 /*allow_ack =*/ 1,
1134 /*allow_ping =*/ 1,
1135 /*allow_crypto =*/ 1,
1136 /*allow_handshake_done =*/ 1,
1137 /*allow_path_challenge =*/ 0,
1138 /*allow_path_response =*/ 1,
1139 /*allow_new_conn_id =*/ 1,
1140 /*allow_retire_conn_id =*/ 1,
1141 /*allow_stream_rel =*/ 1,
1142 /*allow_conn_fc =*/ 1,
1143 /*allow_conn_close =*/ 1,
1144 /*allow_cfq_other =*/ 1,
1145 /*allow_new_token =*/ 1,
1146 /*allow_force_ack_eliciting =*/ 1,
1147 /*allow_padding =*/ 1,
1148 /*require_ack_eliciting =*/ 0,
1149 /*bypass_cc =*/ 0,
1150 },
1151 /* EL 3(1RTT) - Archetype 1(PROBE) */
1152 {
1153 /*allow_ack =*/ 1,
1154 /*allow_ping =*/ 1,
1155 /*allow_crypto =*/ 1,
1156 /*allow_handshake_done =*/ 1,
1157 /*allow_path_challenge =*/ 0,
1158 /*allow_path_response =*/ 1,
1159 /*allow_new_conn_id =*/ 1,
1160 /*allow_retire_conn_id =*/ 1,
1161 /*allow_stream_rel =*/ 1,
1162 /*allow_conn_fc =*/ 1,
1163 /*allow_conn_close =*/ 1,
1164 /*allow_cfq_other =*/ 1,
1165 /*allow_new_token =*/ 1,
1166 /*allow_force_ack_eliciting =*/ 1,
1167 /*allow_padding =*/ 1,
1168 /*require_ack_eliciting =*/ 1,
1169 /*bypass_cc =*/ 1,
1170 },
1171 /* EL 3(1RTT) - Archetype 2(ACK_ONLY) */
1172 {
1173 /*allow_ack =*/ 1,
1174 /*allow_ping =*/ 0,
1175 /*allow_crypto =*/ 0,
1176 /*allow_handshake_done =*/ 0,
1177 /*allow_path_challenge =*/ 0,
1178 /*allow_path_response =*/ 0,
1179 /*allow_new_conn_id =*/ 0,
1180 /*allow_retire_conn_id =*/ 0,
1181 /*allow_stream_rel =*/ 0,
1182 /*allow_conn_fc =*/ 0,
1183 /*allow_conn_close =*/ 0,
1184 /*allow_cfq_other =*/ 0,
1185 /*allow_new_token =*/ 0,
1186 /*allow_force_ack_eliciting =*/ 1,
1187 /*allow_padding =*/ 0,
1188 /*require_ack_eliciting =*/ 0,
1189 /*bypass_cc =*/ 1,
1190 }
1191 }
1192 };
1193
txp_get_archetype_data(uint32_t enc_level,uint32_t archetype,struct archetype_data * a)1194 static int txp_get_archetype_data(uint32_t enc_level,
1195 uint32_t archetype,
1196 struct archetype_data *a)
1197 {
1198 if (enc_level >= QUIC_ENC_LEVEL_NUM
1199 || archetype >= TX_PACKETISER_ARCHETYPE_NUM)
1200 return 0;
1201
1202 /* No need to avoid copying this as it should not exceed one int in size. */
1203 *a = archetypes[enc_level][archetype];
1204 return 1;
1205 }
1206
txp_determine_geometry(OSSL_QUIC_TX_PACKETISER * txp,uint32_t archetype,uint32_t enc_level,size_t running_total,QUIC_PKT_HDR * phdr,struct txp_pkt_geom * geom)1207 static int txp_determine_geometry(OSSL_QUIC_TX_PACKETISER *txp,
1208 uint32_t archetype,
1209 uint32_t enc_level,
1210 size_t running_total,
1211 QUIC_PKT_HDR *phdr,
1212 struct txp_pkt_geom *geom)
1213 {
1214 size_t mdpl, cmpl, hdr_len;
1215
1216 /* Get information about packet archetype. */
1217 if (!txp_get_archetype_data(enc_level, archetype, &geom->adata))
1218 return 0;
1219
1220 /* Assemble packet header. */
1221 phdr->type = ossl_quic_enc_level_to_pkt_type(enc_level);
1222 phdr->spin_bit = 0;
1223 phdr->pn_len = txp_determine_pn_len(txp);
1224 phdr->partial = 0;
1225 phdr->fixed = 1;
1226 phdr->reserved = 0;
1227 phdr->version = QUIC_VERSION_1;
1228 phdr->dst_conn_id = txp->args.cur_dcid;
1229 phdr->src_conn_id = txp->args.cur_scid;
1230
1231 /*
1232 * We need to know the length of the payload to get an accurate header
1233 * length for non-1RTT packets, because the Length field found in
1234 * Initial/Handshake/0-RTT packets uses a variable-length encoding. However,
1235 * we don't have a good idea of the length of our payload, because the
1236 * length of the payload depends on the room in the datagram after fitting
1237 * the header, which depends on the size of the header.
1238 *
1239 * In general, it does not matter if a packet is slightly shorter (because
1240 * e.g. we predicted use of a 2-byte length field, but ended up only needing
1241 * a 1-byte length field). However this does matter for Initial packets
1242 * which must be at least 1200 bytes, which is also the assumed default MTU;
1243 * therefore in many cases Initial packets will be padded to 1200 bytes,
1244 * which means if we overestimated the header size, we will be short by a
1245 * few bytes and the server will ignore the packet for being too short. In
1246 * this case, however, such packets always *will* be padded to meet 1200
1247 * bytes, which requires a 2-byte length field, so we don't actually need to
1248 * worry about this. Thus we estimate the header length assuming a 2-byte
1249 * length field here, which should in practice work well in all cases.
1250 */
1251 phdr->len = OSSL_QUIC_VLINT_2B_MAX - phdr->pn_len;
1252
1253 if (enc_level == QUIC_ENC_LEVEL_INITIAL) {
1254 phdr->token = txp->initial_token;
1255 phdr->token_len = txp->initial_token_len;
1256 } else {
1257 phdr->token = NULL;
1258 phdr->token_len = 0;
1259 }
1260
1261 hdr_len = ossl_quic_wire_get_encoded_pkt_hdr_len(phdr->dst_conn_id.id_len,
1262 phdr);
1263 if (hdr_len == 0)
1264 return 0;
1265
1266 /* MDPL: Maximum datagram payload length. */
1267 mdpl = txp_get_mdpl(txp);
1268
1269 /*
1270 * CMPL: Maximum encoded packet size we can put into this datagram given any
1271 * previous packets coalesced into it.
1272 */
1273 if (running_total > mdpl)
1274 /* Should not be possible, but if it happens: */
1275 cmpl = 0;
1276 else
1277 cmpl = mdpl - running_total;
1278
1279 /* CMPPL: Maximum amount we can put into the current packet payload */
1280 if (!txp_determine_ppl_from_pl(txp, cmpl, enc_level, hdr_len, &geom->cmppl))
1281 return 0;
1282
1283 geom->cmpl = cmpl;
1284 geom->pkt_overhead = cmpl - geom->cmppl;
1285 geom->archetype = archetype;
1286 return 1;
1287 }
1288
txp_determine_archetype(OSSL_QUIC_TX_PACKETISER * txp,uint64_t cc_limit)1289 static uint32_t txp_determine_archetype(OSSL_QUIC_TX_PACKETISER *txp,
1290 uint64_t cc_limit)
1291 {
1292 OSSL_ACKM_PROBE_INFO *probe_info
1293 = ossl_ackm_get0_probe_request(txp->args.ackm);
1294 uint32_t pn_space;
1295
1296 /*
1297 * If ACKM has requested probe generation (e.g. due to PTO), we generate a
1298 * Probe-archetype packet. Actually, we determine archetype on a
1299 * per-datagram basis, so if any EL wants a probe, do a pass in which
1300 * we try and generate a probe (if needed) for all ELs.
1301 */
1302 if (probe_info->anti_deadlock_initial > 0
1303 || probe_info->anti_deadlock_handshake > 0)
1304 return TX_PACKETISER_ARCHETYPE_PROBE;
1305
1306 for (pn_space = QUIC_PN_SPACE_INITIAL;
1307 pn_space < QUIC_PN_SPACE_NUM;
1308 ++pn_space)
1309 if (probe_info->pto[pn_space] > 0)
1310 return TX_PACKETISER_ARCHETYPE_PROBE;
1311
1312 /*
1313 * If we are out of CC budget, we cannot send a normal packet,
1314 * but we can do an ACK-only packet (potentially, if we
1315 * want to send an ACK).
1316 */
1317 if (cc_limit == 0)
1318 return TX_PACKETISER_ARCHETYPE_ACK_ONLY;
1319
1320 /* All other packets. */
1321 return TX_PACKETISER_ARCHETYPE_NORMAL;
1322 }
1323
txp_should_try_staging(OSSL_QUIC_TX_PACKETISER * txp,uint32_t enc_level,uint32_t archetype,uint64_t cc_limit,uint32_t * conn_close_enc_level)1324 static int txp_should_try_staging(OSSL_QUIC_TX_PACKETISER *txp,
1325 uint32_t enc_level,
1326 uint32_t archetype,
1327 uint64_t cc_limit,
1328 uint32_t *conn_close_enc_level)
1329 {
1330 struct archetype_data a;
1331 uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
1332 QUIC_CFQ_ITEM *cfq_item;
1333
1334 if (!ossl_qtx_is_enc_level_provisioned(txp->args.qtx, enc_level))
1335 return 0;
1336
1337 if (!txp_get_archetype_data(enc_level, archetype, &a))
1338 return 0;
1339
1340 if (!a.bypass_cc && cc_limit == 0)
1341 /* CC not allowing us to send. */
1342 return 0;
1343
1344 /*
1345 * We can produce CONNECTION_CLOSE frames on any EL in principle, which
1346 * means we need to choose which EL we would prefer to use. After a
1347 * connection is fully established we have only one provisioned EL and this
1348 * is a non-issue. Where multiple ELs are provisioned, it is possible the
1349 * peer does not have the keys for the EL yet, which suggests in general it
1350 * is preferable to use the lowest EL which is still provisioned.
1351 *
1352 * However (RFC 9000 s. 10.2.3 & 12.5) we are also required to not send
1353 * application CONNECTION_CLOSE frames in non-1-RTT ELs, so as to not
1354 * potentially leak application data on a connection which has yet to be
1355 * authenticated. Thus when we have an application CONNECTION_CLOSE frame
1356 * queued and need to send it on a non-1-RTT EL, we have to convert it
1357 * into a transport CONNECTION_CLOSE frame which contains no application
1358 * data. Since this loses information, it suggests we should use the 1-RTT
1359 * EL to avoid this if possible, even if a lower EL is also available.
1360 *
1361 * At the same time, just because we have the 1-RTT EL provisioned locally
1362 * does not necessarily mean the peer does, for example if a handshake
1363 * CRYPTO frame has been lost. It is fairly important that CONNECTION_CLOSE
1364 * is signalled in a way we know our peer can decrypt, as we stop processing
1365 * connection retransmission logic for real after connection close and
1366 * simply 'blindly' retransmit the same CONNECTION_CLOSE frame.
1367 *
1368 * This is not a major concern for clients, since if a client has a 1-RTT EL
1369 * provisioned the server is guaranteed to also have a 1-RTT EL provisioned.
1370 *
1371 * TODO(QUIC SERVER): Revisit this when server support is added.
1372 */
1373 if (*conn_close_enc_level > enc_level
1374 && *conn_close_enc_level != QUIC_ENC_LEVEL_1RTT)
1375 *conn_close_enc_level = enc_level;
1376
1377 /* Do we need to send a PTO probe? */
1378 if (a.allow_force_ack_eliciting) {
1379 OSSL_ACKM_PROBE_INFO *probe_info
1380 = ossl_ackm_get0_probe_request(txp->args.ackm);
1381
1382 if ((enc_level == QUIC_ENC_LEVEL_INITIAL
1383 && probe_info->anti_deadlock_initial > 0)
1384 || (enc_level == QUIC_ENC_LEVEL_HANDSHAKE
1385 && probe_info->anti_deadlock_handshake > 0)
1386 || probe_info->pto[pn_space] > 0)
1387 return 1;
1388 }
1389
1390 /* Does the crypto stream for this EL want to produce anything? */
1391 if (a.allow_crypto && sstream_is_pending(txp->args.crypto[pn_space]))
1392 return 1;
1393
1394 /* Does the ACKM for this PN space want to produce anything? */
1395 if (a.allow_ack && (ossl_ackm_is_ack_desired(txp->args.ackm, pn_space)
1396 || (txp->want_ack & (1UL << pn_space)) != 0))
1397 return 1;
1398
1399 /* Do we need to force emission of an ACK-eliciting packet? */
1400 if (a.allow_force_ack_eliciting
1401 && (txp->force_ack_eliciting & (1UL << pn_space)) != 0)
1402 return 1;
1403
1404 /* Does the connection-level RXFC want to produce a frame? */
1405 if (a.allow_conn_fc && (txp->want_max_data
1406 || ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 0)))
1407 return 1;
1408
1409 /* Do we want to produce a MAX_STREAMS frame? */
1410 if (a.allow_conn_fc
1411 && (txp->want_max_streams_bidi
1412 || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_bidi_rxfc,
1413 0)
1414 || txp->want_max_streams_uni
1415 || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_uni_rxfc,
1416 0)))
1417 return 1;
1418
1419 /* Do we want to produce a HANDSHAKE_DONE frame? */
1420 if (a.allow_handshake_done && txp->want_handshake_done)
1421 return 1;
1422
1423 /* Do we want to produce a CONNECTION_CLOSE frame? */
1424 if (a.allow_conn_close && txp->want_conn_close &&
1425 *conn_close_enc_level == enc_level)
1426 /*
1427 * This is a bit of a special case since CONNECTION_CLOSE can appear in
1428 * most packet types, and when we decide we want to send it this status
1429 * isn't tied to a specific EL. So if we want to send it, we send it
1430 * only on the lowest non-dropped EL.
1431 */
1432 return 1;
1433
1434 /* Does the CFQ have any frames queued for this PN space? */
1435 if (enc_level != QUIC_ENC_LEVEL_0RTT)
1436 for (cfq_item = ossl_quic_cfq_get_priority_head(txp->args.cfq, pn_space);
1437 cfq_item != NULL;
1438 cfq_item = ossl_quic_cfq_item_get_priority_next(cfq_item, pn_space)) {
1439 uint64_t frame_type = ossl_quic_cfq_item_get_frame_type(cfq_item);
1440
1441 switch (frame_type) {
1442 case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID:
1443 if (a.allow_new_conn_id)
1444 return 1;
1445 break;
1446 case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID:
1447 if (a.allow_retire_conn_id)
1448 return 1;
1449 break;
1450 case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN:
1451 if (a.allow_new_token)
1452 return 1;
1453 break;
1454 case OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE:
1455 if (a.allow_path_response)
1456 return 1;
1457 break;
1458 default:
1459 if (a.allow_cfq_other)
1460 return 1;
1461 break;
1462 }
1463 }
1464
1465 if (a.allow_stream_rel && txp->handshake_complete) {
1466 QUIC_STREAM_ITER it;
1467
1468 /* If there are any active streams, 0/1-RTT wants to produce a packet.
1469 * Whether a stream is on the active list is required to be precise
1470 * (i.e., a stream is never on the active list if we cannot produce a
1471 * frame for it), and all stream-related frames are governed by
1472 * a.allow_stream_rel (i.e., if we can send one type of stream-related
1473 * frame, we can send any of them), so we don't need to inspect
1474 * individual streams on the active list, just confirm that the active
1475 * list is non-empty.
1476 */
1477 ossl_quic_stream_iter_init(&it, txp->args.qsm, 0);
1478 if (it.stream != NULL)
1479 return 1;
1480 }
1481
1482 return 0;
1483 }
1484
sstream_is_pending(QUIC_SSTREAM * sstream)1485 static int sstream_is_pending(QUIC_SSTREAM *sstream)
1486 {
1487 OSSL_QUIC_FRAME_STREAM hdr;
1488 OSSL_QTX_IOVEC iov[2];
1489 size_t num_iov = OSSL_NELEM(iov);
1490
1491 return ossl_quic_sstream_get_stream_frame(sstream, 0, &hdr, iov, &num_iov);
1492 }
1493
1494 /* Determine how many bytes we should use for the encoded PN. */
txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER * txp)1495 static size_t txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER *txp)
1496 {
1497 return 4; /* TODO(QUIC FUTURE) */
1498 }
1499
1500 /* Determine plaintext packet payload length from payload length. */
txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER * txp,size_t pl,uint32_t enc_level,size_t hdr_len,size_t * r)1501 static int txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER *txp,
1502 size_t pl,
1503 uint32_t enc_level,
1504 size_t hdr_len,
1505 size_t *r)
1506 {
1507 if (pl < hdr_len)
1508 return 0;
1509
1510 pl -= hdr_len;
1511
1512 if (!ossl_qtx_calculate_plaintext_payload_len(txp->args.qtx, enc_level,
1513 pl, &pl))
1514 return 0;
1515
1516 *r = pl;
1517 return 1;
1518 }
1519
txp_get_mdpl(OSSL_QUIC_TX_PACKETISER * txp)1520 static size_t txp_get_mdpl(OSSL_QUIC_TX_PACKETISER *txp)
1521 {
1522 return ossl_qtx_get_mdpl(txp->args.qtx);
1523 }
1524
get_sstream_by_id(uint64_t stream_id,uint32_t pn_space,void * arg)1525 static QUIC_SSTREAM *get_sstream_by_id(uint64_t stream_id, uint32_t pn_space,
1526 void *arg)
1527 {
1528 OSSL_QUIC_TX_PACKETISER *txp = arg;
1529 QUIC_STREAM *s;
1530
1531 if (stream_id == UINT64_MAX)
1532 return txp->args.crypto[pn_space];
1533
1534 s = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1535 if (s == NULL)
1536 return NULL;
1537
1538 return s->sstream;
1539 }
1540
on_regen_notify(uint64_t frame_type,uint64_t stream_id,QUIC_TXPIM_PKT * pkt,void * arg)1541 static void on_regen_notify(uint64_t frame_type, uint64_t stream_id,
1542 QUIC_TXPIM_PKT *pkt, void *arg)
1543 {
1544 OSSL_QUIC_TX_PACKETISER *txp = arg;
1545
1546 switch (frame_type) {
1547 case OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE:
1548 txp->want_handshake_done = 1;
1549 break;
1550 case OSSL_QUIC_FRAME_TYPE_MAX_DATA:
1551 txp->want_max_data = 1;
1552 break;
1553 case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI:
1554 txp->want_max_streams_bidi = 1;
1555 break;
1556 case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_UNI:
1557 txp->want_max_streams_uni = 1;
1558 break;
1559 case OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN:
1560 txp->want_ack |= (1UL << pkt->ackm_pkt.pkt_space);
1561 break;
1562 case OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA:
1563 {
1564 QUIC_STREAM *s
1565 = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1566
1567 if (s == NULL)
1568 return;
1569
1570 s->want_max_stream_data = 1;
1571 ossl_quic_stream_map_update_state(txp->args.qsm, s);
1572 }
1573 break;
1574 case OSSL_QUIC_FRAME_TYPE_STOP_SENDING:
1575 {
1576 QUIC_STREAM *s
1577 = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1578
1579 if (s == NULL)
1580 return;
1581
1582 ossl_quic_stream_map_schedule_stop_sending(txp->args.qsm, s);
1583 }
1584 break;
1585 case OSSL_QUIC_FRAME_TYPE_RESET_STREAM:
1586 {
1587 QUIC_STREAM *s
1588 = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1589
1590 if (s == NULL)
1591 return;
1592
1593 s->want_reset_stream = 1;
1594 ossl_quic_stream_map_update_state(txp->args.qsm, s);
1595 }
1596 break;
1597 default:
1598 assert(0);
1599 break;
1600 }
1601 }
1602
txp_need_ping(OSSL_QUIC_TX_PACKETISER * txp,uint32_t pn_space,const struct archetype_data * adata)1603 static int txp_need_ping(OSSL_QUIC_TX_PACKETISER *txp,
1604 uint32_t pn_space,
1605 const struct archetype_data *adata)
1606 {
1607 return adata->allow_ping
1608 && (adata->require_ack_eliciting
1609 || (txp->force_ack_eliciting & (1UL << pn_space)) != 0);
1610 }
1611
txp_pkt_init(struct txp_pkt * pkt,OSSL_QUIC_TX_PACKETISER * txp,uint32_t enc_level,uint32_t archetype,size_t running_total)1612 static int txp_pkt_init(struct txp_pkt *pkt, OSSL_QUIC_TX_PACKETISER *txp,
1613 uint32_t enc_level, uint32_t archetype,
1614 size_t running_total)
1615 {
1616 uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
1617
1618 if (!txp_determine_geometry(txp, archetype, enc_level,
1619 running_total, &pkt->phdr, &pkt->geom))
1620 return 0;
1621
1622 /*
1623 * Initialise TX helper. If we must be ACK eliciting, reserve 1 byte for
1624 * PING.
1625 */
1626 if (!tx_helper_init(&pkt->h, txp, enc_level,
1627 pkt->geom.cmppl,
1628 txp_need_ping(txp, pn_space, &pkt->geom.adata) ? 1 : 0))
1629 return 0;
1630
1631 pkt->h_valid = 1;
1632 pkt->tpkt = NULL;
1633 pkt->stream_head = NULL;
1634 pkt->force_pad = 0;
1635 return 1;
1636 }
1637
txp_pkt_cleanup(struct txp_pkt * pkt,OSSL_QUIC_TX_PACKETISER * txp)1638 static void txp_pkt_cleanup(struct txp_pkt *pkt, OSSL_QUIC_TX_PACKETISER *txp)
1639 {
1640 if (!pkt->h_valid)
1641 return;
1642
1643 tx_helper_cleanup(&pkt->h);
1644 pkt->h_valid = 0;
1645
1646 if (pkt->tpkt != NULL) {
1647 ossl_quic_txpim_pkt_release(txp->args.txpim, pkt->tpkt);
1648 pkt->tpkt = NULL;
1649 }
1650 }
1651
txp_pkt_postgen_update_pkt_overhead(struct txp_pkt * pkt,OSSL_QUIC_TX_PACKETISER * txp)1652 static int txp_pkt_postgen_update_pkt_overhead(struct txp_pkt *pkt,
1653 OSSL_QUIC_TX_PACKETISER *txp)
1654 {
1655 /*
1656 * After we have staged and generated our packets, but before we commit
1657 * them, it is possible for the estimated packet overhead (packet header +
1658 * AEAD tag size) to shrink slightly because we generated a short packet
1659 * whose which can be represented in fewer bytes as a variable-length
1660 * integer than we were (pessimistically) budgeting for. We need to account
1661 * for this to ensure that we get our padding calculation exactly right.
1662 *
1663 * Update pkt_overhead to be accurate now that we know how much data is
1664 * going in a packet.
1665 */
1666 size_t hdr_len, ciphertext_len;
1667
1668 if (pkt->h.enc_level == QUIC_ENC_LEVEL_INITIAL)
1669 /*
1670 * Don't update overheads for the INITIAL EL - we have not finished
1671 * appending padding to it and would potentially miscalculate the
1672 * correct padding if we now update the pkt_overhead field to switch to
1673 * e.g. a 1-byte length field in the packet header. Since we are padding
1674 * to QUIC_MIN_INITIAL_DGRAM_LEN which requires a 2-byte length field,
1675 * this is guaranteed to be moot anyway. See comment in
1676 * txp_determine_geometry for more information.
1677 */
1678 return 1;
1679
1680 if (!ossl_qtx_calculate_ciphertext_payload_len(txp->args.qtx, pkt->h.enc_level,
1681 pkt->h.bytes_appended,
1682 &ciphertext_len))
1683 return 0;
1684
1685 pkt->phdr.len = ciphertext_len;
1686
1687 hdr_len = ossl_quic_wire_get_encoded_pkt_hdr_len(pkt->phdr.dst_conn_id.id_len,
1688 &pkt->phdr);
1689
1690 pkt->geom.pkt_overhead = hdr_len + ciphertext_len - pkt->h.bytes_appended;
1691 return 1;
1692 }
1693
on_confirm_notify(uint64_t frame_type,uint64_t stream_id,QUIC_TXPIM_PKT * pkt,void * arg)1694 static void on_confirm_notify(uint64_t frame_type, uint64_t stream_id,
1695 QUIC_TXPIM_PKT *pkt, void *arg)
1696 {
1697 OSSL_QUIC_TX_PACKETISER *txp = arg;
1698
1699 switch (frame_type) {
1700 case OSSL_QUIC_FRAME_TYPE_STOP_SENDING:
1701 {
1702 QUIC_STREAM *s
1703 = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1704
1705 if (s == NULL)
1706 return;
1707
1708 s->acked_stop_sending = 1;
1709 ossl_quic_stream_map_update_state(txp->args.qsm, s);
1710 }
1711 break;
1712 case OSSL_QUIC_FRAME_TYPE_RESET_STREAM:
1713 {
1714 QUIC_STREAM *s
1715 = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1716
1717 if (s == NULL)
1718 return;
1719
1720 /*
1721 * We must already be in RESET_SENT or RESET_RECVD if we are
1722 * here, so we don't need to check state here.
1723 */
1724 ossl_quic_stream_map_notify_reset_stream_acked(txp->args.qsm, s);
1725 ossl_quic_stream_map_update_state(txp->args.qsm, s);
1726 }
1727 break;
1728 default:
1729 assert(0);
1730 break;
1731 }
1732 }
1733
txp_pkt_append_padding(struct txp_pkt * pkt,OSSL_QUIC_TX_PACKETISER * txp,size_t num_bytes)1734 static int txp_pkt_append_padding(struct txp_pkt *pkt,
1735 OSSL_QUIC_TX_PACKETISER *txp, size_t num_bytes)
1736 {
1737 WPACKET *wpkt;
1738
1739 if (num_bytes == 0)
1740 return 1;
1741
1742 if (!ossl_assert(pkt->h_valid))
1743 return 0;
1744
1745 if (!ossl_assert(pkt->tpkt != NULL))
1746 return 0;
1747
1748 wpkt = tx_helper_begin(&pkt->h);
1749 if (wpkt == NULL)
1750 return 0;
1751
1752 if (!ossl_quic_wire_encode_padding(wpkt, num_bytes)) {
1753 tx_helper_rollback(&pkt->h);
1754 return 0;
1755 }
1756
1757 if (!tx_helper_commit(&pkt->h))
1758 return 0;
1759
1760 pkt->tpkt->ackm_pkt.num_bytes += num_bytes;
1761 /* Cannot be non-inflight if we have a PADDING frame */
1762 pkt->tpkt->ackm_pkt.is_inflight = 1;
1763 return 1;
1764 }
1765
on_sstream_updated(uint64_t stream_id,void * arg)1766 static void on_sstream_updated(uint64_t stream_id, void *arg)
1767 {
1768 OSSL_QUIC_TX_PACKETISER *txp = arg;
1769 QUIC_STREAM *s;
1770
1771 s = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1772 if (s == NULL)
1773 return;
1774
1775 ossl_quic_stream_map_update_state(txp->args.qsm, s);
1776 }
1777
1778 /*
1779 * Returns 1 if we can send that many bytes in closing state, 0 otherwise.
1780 * Also maintains the bytes sent state if it returns a success.
1781 */
try_commit_conn_close(OSSL_QUIC_TX_PACKETISER * txp,size_t n)1782 static int try_commit_conn_close(OSSL_QUIC_TX_PACKETISER *txp, size_t n)
1783 {
1784 int res;
1785
1786 /* We can always send the first connection close frame */
1787 if (txp->closing_bytes_recv == 0)
1788 return 1;
1789
1790 /*
1791 * RFC 9000 s. 10.2.1 Closing Connection State:
1792 * To avoid being used for an amplification attack, such
1793 * endpoints MUST limit the cumulative size of packets it sends
1794 * to three times the cumulative size of the packets that are
1795 * received and attributed to the connection.
1796 * and:
1797 * An endpoint in the closing state MUST either discard packets
1798 * received from an unvalidated address or limit the cumulative
1799 * size of packets it sends to an unvalidated address to three
1800 * times the size of packets it receives from that address.
1801 */
1802 res = txp->closing_bytes_xmit + n <= txp->closing_bytes_recv * 3;
1803
1804 /*
1805 * Attribute the bytes to the connection, if we are allowed to send them
1806 * and this isn't the first closing frame.
1807 */
1808 if (res && txp->closing_bytes_recv != 0)
1809 txp->closing_bytes_xmit += n;
1810 return res;
1811 }
1812
ossl_quic_tx_packetiser_record_received_closing_bytes(OSSL_QUIC_TX_PACKETISER * txp,size_t n)1813 void ossl_quic_tx_packetiser_record_received_closing_bytes(
1814 OSSL_QUIC_TX_PACKETISER *txp, size_t n)
1815 {
1816 txp->closing_bytes_recv += n;
1817 }
1818
txp_generate_pre_token(OSSL_QUIC_TX_PACKETISER * txp,struct txp_pkt * pkt,int chosen_for_conn_close,int * can_be_non_inflight)1819 static int txp_generate_pre_token(OSSL_QUIC_TX_PACKETISER *txp,
1820 struct txp_pkt *pkt,
1821 int chosen_for_conn_close,
1822 int *can_be_non_inflight)
1823 {
1824 const uint32_t enc_level = pkt->h.enc_level;
1825 const uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
1826 const struct archetype_data *a = &pkt->geom.adata;
1827 QUIC_TXPIM_PKT *tpkt = pkt->tpkt;
1828 struct tx_helper *h = &pkt->h;
1829 const OSSL_QUIC_FRAME_ACK *ack;
1830 OSSL_QUIC_FRAME_ACK ack2;
1831
1832 tpkt->ackm_pkt.largest_acked = QUIC_PN_INVALID;
1833
1834 /* ACK Frames (Regenerate) */
1835 if (a->allow_ack
1836 && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_ACK
1837 && (((txp->want_ack & (1UL << pn_space)) != 0)
1838 || ossl_ackm_is_ack_desired(txp->args.ackm, pn_space))
1839 && (ack = ossl_ackm_get_ack_frame(txp->args.ackm, pn_space)) != NULL) {
1840 WPACKET *wpkt = tx_helper_begin(h);
1841
1842 if (wpkt == NULL)
1843 return 0;
1844
1845 /* We do not currently support ECN */
1846 ack2 = *ack;
1847 ack2.ecn_present = 0;
1848
1849 if (ossl_quic_wire_encode_frame_ack(wpkt,
1850 txp->args.ack_delay_exponent,
1851 &ack2)) {
1852 if (!tx_helper_commit(h))
1853 return 0;
1854
1855 tpkt->had_ack_frame = 1;
1856
1857 if (ack->num_ack_ranges > 0)
1858 tpkt->ackm_pkt.largest_acked = ack->ack_ranges[0].end;
1859
1860 if (txp->ack_tx_cb != NULL)
1861 txp->ack_tx_cb(&ack2, pn_space, txp->ack_tx_cb_arg);
1862 } else {
1863 tx_helper_rollback(h);
1864 }
1865 }
1866
1867 /* CONNECTION_CLOSE Frames (Regenerate) */
1868 if (a->allow_conn_close && txp->want_conn_close && chosen_for_conn_close) {
1869 WPACKET *wpkt = tx_helper_begin(h);
1870 OSSL_QUIC_FRAME_CONN_CLOSE f, *pf = &txp->conn_close_frame;
1871 size_t l;
1872
1873 if (wpkt == NULL)
1874 return 0;
1875
1876 /*
1877 * Application CONNECTION_CLOSE frames may only be sent in the
1878 * Application PN space, as otherwise they may be sent before a
1879 * connection is authenticated and leak application data. Therefore, if
1880 * we need to send a CONNECTION_CLOSE frame in another PN space and were
1881 * given an application CONNECTION_CLOSE frame, convert it into a
1882 * transport CONNECTION_CLOSE frame, removing any sensitive application
1883 * data.
1884 *
1885 * RFC 9000 s. 10.2.3: "A CONNECTION_CLOSE of type 0x1d MUST be replaced
1886 * by a CONNECTION_CLOSE of type 0x1c when sending the frame in Initial
1887 * or Handshake packets. Otherwise, information about the application
1888 * state might be revealed. Endpoints MUST clear the value of the Reason
1889 * Phrase field and SHOULD use the APPLICATION_ERROR code when
1890 * converting to a CONNECTION_CLOSE of type 0x1c."
1891 */
1892 if (pn_space != QUIC_PN_SPACE_APP && pf->is_app) {
1893 pf = &f;
1894 pf->is_app = 0;
1895 pf->frame_type = 0;
1896 pf->error_code = OSSL_QUIC_ERR_APPLICATION_ERROR;
1897 pf->reason = NULL;
1898 pf->reason_len = 0;
1899 }
1900
1901 if (ossl_quic_wire_encode_frame_conn_close(wpkt, pf)
1902 && WPACKET_get_total_written(wpkt, &l)
1903 && try_commit_conn_close(txp, l)) {
1904 if (!tx_helper_commit(h))
1905 return 0;
1906
1907 tpkt->had_conn_close = 1;
1908 *can_be_non_inflight = 0;
1909 } else {
1910 tx_helper_rollback(h);
1911 }
1912 }
1913
1914 return 1;
1915 }
1916
try_len(size_t space_left,size_t orig_len,size_t base_hdr_len,size_t lenbytes,uint64_t maxn,size_t * hdr_len,size_t * payload_len)1917 static int try_len(size_t space_left, size_t orig_len,
1918 size_t base_hdr_len, size_t lenbytes,
1919 uint64_t maxn, size_t *hdr_len, size_t *payload_len)
1920 {
1921 size_t n;
1922 size_t maxn_ = maxn > SIZE_MAX ? SIZE_MAX : (size_t)maxn;
1923
1924 *hdr_len = base_hdr_len + lenbytes;
1925
1926 if (orig_len == 0 && space_left >= *hdr_len) {
1927 *payload_len = 0;
1928 return 1;
1929 }
1930
1931 n = orig_len;
1932 if (n > maxn_)
1933 n = maxn_;
1934 if (n + *hdr_len > space_left)
1935 n = (space_left >= *hdr_len) ? space_left - *hdr_len : 0;
1936
1937 *payload_len = n;
1938 return n > 0;
1939 }
1940
determine_len(size_t space_left,size_t orig_len,size_t base_hdr_len,uint64_t * hlen,uint64_t * len)1941 static int determine_len(size_t space_left, size_t orig_len,
1942 size_t base_hdr_len,
1943 uint64_t *hlen, uint64_t *len)
1944 {
1945 int ok = 0;
1946 size_t chosen_payload_len = 0;
1947 size_t chosen_hdr_len = 0;
1948 size_t payload_len[4], hdr_len[4];
1949 int i, valid[4] = {0};
1950
1951 valid[0] = try_len(space_left, orig_len, base_hdr_len,
1952 1, OSSL_QUIC_VLINT_1B_MAX,
1953 &hdr_len[0], &payload_len[0]);
1954 valid[1] = try_len(space_left, orig_len, base_hdr_len,
1955 2, OSSL_QUIC_VLINT_2B_MAX,
1956 &hdr_len[1], &payload_len[1]);
1957 valid[2] = try_len(space_left, orig_len, base_hdr_len,
1958 4, OSSL_QUIC_VLINT_4B_MAX,
1959 &hdr_len[2], &payload_len[2]);
1960 valid[3] = try_len(space_left, orig_len, base_hdr_len,
1961 8, OSSL_QUIC_VLINT_8B_MAX,
1962 &hdr_len[3], &payload_len[3]);
1963
1964 for (i = OSSL_NELEM(valid) - 1; i >= 0; --i)
1965 if (valid[i] && payload_len[i] >= chosen_payload_len) {
1966 chosen_payload_len = payload_len[i];
1967 chosen_hdr_len = hdr_len[i];
1968 ok = 1;
1969 }
1970
1971 *hlen = chosen_hdr_len;
1972 *len = chosen_payload_len;
1973 return ok;
1974 }
1975
1976 /*
1977 * Given a CRYPTO frame header with accurate chdr->len and a budget
1978 * (space_left), try to find the optimal value of chdr->len to fill as much of
1979 * the budget as possible. This is slightly hairy because larger values of
1980 * chdr->len cause larger encoded sizes of the length field of the frame, which
1981 * in turn mean less space available for payload data. We check all possible
1982 * encodings and choose the optimal encoding.
1983 */
determine_crypto_len(struct tx_helper * h,OSSL_QUIC_FRAME_CRYPTO * chdr,size_t space_left,uint64_t * hlen,uint64_t * len)1984 static int determine_crypto_len(struct tx_helper *h,
1985 OSSL_QUIC_FRAME_CRYPTO *chdr,
1986 size_t space_left,
1987 uint64_t *hlen,
1988 uint64_t *len)
1989 {
1990 size_t orig_len;
1991 size_t base_hdr_len; /* CRYPTO header length without length field */
1992
1993 if (chdr->len > SIZE_MAX)
1994 return 0;
1995
1996 orig_len = (size_t)chdr->len;
1997
1998 chdr->len = 0;
1999 base_hdr_len = ossl_quic_wire_get_encoded_frame_len_crypto_hdr(chdr);
2000 chdr->len = orig_len;
2001 if (base_hdr_len == 0)
2002 return 0;
2003
2004 --base_hdr_len;
2005
2006 return determine_len(space_left, orig_len, base_hdr_len, hlen, len);
2007 }
2008
determine_stream_len(struct tx_helper * h,OSSL_QUIC_FRAME_STREAM * shdr,size_t space_left,uint64_t * hlen,uint64_t * len)2009 static int determine_stream_len(struct tx_helper *h,
2010 OSSL_QUIC_FRAME_STREAM *shdr,
2011 size_t space_left,
2012 uint64_t *hlen,
2013 uint64_t *len)
2014 {
2015 size_t orig_len;
2016 size_t base_hdr_len; /* STREAM header length without length field */
2017
2018 if (shdr->len > SIZE_MAX)
2019 return 0;
2020
2021 orig_len = (size_t)shdr->len;
2022
2023 shdr->len = 0;
2024 base_hdr_len = ossl_quic_wire_get_encoded_frame_len_stream_hdr(shdr);
2025 shdr->len = orig_len;
2026 if (base_hdr_len == 0)
2027 return 0;
2028
2029 if (shdr->has_explicit_len)
2030 --base_hdr_len;
2031
2032 return determine_len(space_left, orig_len, base_hdr_len, hlen, len);
2033 }
2034
txp_generate_crypto_frames(OSSL_QUIC_TX_PACKETISER * txp,struct txp_pkt * pkt,int * have_ack_eliciting)2035 static int txp_generate_crypto_frames(OSSL_QUIC_TX_PACKETISER *txp,
2036 struct txp_pkt *pkt,
2037 int *have_ack_eliciting)
2038 {
2039 const uint32_t enc_level = pkt->h.enc_level;
2040 const uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
2041 QUIC_TXPIM_PKT *tpkt = pkt->tpkt;
2042 struct tx_helper *h = &pkt->h;
2043 size_t num_stream_iovec;
2044 OSSL_QUIC_FRAME_STREAM shdr = {0};
2045 OSSL_QUIC_FRAME_CRYPTO chdr = {0};
2046 OSSL_QTX_IOVEC iov[2];
2047 uint64_t hdr_bytes;
2048 WPACKET *wpkt;
2049 QUIC_TXPIM_CHUNK chunk = {0};
2050 size_t i, space_left;
2051
2052 for (i = 0;; ++i) {
2053 space_left = tx_helper_get_space_left(h);
2054
2055 if (space_left < MIN_FRAME_SIZE_CRYPTO)
2056 return 1; /* no point trying */
2057
2058 /* Do we have any CRYPTO data waiting? */
2059 num_stream_iovec = OSSL_NELEM(iov);
2060 if (!ossl_quic_sstream_get_stream_frame(txp->args.crypto[pn_space],
2061 i, &shdr, iov,
2062 &num_stream_iovec))
2063 return 1; /* nothing to do */
2064
2065 /* Convert STREAM frame header to CRYPTO frame header */
2066 chdr.offset = shdr.offset;
2067 chdr.len = shdr.len;
2068
2069 if (chdr.len == 0)
2070 return 1; /* nothing to do */
2071
2072 /* Find best fit (header length, payload length) combination. */
2073 if (!determine_crypto_len(h, &chdr, space_left, &hdr_bytes,
2074 &chdr.len))
2075 return 1; /* can't fit anything */
2076
2077 /*
2078 * Truncate IOVs to match our chosen length.
2079 *
2080 * The length cannot be more than SIZE_MAX because this length comes
2081 * from our send stream buffer.
2082 */
2083 ossl_quic_sstream_adjust_iov((size_t)chdr.len, iov, num_stream_iovec);
2084
2085 /*
2086 * Ensure we have enough iovecs allocated (1 for the header, up to 2 for
2087 * the stream data.)
2088 */
2089 if (!txp_el_ensure_iovec(&txp->el[enc_level], h->num_iovec + 3))
2090 return 0; /* alloc error */
2091
2092 /* Encode the header. */
2093 wpkt = tx_helper_begin(h);
2094 if (wpkt == NULL)
2095 return 0; /* alloc error */
2096
2097 if (!ossl_quic_wire_encode_frame_crypto_hdr(wpkt, &chdr)) {
2098 tx_helper_rollback(h);
2099 return 1; /* can't fit */
2100 }
2101
2102 if (!tx_helper_commit(h))
2103 return 0; /* alloc error */
2104
2105 /* Add payload iovecs to the helper (infallible). */
2106 for (i = 0; i < num_stream_iovec; ++i)
2107 tx_helper_append_iovec(h, iov[i].buf, iov[i].buf_len);
2108
2109 *have_ack_eliciting = 1;
2110 tx_helper_unrestrict(h); /* no longer need PING */
2111
2112 /* Log chunk to TXPIM. */
2113 chunk.stream_id = UINT64_MAX; /* crypto stream */
2114 chunk.start = chdr.offset;
2115 chunk.end = chdr.offset + chdr.len - 1;
2116 chunk.has_fin = 0; /* Crypto stream never ends */
2117 if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk))
2118 return 0; /* alloc error */
2119 }
2120 }
2121
2122 struct chunk_info {
2123 OSSL_QUIC_FRAME_STREAM shdr;
2124 uint64_t orig_len;
2125 OSSL_QTX_IOVEC iov[2];
2126 size_t num_stream_iovec;
2127 int valid;
2128 };
2129
txp_plan_stream_chunk(OSSL_QUIC_TX_PACKETISER * txp,struct tx_helper * h,QUIC_SSTREAM * sstream,QUIC_TXFC * stream_txfc,size_t skip,struct chunk_info * chunk,uint64_t consumed)2130 static int txp_plan_stream_chunk(OSSL_QUIC_TX_PACKETISER *txp,
2131 struct tx_helper *h,
2132 QUIC_SSTREAM *sstream,
2133 QUIC_TXFC *stream_txfc,
2134 size_t skip,
2135 struct chunk_info *chunk,
2136 uint64_t consumed)
2137 {
2138 uint64_t fc_credit, fc_swm, fc_limit;
2139
2140 chunk->num_stream_iovec = OSSL_NELEM(chunk->iov);
2141 chunk->valid = ossl_quic_sstream_get_stream_frame(sstream, skip,
2142 &chunk->shdr,
2143 chunk->iov,
2144 &chunk->num_stream_iovec);
2145 if (!chunk->valid)
2146 return 1;
2147
2148 if (!ossl_assert(chunk->shdr.len > 0 || chunk->shdr.is_fin))
2149 /* Should only have 0-length chunk if FIN */
2150 return 0;
2151
2152 chunk->orig_len = chunk->shdr.len;
2153
2154 /* Clamp according to connection and stream-level TXFC. */
2155 fc_credit = ossl_quic_txfc_get_credit(stream_txfc, consumed);
2156 fc_swm = ossl_quic_txfc_get_swm(stream_txfc);
2157 fc_limit = fc_swm + fc_credit;
2158
2159 if (chunk->shdr.len > 0 && chunk->shdr.offset + chunk->shdr.len > fc_limit) {
2160 chunk->shdr.len = (fc_limit <= chunk->shdr.offset)
2161 ? 0 : fc_limit - chunk->shdr.offset;
2162 chunk->shdr.is_fin = 0;
2163 }
2164
2165 if (chunk->shdr.len == 0 && !chunk->shdr.is_fin) {
2166 /*
2167 * Nothing to do due to TXFC. Since SSTREAM returns chunks in ascending
2168 * order of offset we don't need to check any later chunks, so stop
2169 * iterating here.
2170 */
2171 chunk->valid = 0;
2172 return 1;
2173 }
2174
2175 return 1;
2176 }
2177
2178 /*
2179 * Returns 0 on fatal error (e.g. allocation failure), 1 on success.
2180 * *packet_full is set to 1 if there is no longer enough room for another STREAM
2181 * frame.
2182 */
txp_generate_stream_frames(OSSL_QUIC_TX_PACKETISER * txp,struct txp_pkt * pkt,uint64_t id,QUIC_SSTREAM * sstream,QUIC_TXFC * stream_txfc,QUIC_STREAM * next_stream,int * have_ack_eliciting,int * packet_full,uint64_t * new_credit_consumed,uint64_t conn_consumed)2183 static int txp_generate_stream_frames(OSSL_QUIC_TX_PACKETISER *txp,
2184 struct txp_pkt *pkt,
2185 uint64_t id,
2186 QUIC_SSTREAM *sstream,
2187 QUIC_TXFC *stream_txfc,
2188 QUIC_STREAM *next_stream,
2189 int *have_ack_eliciting,
2190 int *packet_full,
2191 uint64_t *new_credit_consumed,
2192 uint64_t conn_consumed)
2193 {
2194 int rc = 0;
2195 struct chunk_info chunks[2] = {0};
2196 const uint32_t enc_level = pkt->h.enc_level;
2197 QUIC_TXPIM_PKT *tpkt = pkt->tpkt;
2198 struct tx_helper *h = &pkt->h;
2199 OSSL_QUIC_FRAME_STREAM *shdr;
2200 WPACKET *wpkt;
2201 QUIC_TXPIM_CHUNK chunk;
2202 size_t i, j, space_left;
2203 int can_fill_payload, use_explicit_len;
2204 int could_have_following_chunk;
2205 uint64_t orig_len;
2206 uint64_t hdr_len_implicit, payload_len_implicit;
2207 uint64_t hdr_len_explicit, payload_len_explicit;
2208 uint64_t fc_swm, fc_new_hwm;
2209
2210 fc_swm = ossl_quic_txfc_get_swm(stream_txfc);
2211 fc_new_hwm = fc_swm;
2212
2213 /*
2214 * Load the first two chunks if any offered by the send stream. We retrieve
2215 * the next chunk in advance so we can determine if we need to send any more
2216 * chunks from the same stream after this one, which is needed when
2217 * determining when we can use an implicit length in a STREAM frame.
2218 */
2219 for (i = 0; i < 2; ++i) {
2220 if (!txp_plan_stream_chunk(txp, h, sstream, stream_txfc, i, &chunks[i],
2221 conn_consumed))
2222 goto err;
2223
2224 if (i == 0 && !chunks[i].valid) {
2225 /* No chunks, nothing to do. */
2226 rc = 1;
2227 goto err;
2228 }
2229 }
2230
2231 for (i = 0;; ++i) {
2232 space_left = tx_helper_get_space_left(h);
2233
2234 if (!chunks[i % 2].valid) {
2235 /* Out of chunks; we're done. */
2236 rc = 1;
2237 goto err;
2238 }
2239
2240 if (space_left < MIN_FRAME_SIZE_STREAM) {
2241 *packet_full = 1;
2242 rc = 1;
2243 goto err;
2244 }
2245
2246 if (!ossl_assert(!h->done_implicit))
2247 /*
2248 * Logic below should have ensured we didn't append an
2249 * implicit-length unless we filled the packet or didn't have
2250 * another stream to handle, so this should not be possible.
2251 */
2252 goto err;
2253
2254 shdr = &chunks[i % 2].shdr;
2255 orig_len = chunks[i % 2].orig_len;
2256 if (i > 0)
2257 /* Load next chunk for lookahead. */
2258 if (!txp_plan_stream_chunk(txp, h, sstream, stream_txfc, i + 1,
2259 &chunks[(i + 1) % 2], conn_consumed))
2260 goto err;
2261
2262 /*
2263 * Find best fit (header length, payload length) combination for if we
2264 * use an implicit length.
2265 */
2266 shdr->has_explicit_len = 0;
2267 hdr_len_implicit = payload_len_implicit = 0;
2268 if (!determine_stream_len(h, shdr, space_left,
2269 &hdr_len_implicit, &payload_len_implicit)) {
2270 *packet_full = 1;
2271 rc = 1;
2272 goto err; /* can't fit anything */
2273 }
2274
2275 /*
2276 * If there is a next stream, we don't use the implicit length so we can
2277 * add more STREAM frames after this one, unless there is enough data
2278 * for this STREAM frame to fill the packet.
2279 */
2280 can_fill_payload = (hdr_len_implicit + payload_len_implicit
2281 >= space_left);
2282
2283 /*
2284 * Is there is a stream after this one, or another chunk pending
2285 * transmission in this stream?
2286 */
2287 could_have_following_chunk
2288 = (next_stream != NULL || chunks[(i + 1) % 2].valid);
2289
2290 /* Choose between explicit or implicit length representations. */
2291 use_explicit_len = !((can_fill_payload || !could_have_following_chunk)
2292 && !pkt->force_pad);
2293
2294 if (use_explicit_len) {
2295 /*
2296 * Find best fit (header length, payload length) combination for if
2297 * we use an explicit length.
2298 */
2299 shdr->has_explicit_len = 1;
2300 hdr_len_explicit = payload_len_explicit = 0;
2301 if (!determine_stream_len(h, shdr, space_left,
2302 &hdr_len_explicit, &payload_len_explicit)) {
2303 *packet_full = 1;
2304 rc = 1;
2305 goto err; /* can't fit anything */
2306 }
2307
2308 shdr->len = payload_len_explicit;
2309 } else {
2310 *packet_full = 1;
2311 shdr->has_explicit_len = 0;
2312 shdr->len = payload_len_implicit;
2313 }
2314
2315 /* If this is a FIN, don't keep filling the packet with more FINs. */
2316 if (shdr->is_fin)
2317 chunks[(i + 1) % 2].valid = 0;
2318
2319 /*
2320 * We are now committed to our length (shdr->len can't change).
2321 * If we truncated the chunk, clear the FIN bit.
2322 */
2323 if (shdr->len < orig_len)
2324 shdr->is_fin = 0;
2325
2326 /* Truncate IOVs to match our chosen length. */
2327 ossl_quic_sstream_adjust_iov((size_t)shdr->len, chunks[i % 2].iov,
2328 chunks[i % 2].num_stream_iovec);
2329
2330 /*
2331 * Ensure we have enough iovecs allocated (1 for the header, up to 2 for
2332 * the stream data.)
2333 */
2334 if (!txp_el_ensure_iovec(&txp->el[enc_level], h->num_iovec + 3))
2335 goto err; /* alloc error */
2336
2337 /* Encode the header. */
2338 wpkt = tx_helper_begin(h);
2339 if (wpkt == NULL)
2340 goto err; /* alloc error */
2341
2342 shdr->stream_id = id;
2343 if (!ossl_assert(ossl_quic_wire_encode_frame_stream_hdr(wpkt, shdr))) {
2344 /* (Should not be possible.) */
2345 tx_helper_rollback(h);
2346 *packet_full = 1;
2347 rc = 1;
2348 goto err; /* can't fit */
2349 }
2350
2351 if (!tx_helper_commit(h))
2352 goto err; /* alloc error */
2353
2354 /* Add payload iovecs to the helper (infallible). */
2355 for (j = 0; j < chunks[i % 2].num_stream_iovec; ++j)
2356 tx_helper_append_iovec(h, chunks[i % 2].iov[j].buf,
2357 chunks[i % 2].iov[j].buf_len);
2358
2359 *have_ack_eliciting = 1;
2360 tx_helper_unrestrict(h); /* no longer need PING */
2361 if (!shdr->has_explicit_len)
2362 h->done_implicit = 1;
2363
2364 /* Log new TXFC credit which was consumed. */
2365 if (shdr->len > 0 && shdr->offset + shdr->len > fc_new_hwm)
2366 fc_new_hwm = shdr->offset + shdr->len;
2367
2368 /* Log chunk to TXPIM. */
2369 chunk.stream_id = shdr->stream_id;
2370 chunk.start = shdr->offset;
2371 chunk.end = shdr->offset + shdr->len - 1;
2372 chunk.has_fin = shdr->is_fin;
2373 chunk.has_stop_sending = 0;
2374 chunk.has_reset_stream = 0;
2375 if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk))
2376 goto err; /* alloc error */
2377
2378 if (shdr->len < orig_len) {
2379 /*
2380 * If we did not serialize all of this chunk we definitely do not
2381 * want to try the next chunk
2382 */
2383 rc = 1;
2384 goto err;
2385 }
2386 }
2387
2388 err:
2389 *new_credit_consumed = fc_new_hwm - fc_swm;
2390 return rc;
2391 }
2392
txp_enlink_tmp(QUIC_STREAM ** tmp_head,QUIC_STREAM * stream)2393 static void txp_enlink_tmp(QUIC_STREAM **tmp_head, QUIC_STREAM *stream)
2394 {
2395 stream->txp_next = *tmp_head;
2396 *tmp_head = stream;
2397 }
2398
txp_generate_stream_related(OSSL_QUIC_TX_PACKETISER * txp,struct txp_pkt * pkt,int * have_ack_eliciting,QUIC_STREAM ** tmp_head)2399 static int txp_generate_stream_related(OSSL_QUIC_TX_PACKETISER *txp,
2400 struct txp_pkt *pkt,
2401 int *have_ack_eliciting,
2402 QUIC_STREAM **tmp_head)
2403 {
2404 QUIC_STREAM_ITER it;
2405 WPACKET *wpkt;
2406 uint64_t cwm;
2407 QUIC_STREAM *stream, *snext;
2408 struct tx_helper *h = &pkt->h;
2409 uint64_t conn_consumed = 0;
2410
2411 for (ossl_quic_stream_iter_init(&it, txp->args.qsm, 1);
2412 it.stream != NULL;) {
2413
2414 stream = it.stream;
2415 ossl_quic_stream_iter_next(&it);
2416 snext = it.stream;
2417
2418 stream->txp_sent_fc = 0;
2419 stream->txp_sent_stop_sending = 0;
2420 stream->txp_sent_reset_stream = 0;
2421 stream->txp_blocked = 0;
2422 stream->txp_txfc_new_credit_consumed = 0;
2423
2424 /* Stream Abort Frames (STOP_SENDING, RESET_STREAM) */
2425 if (stream->want_stop_sending) {
2426 OSSL_QUIC_FRAME_STOP_SENDING f;
2427
2428 wpkt = tx_helper_begin(h);
2429 if (wpkt == NULL)
2430 return 0; /* alloc error */
2431
2432 f.stream_id = stream->id;
2433 f.app_error_code = stream->stop_sending_aec;
2434 if (!ossl_quic_wire_encode_frame_stop_sending(wpkt, &f)) {
2435 tx_helper_rollback(h); /* can't fit */
2436 txp_enlink_tmp(tmp_head, stream);
2437 break;
2438 }
2439
2440 if (!tx_helper_commit(h))
2441 return 0; /* alloc error */
2442
2443 *have_ack_eliciting = 1;
2444 tx_helper_unrestrict(h); /* no longer need PING */
2445 stream->txp_sent_stop_sending = 1;
2446 }
2447
2448 if (stream->want_reset_stream) {
2449 OSSL_QUIC_FRAME_RESET_STREAM f;
2450
2451 if (!ossl_assert(stream->send_state == QUIC_SSTREAM_STATE_RESET_SENT))
2452 return 0;
2453
2454 wpkt = tx_helper_begin(h);
2455 if (wpkt == NULL)
2456 return 0; /* alloc error */
2457
2458 f.stream_id = stream->id;
2459 f.app_error_code = stream->reset_stream_aec;
2460 if (!ossl_quic_stream_send_get_final_size(stream, &f.final_size))
2461 return 0; /* should not be possible */
2462
2463 if (!ossl_quic_wire_encode_frame_reset_stream(wpkt, &f)) {
2464 tx_helper_rollback(h); /* can't fit */
2465 txp_enlink_tmp(tmp_head, stream);
2466 break;
2467 }
2468
2469 if (!tx_helper_commit(h))
2470 return 0; /* alloc error */
2471
2472 *have_ack_eliciting = 1;
2473 tx_helper_unrestrict(h); /* no longer need PING */
2474 stream->txp_sent_reset_stream = 1;
2475
2476 /*
2477 * The final size of the stream as indicated by RESET_STREAM is used
2478 * to ensure a consistent view of flow control state by both
2479 * parties; if we happen to send a RESET_STREAM that consumes more
2480 * flow control credit, make sure we account for that.
2481 */
2482 if (!ossl_assert(f.final_size <= ossl_quic_txfc_get_swm(&stream->txfc)))
2483 return 0;
2484
2485 stream->txp_txfc_new_credit_consumed
2486 = f.final_size - ossl_quic_txfc_get_swm(&stream->txfc);
2487 }
2488
2489 /*
2490 * Stream Flow Control Frames (MAX_STREAM_DATA)
2491 *
2492 * RFC 9000 s. 13.3: "An endpoint SHOULD stop sending MAX_STREAM_DATA
2493 * frames when the receiving part of the stream enters a "Size Known" or
2494 * "Reset Recvd" state." -- In practice, RECV is the only state
2495 * in which it makes sense to generate more MAX_STREAM_DATA frames.
2496 */
2497 if (stream->recv_state == QUIC_RSTREAM_STATE_RECV
2498 && (stream->want_max_stream_data
2499 || ossl_quic_rxfc_has_cwm_changed(&stream->rxfc, 0))) {
2500
2501 wpkt = tx_helper_begin(h);
2502 if (wpkt == NULL)
2503 return 0; /* alloc error */
2504
2505 cwm = ossl_quic_rxfc_get_cwm(&stream->rxfc);
2506
2507 if (!ossl_quic_wire_encode_frame_max_stream_data(wpkt, stream->id,
2508 cwm)) {
2509 tx_helper_rollback(h); /* can't fit */
2510 txp_enlink_tmp(tmp_head, stream);
2511 break;
2512 }
2513
2514 if (!tx_helper_commit(h))
2515 return 0; /* alloc error */
2516
2517 *have_ack_eliciting = 1;
2518 tx_helper_unrestrict(h); /* no longer need PING */
2519 stream->txp_sent_fc = 1;
2520 }
2521
2522 /*
2523 * Stream Data Frames (STREAM)
2524 *
2525 * RFC 9000 s. 3.3: A sender MUST NOT send a STREAM [...] frame for a
2526 * stream in the "Reset Sent" state [or any terminal state]. We don't
2527 * send any more STREAM frames if we are sending, have sent, or are
2528 * planning to send, RESET_STREAM. The other terminal state is Data
2529 * Recvd, but txp_generate_stream_frames() is guaranteed to generate
2530 * nothing in this case.
2531 */
2532 if (ossl_quic_stream_has_send_buffer(stream)
2533 && !ossl_quic_stream_send_is_reset(stream)) {
2534 int packet_full = 0;
2535
2536 if (!ossl_assert(!stream->want_reset_stream))
2537 return 0;
2538
2539 if (!txp_generate_stream_frames(txp, pkt,
2540 stream->id, stream->sstream,
2541 &stream->txfc,
2542 snext,
2543 have_ack_eliciting,
2544 &packet_full,
2545 &stream->txp_txfc_new_credit_consumed,
2546 conn_consumed)) {
2547 /* Fatal error (allocation, etc.) */
2548 txp_enlink_tmp(tmp_head, stream);
2549 return 0;
2550 }
2551 conn_consumed += stream->txp_txfc_new_credit_consumed;
2552
2553 if (packet_full) {
2554 txp_enlink_tmp(tmp_head, stream);
2555 break;
2556 }
2557 }
2558
2559 txp_enlink_tmp(tmp_head, stream);
2560 }
2561
2562 return 1;
2563 }
2564
txp_generate_for_el(OSSL_QUIC_TX_PACKETISER * txp,struct txp_pkt * pkt,int chosen_for_conn_close)2565 static int txp_generate_for_el(OSSL_QUIC_TX_PACKETISER *txp,
2566 struct txp_pkt *pkt,
2567 int chosen_for_conn_close)
2568 {
2569 int rc = TXP_ERR_SUCCESS;
2570 const uint32_t enc_level = pkt->h.enc_level;
2571 const uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
2572 int have_ack_eliciting = 0, done_pre_token = 0;
2573 const struct archetype_data a = pkt->geom.adata;
2574 /*
2575 * Cleared if we encode any non-ACK-eliciting frame type which rules out the
2576 * packet being a non-inflight frame. This means any non-ACK ACK-eliciting
2577 * frame, even PADDING frames. ACK eliciting frames always cause a packet to
2578 * become ineligible for non-inflight treatment so it is not necessary to
2579 * clear this in cases where have_ack_eliciting is set, as it is ignored in
2580 * that case.
2581 */
2582 int can_be_non_inflight = 1;
2583 QUIC_CFQ_ITEM *cfq_item;
2584 QUIC_TXPIM_PKT *tpkt = NULL;
2585 struct tx_helper *h = &pkt->h;
2586
2587 /* Maximum PN reached? */
2588 if (!ossl_quic_pn_valid(txp->next_pn[pn_space]))
2589 goto fatal_err;
2590
2591 if (!ossl_assert(pkt->tpkt == NULL))
2592 goto fatal_err;
2593
2594 if ((pkt->tpkt = tpkt = ossl_quic_txpim_pkt_alloc(txp->args.txpim)) == NULL)
2595 goto fatal_err;
2596
2597 /*
2598 * Frame Serialization
2599 * ===================
2600 *
2601 * We now serialize frames into the packet in descending order of priority.
2602 */
2603
2604 /* HANDSHAKE_DONE (Regenerate) */
2605 if (a.allow_handshake_done && txp->want_handshake_done
2606 && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_HANDSHAKE_DONE) {
2607 WPACKET *wpkt = tx_helper_begin(h);
2608
2609 if (wpkt == NULL)
2610 goto fatal_err;
2611
2612 if (ossl_quic_wire_encode_frame_handshake_done(wpkt)) {
2613 tpkt->had_handshake_done_frame = 1;
2614 have_ack_eliciting = 1;
2615
2616 if (!tx_helper_commit(h))
2617 goto fatal_err;
2618
2619 tx_helper_unrestrict(h); /* no longer need PING */
2620 } else {
2621 tx_helper_rollback(h);
2622 }
2623 }
2624
2625 /* MAX_DATA (Regenerate) */
2626 if (a.allow_conn_fc
2627 && (txp->want_max_data
2628 || ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 0))
2629 && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_MAX_DATA) {
2630 WPACKET *wpkt = tx_helper_begin(h);
2631 uint64_t cwm = ossl_quic_rxfc_get_cwm(txp->args.conn_rxfc);
2632
2633 if (wpkt == NULL)
2634 goto fatal_err;
2635
2636 if (ossl_quic_wire_encode_frame_max_data(wpkt, cwm)) {
2637 tpkt->had_max_data_frame = 1;
2638 have_ack_eliciting = 1;
2639
2640 if (!tx_helper_commit(h))
2641 goto fatal_err;
2642
2643 tx_helper_unrestrict(h); /* no longer need PING */
2644 } else {
2645 tx_helper_rollback(h);
2646 }
2647 }
2648
2649 /* MAX_STREAMS_BIDI (Regenerate) */
2650 if (a.allow_conn_fc
2651 && (txp->want_max_streams_bidi
2652 || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_bidi_rxfc, 0))
2653 && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_MAX_STREAMS_BIDI) {
2654 WPACKET *wpkt = tx_helper_begin(h);
2655 uint64_t max_streams
2656 = ossl_quic_rxfc_get_cwm(txp->args.max_streams_bidi_rxfc);
2657
2658 if (wpkt == NULL)
2659 goto fatal_err;
2660
2661 if (ossl_quic_wire_encode_frame_max_streams(wpkt, /*is_uni=*/0,
2662 max_streams)) {
2663 tpkt->had_max_streams_bidi_frame = 1;
2664 have_ack_eliciting = 1;
2665
2666 if (!tx_helper_commit(h))
2667 goto fatal_err;
2668
2669 tx_helper_unrestrict(h); /* no longer need PING */
2670 } else {
2671 tx_helper_rollback(h);
2672 }
2673 }
2674
2675 /* MAX_STREAMS_UNI (Regenerate) */
2676 if (a.allow_conn_fc
2677 && (txp->want_max_streams_uni
2678 || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_uni_rxfc, 0))
2679 && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_MAX_STREAMS_UNI) {
2680 WPACKET *wpkt = tx_helper_begin(h);
2681 uint64_t max_streams
2682 = ossl_quic_rxfc_get_cwm(txp->args.max_streams_uni_rxfc);
2683
2684 if (wpkt == NULL)
2685 goto fatal_err;
2686
2687 if (ossl_quic_wire_encode_frame_max_streams(wpkt, /*is_uni=*/1,
2688 max_streams)) {
2689 tpkt->had_max_streams_uni_frame = 1;
2690 have_ack_eliciting = 1;
2691
2692 if (!tx_helper_commit(h))
2693 goto fatal_err;
2694
2695 tx_helper_unrestrict(h); /* no longer need PING */
2696 } else {
2697 tx_helper_rollback(h);
2698 }
2699 }
2700
2701 /* GCR Frames */
2702 for (cfq_item = ossl_quic_cfq_get_priority_head(txp->args.cfq, pn_space);
2703 cfq_item != NULL;
2704 cfq_item = ossl_quic_cfq_item_get_priority_next(cfq_item, pn_space)) {
2705 uint64_t frame_type = ossl_quic_cfq_item_get_frame_type(cfq_item);
2706 const unsigned char *encoded = ossl_quic_cfq_item_get_encoded(cfq_item);
2707 size_t encoded_len = ossl_quic_cfq_item_get_encoded_len(cfq_item);
2708
2709 switch (frame_type) {
2710 case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID:
2711 if (!a.allow_new_conn_id)
2712 continue;
2713 break;
2714 case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID:
2715 if (!a.allow_retire_conn_id)
2716 continue;
2717 break;
2718 case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN:
2719 if (!a.allow_new_token)
2720 continue;
2721
2722 /*
2723 * NEW_TOKEN frames are handled via GCR, but some
2724 * Regenerate-strategy frames should come before them (namely
2725 * ACK, CONNECTION_CLOSE, PATH_CHALLENGE and PATH_RESPONSE). If
2726 * we find a NEW_TOKEN frame, do these now. If there are no
2727 * NEW_TOKEN frames in the GCR queue we will handle these below.
2728 */
2729 if (!done_pre_token)
2730 if (txp_generate_pre_token(txp, pkt,
2731 chosen_for_conn_close,
2732 &can_be_non_inflight))
2733 done_pre_token = 1;
2734
2735 break;
2736 case OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE:
2737 if (!a.allow_path_response)
2738 continue;
2739
2740 /*
2741 * RFC 9000 s. 8.2.2: An endpoint MUST expand datagrams that
2742 * contain a PATH_RESPONSE frame to at least the smallest
2743 * allowed maximum datagram size of 1200 bytes.
2744 */
2745 pkt->force_pad = 1;
2746 break;
2747 default:
2748 if (!a.allow_cfq_other)
2749 continue;
2750 break;
2751 }
2752
2753 /*
2754 * If the frame is too big, don't try to schedule any more GCR frames in
2755 * this packet rather than sending subsequent ones out of order.
2756 */
2757 if (encoded_len > tx_helper_get_space_left(h))
2758 break;
2759
2760 if (!tx_helper_append_iovec(h, encoded, encoded_len))
2761 goto fatal_err;
2762
2763 ossl_quic_txpim_pkt_add_cfq_item(tpkt, cfq_item);
2764
2765 if (ossl_quic_frame_type_is_ack_eliciting(frame_type)) {
2766 have_ack_eliciting = 1;
2767 tx_helper_unrestrict(h); /* no longer need PING */
2768 }
2769 }
2770
2771 /*
2772 * If we didn't generate ACK, CONNECTION_CLOSE, PATH_CHALLENGE or
2773 * PATH_RESPONSE (as desired) before, do so now.
2774 */
2775 if (!done_pre_token)
2776 if (txp_generate_pre_token(txp, pkt,
2777 chosen_for_conn_close,
2778 &can_be_non_inflight))
2779 done_pre_token = 1;
2780
2781 /* CRYPTO Frames */
2782 if (a.allow_crypto)
2783 if (!txp_generate_crypto_frames(txp, pkt, &have_ack_eliciting))
2784 goto fatal_err;
2785
2786 /* Stream-specific frames */
2787 if (a.allow_stream_rel && txp->handshake_complete)
2788 if (!txp_generate_stream_related(txp, pkt,
2789 &have_ack_eliciting,
2790 &pkt->stream_head))
2791 goto fatal_err;
2792
2793 /* PING */
2794 tx_helper_unrestrict(h);
2795
2796 if (!have_ack_eliciting && txp_need_ping(txp, pn_space, &a)) {
2797 WPACKET *wpkt;
2798
2799 assert(h->reserve > 0);
2800 wpkt = tx_helper_begin(h);
2801 if (wpkt == NULL)
2802 goto fatal_err;
2803
2804 if (!ossl_quic_wire_encode_frame_ping(wpkt)
2805 || !tx_helper_commit(h))
2806 /*
2807 * We treat a request to be ACK-eliciting as a requirement, so this
2808 * is an error.
2809 */
2810 goto fatal_err;
2811
2812 have_ack_eliciting = 1;
2813 }
2814
2815 /* PADDING is added by ossl_quic_tx_packetiser_generate(). */
2816
2817 /*
2818 * ACKM Data
2819 * =========
2820 */
2821 if (have_ack_eliciting)
2822 can_be_non_inflight = 0;
2823
2824 /* ACKM Data */
2825 tpkt->ackm_pkt.num_bytes = h->bytes_appended + pkt->geom.pkt_overhead;
2826 tpkt->ackm_pkt.pkt_num = txp->next_pn[pn_space];
2827 /* largest_acked is set in txp_generate_pre_token */
2828 tpkt->ackm_pkt.pkt_space = pn_space;
2829 tpkt->ackm_pkt.is_inflight = !can_be_non_inflight;
2830 tpkt->ackm_pkt.is_ack_eliciting = have_ack_eliciting;
2831 tpkt->ackm_pkt.is_pto_probe = 0;
2832 tpkt->ackm_pkt.is_mtu_probe = 0;
2833 tpkt->ackm_pkt.time = txp->args.now(txp->args.now_arg);
2834 tpkt->pkt_type = pkt->phdr.type;
2835
2836 /* Done. */
2837 return rc;
2838
2839 fatal_err:
2840 /*
2841 * Handler for fatal errors, i.e. errors causing us to abort the entire
2842 * packet rather than just one frame. Examples of such errors include
2843 * allocation errors.
2844 */
2845 if (tpkt != NULL) {
2846 ossl_quic_txpim_pkt_release(txp->args.txpim, tpkt);
2847 pkt->tpkt = NULL;
2848 }
2849 return TXP_ERR_INTERNAL;
2850 }
2851
2852 /*
2853 * Commits and queues a packet for transmission. There is no backing out after
2854 * this.
2855 *
2856 * This:
2857 *
2858 * - Sends the packet to the QTX for encryption and transmission;
2859 *
2860 * - Records the packet as having been transmitted in FIFM. ACKM is informed,
2861 * etc. and the TXPIM record is filed.
2862 *
2863 * - Informs various subsystems of frames that were sent and clears frame
2864 * wanted flags so that we do not generate the same frames again.
2865 *
2866 * Assumptions:
2867 *
2868 * - pkt is a txp_pkt for the correct EL;
2869 *
2870 * - pkt->tpkt is valid;
2871 *
2872 * - pkt->tpkt->ackm_pkt has been fully filled in;
2873 *
2874 * - Stream chunk records have been appended to pkt->tpkt for STREAM and
2875 * CRYPTO frames, but not for RESET_STREAM or STOP_SENDING frames;
2876 *
2877 * - The chosen stream list for the packet can be fully walked from
2878 * pkt->stream_head using stream->txp_next;
2879 *
2880 * - pkt->has_ack_eliciting is set correctly.
2881 *
2882 */
txp_pkt_commit(OSSL_QUIC_TX_PACKETISER * txp,struct txp_pkt * pkt,uint32_t archetype,int * txpim_pkt_reffed)2883 static int txp_pkt_commit(OSSL_QUIC_TX_PACKETISER *txp,
2884 struct txp_pkt *pkt,
2885 uint32_t archetype,
2886 int *txpim_pkt_reffed)
2887 {
2888 int rc = 1;
2889 uint32_t enc_level = pkt->h.enc_level;
2890 uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
2891 QUIC_TXPIM_PKT *tpkt = pkt->tpkt;
2892 QUIC_STREAM *stream;
2893 OSSL_QTX_PKT txpkt;
2894 struct archetype_data a;
2895
2896 *txpim_pkt_reffed = 0;
2897
2898 /* Cannot send a packet with an empty payload. */
2899 if (pkt->h.bytes_appended == 0)
2900 return 0;
2901
2902 if (!txp_get_archetype_data(enc_level, archetype, &a))
2903 return 0;
2904
2905 /* Packet Information for QTX */
2906 txpkt.hdr = &pkt->phdr;
2907 txpkt.iovec = txp->el[enc_level].iovec;
2908 txpkt.num_iovec = pkt->h.num_iovec;
2909 txpkt.local = NULL;
2910 txpkt.peer = BIO_ADDR_family(&txp->args.peer) == AF_UNSPEC
2911 ? NULL : &txp->args.peer;
2912 txpkt.pn = txp->next_pn[pn_space];
2913 txpkt.flags = OSSL_QTX_PKT_FLAG_COALESCE; /* always try to coalesce */
2914
2915 /* Generate TXPIM chunks representing STOP_SENDING and RESET_STREAM frames. */
2916 for (stream = pkt->stream_head; stream != NULL; stream = stream->txp_next)
2917 if (stream->txp_sent_stop_sending || stream->txp_sent_reset_stream) {
2918 /* Log STOP_SENDING/RESET_STREAM chunk to TXPIM. */
2919 QUIC_TXPIM_CHUNK chunk;
2920
2921 chunk.stream_id = stream->id;
2922 chunk.start = UINT64_MAX;
2923 chunk.end = 0;
2924 chunk.has_fin = 0;
2925 chunk.has_stop_sending = stream->txp_sent_stop_sending;
2926 chunk.has_reset_stream = stream->txp_sent_reset_stream;
2927 if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk))
2928 return 0; /* alloc error */
2929 }
2930
2931 /* Dispatch to FIFD. */
2932 if (!ossl_quic_fifd_pkt_commit(&txp->fifd, tpkt))
2933 return 0;
2934
2935 /*
2936 * Transmission and Post-Packet Generation Bookkeeping
2937 * ===================================================
2938 *
2939 * No backing out anymore - at this point the ACKM has recorded the packet
2940 * as having been sent, so we need to increment our next PN counter, or
2941 * the ACKM will complain when we try to record a duplicate packet with
2942 * the same PN later. At this point actually sending the packet may still
2943 * fail. In this unlikely event it will simply be handled as though it
2944 * were a lost packet.
2945 */
2946 ++txp->next_pn[pn_space];
2947 *txpim_pkt_reffed = 1;
2948
2949 /* Send the packet. */
2950 if (!ossl_qtx_write_pkt(txp->args.qtx, &txpkt))
2951 return 0;
2952
2953 /*
2954 * Record FC and stream abort frames as sent; deactivate streams which no
2955 * longer have anything to do.
2956 */
2957 for (stream = pkt->stream_head; stream != NULL; stream = stream->txp_next) {
2958 if (stream->txp_sent_fc) {
2959 stream->want_max_stream_data = 0;
2960 ossl_quic_rxfc_has_cwm_changed(&stream->rxfc, 1);
2961 }
2962
2963 if (stream->txp_sent_stop_sending)
2964 stream->want_stop_sending = 0;
2965
2966 if (stream->txp_sent_reset_stream)
2967 stream->want_reset_stream = 0;
2968
2969 if (stream->txp_txfc_new_credit_consumed > 0) {
2970 if (!ossl_assert(ossl_quic_txfc_consume_credit(&stream->txfc,
2971 stream->txp_txfc_new_credit_consumed)))
2972 /*
2973 * Should not be possible, but we should continue with our
2974 * bookkeeping as we have already committed the packet to the
2975 * FIFD. Just change the value we return.
2976 */
2977 rc = 0;
2978
2979 stream->txp_txfc_new_credit_consumed = 0;
2980 }
2981
2982 /*
2983 * If we no longer need to generate any flow control (MAX_STREAM_DATA),
2984 * STOP_SENDING or RESET_STREAM frames, nor any STREAM frames (because
2985 * the stream is drained of data or TXFC-blocked), we can mark the
2986 * stream as inactive.
2987 */
2988 ossl_quic_stream_map_update_state(txp->args.qsm, stream);
2989
2990 if (ossl_quic_stream_has_send_buffer(stream)
2991 && !ossl_quic_sstream_has_pending(stream->sstream)
2992 && ossl_quic_sstream_get_final_size(stream->sstream, NULL))
2993 /*
2994 * Transition to DATA_SENT if stream has a final size and we have
2995 * sent all data.
2996 */
2997 ossl_quic_stream_map_notify_all_data_sent(txp->args.qsm, stream);
2998 }
2999
3000 /* We have now sent the packet, so update state accordingly. */
3001 if (tpkt->ackm_pkt.is_ack_eliciting)
3002 txp->force_ack_eliciting &= ~(1UL << pn_space);
3003
3004 if (tpkt->had_handshake_done_frame)
3005 txp->want_handshake_done = 0;
3006
3007 if (tpkt->had_max_data_frame) {
3008 txp->want_max_data = 0;
3009 ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 1);
3010 }
3011
3012 if (tpkt->had_max_streams_bidi_frame) {
3013 txp->want_max_streams_bidi = 0;
3014 ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_bidi_rxfc, 1);
3015 }
3016
3017 if (tpkt->had_max_streams_uni_frame) {
3018 txp->want_max_streams_uni = 0;
3019 ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_uni_rxfc, 1);
3020 }
3021
3022 if (tpkt->had_ack_frame)
3023 txp->want_ack &= ~(1UL << pn_space);
3024
3025 if (tpkt->had_conn_close)
3026 txp->want_conn_close = 0;
3027
3028 /*
3029 * Decrement probe request counts if we have sent a packet that meets
3030 * the requirement of a probe, namely being ACK-eliciting.
3031 */
3032 if (tpkt->ackm_pkt.is_ack_eliciting) {
3033 OSSL_ACKM_PROBE_INFO *probe_info
3034 = ossl_ackm_get0_probe_request(txp->args.ackm);
3035
3036 if (enc_level == QUIC_ENC_LEVEL_INITIAL
3037 && probe_info->anti_deadlock_initial > 0)
3038 --probe_info->anti_deadlock_initial;
3039
3040 if (enc_level == QUIC_ENC_LEVEL_HANDSHAKE
3041 && probe_info->anti_deadlock_handshake > 0)
3042 --probe_info->anti_deadlock_handshake;
3043
3044 if (a.allow_force_ack_eliciting /* (i.e., not for 0-RTT) */
3045 && probe_info->pto[pn_space] > 0)
3046 --probe_info->pto[pn_space];
3047 }
3048
3049 return rc;
3050 }
3051
3052 /* Ensure the iovec array is at least num elements long. */
txp_el_ensure_iovec(struct txp_el * el,size_t num)3053 static int txp_el_ensure_iovec(struct txp_el *el, size_t num)
3054 {
3055 OSSL_QTX_IOVEC *iovec;
3056
3057 if (el->alloc_iovec >= num)
3058 return 1;
3059
3060 num = el->alloc_iovec != 0 ? el->alloc_iovec * 2 : 8;
3061
3062 iovec = OPENSSL_realloc(el->iovec, sizeof(OSSL_QTX_IOVEC) * num);
3063 if (iovec == NULL)
3064 return 0;
3065
3066 el->iovec = iovec;
3067 el->alloc_iovec = num;
3068 return 1;
3069 }
3070
ossl_quic_tx_packetiser_schedule_conn_close(OSSL_QUIC_TX_PACKETISER * txp,const OSSL_QUIC_FRAME_CONN_CLOSE * f)3071 int ossl_quic_tx_packetiser_schedule_conn_close(OSSL_QUIC_TX_PACKETISER *txp,
3072 const OSSL_QUIC_FRAME_CONN_CLOSE *f)
3073 {
3074 char *reason = NULL;
3075 size_t reason_len = f->reason_len;
3076 size_t max_reason_len = txp_get_mdpl(txp) / 2;
3077
3078 if (txp->want_conn_close)
3079 return 0;
3080
3081 /*
3082 * Arbitrarily limit the length of the reason length string to half of the
3083 * MDPL.
3084 */
3085 if (reason_len > max_reason_len)
3086 reason_len = max_reason_len;
3087
3088 if (reason_len > 0) {
3089 reason = OPENSSL_memdup(f->reason, reason_len);
3090 if (reason == NULL)
3091 return 0;
3092 }
3093
3094 txp->conn_close_frame = *f;
3095 txp->conn_close_frame.reason = reason;
3096 txp->conn_close_frame.reason_len = reason_len;
3097 txp->want_conn_close = 1;
3098 return 1;
3099 }
3100
ossl_quic_tx_packetiser_set_msg_callback(OSSL_QUIC_TX_PACKETISER * txp,ossl_msg_cb msg_callback,SSL * msg_callback_ssl)3101 void ossl_quic_tx_packetiser_set_msg_callback(OSSL_QUIC_TX_PACKETISER *txp,
3102 ossl_msg_cb msg_callback,
3103 SSL *msg_callback_ssl)
3104 {
3105 txp->msg_callback = msg_callback;
3106 txp->msg_callback_ssl = msg_callback_ssl;
3107 }
3108
ossl_quic_tx_packetiser_set_msg_callback_arg(OSSL_QUIC_TX_PACKETISER * txp,void * msg_callback_arg)3109 void ossl_quic_tx_packetiser_set_msg_callback_arg(OSSL_QUIC_TX_PACKETISER *txp,
3110 void *msg_callback_arg)
3111 {
3112 txp->msg_callback_arg = msg_callback_arg;
3113 }
3114
ossl_quic_tx_packetiser_get_next_pn(OSSL_QUIC_TX_PACKETISER * txp,uint32_t pn_space)3115 QUIC_PN ossl_quic_tx_packetiser_get_next_pn(OSSL_QUIC_TX_PACKETISER *txp,
3116 uint32_t pn_space)
3117 {
3118 if (pn_space >= QUIC_PN_SPACE_NUM)
3119 return UINT64_MAX;
3120
3121 return txp->next_pn[pn_space];
3122 }
3123
ossl_quic_tx_packetiser_get_deadline(OSSL_QUIC_TX_PACKETISER * txp)3124 OSSL_TIME ossl_quic_tx_packetiser_get_deadline(OSSL_QUIC_TX_PACKETISER *txp)
3125 {
3126 /*
3127 * TXP-specific deadline computations which rely on TXP innards. This is in
3128 * turn relied on by the QUIC_CHANNEL code to determine the channel event
3129 * handling deadline.
3130 */
3131 OSSL_TIME deadline = ossl_time_infinite();
3132 uint32_t enc_level, pn_space;
3133
3134 /*
3135 * ACK generation is not CC-gated - packets containing only ACKs are allowed
3136 * to bypass CC. We want to generate ACK frames even if we are currently
3137 * restricted by CC so the peer knows we have received data. The generate
3138 * call will take care of selecting the correct packet archetype.
3139 */
3140 for (enc_level = QUIC_ENC_LEVEL_INITIAL;
3141 enc_level < QUIC_ENC_LEVEL_NUM;
3142 ++enc_level)
3143 if (ossl_qtx_is_enc_level_provisioned(txp->args.qtx, enc_level)) {
3144 pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
3145 deadline = ossl_time_min(deadline,
3146 ossl_ackm_get_ack_deadline(txp->args.ackm, pn_space));
3147 }
3148
3149 /* When will CC let us send more? */
3150 if (txp->args.cc_method->get_tx_allowance(txp->args.cc_data) == 0)
3151 deadline = ossl_time_min(deadline,
3152 txp->args.cc_method->get_wakeup_deadline(txp->args.cc_data));
3153
3154 return deadline;
3155 }
3156
