2 * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved.
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
10 #include "internal/quic_txp.h"
11 #include "internal/quic_fifd.h"
12 #include "internal/quic_stream_map.h"
13 #include "internal/common.h"
14 #include <openssl/err.h>
16 #define MIN_CRYPTO_HDR_SIZE 3
18 #define MIN_FRAME_SIZE_HANDSHAKE_DONE 1
19 #define MIN_FRAME_SIZE_MAX_DATA 2
20 #define MIN_FRAME_SIZE_ACK 5
21 #define MIN_FRAME_SIZE_CRYPTO (MIN_CRYPTO_HDR_SIZE + 1)
22 #define MIN_FRAME_SIZE_STREAM 3 /* minimum useful size (for non-FIN) */
23 #define MIN_FRAME_SIZE_MAX_STREAMS_BIDI 2
24 #define MIN_FRAME_SIZE_MAX_STREAMS_UNI 2
26 struct ossl_quic_tx_packetiser_st {
27 OSSL_QUIC_TX_PACKETISER_ARGS args;
30 * Opaque initial token blob provided by caller. TXP frees using the
31 * callback when it is no longer needed.
33 const unsigned char *initial_token;
34 size_t initial_token_len;
35 ossl_quic_initial_token_free_fn *initial_token_free_cb;
36 void *initial_token_free_cb_arg;
38 /* Subcomponents of the TXP that we own. */
39 QUIC_FIFD fifd; /* QUIC Frame-in-Flight Dispatcher */
42 uint64_t next_pn[QUIC_PN_SPACE_NUM]; /* Next PN to use in given PN space. */
43 OSSL_TIME last_tx_time; /* Last time a packet was generated, or 0. */
45 /* Internal state - frame (re)generation flags. */
46 unsigned int want_handshake_done : 1;
47 unsigned int want_max_data : 1;
48 unsigned int want_max_streams_bidi : 1;
49 unsigned int want_max_streams_uni : 1;
51 /* Internal state - frame (re)generation flags - per PN space. */
52 unsigned int want_ack : QUIC_PN_SPACE_NUM;
53 unsigned int force_ack_eliciting : QUIC_PN_SPACE_NUM;
56 * Internal state - connection close terminal state.
57 * Once this is set, it is not unset unlike other want_ flags - we keep
58 * sending it in every packet.
60 unsigned int want_conn_close : 1;
62 /* Has the handshake been completed? */
63 unsigned int handshake_complete : 1;
65 OSSL_QUIC_FRAME_CONN_CLOSE conn_close_frame;
67 /* Internal state - packet assembly. */
68 unsigned char *scratch; /* scratch buffer for packet assembly */
69 size_t scratch_len; /* number of bytes allocated for scratch */
70 OSSL_QTX_IOVEC *iovec; /* scratch iovec array for use with QTX */
71 size_t alloc_iovec; /* size of iovec array */
75 * The TX helper records state used while generating frames into packets. It
76 * enables serialization into the packet to be done "transactionally" where
77 * serialization of a frame can be rolled back if it fails midway (e.g. if it
81 OSSL_QUIC_TX_PACKETISER *txp;
83 * The Maximum Packet Payload Length in bytes. This is the amount of
84 * space we have to generate frames into.
88 * Number of bytes we have generated so far.
90 size_t bytes_appended;
92 * Number of scratch bytes in txp->scratch we have used so far. Some iovecs
93 * will reference this scratch buffer. When we need to use more of it (e.g.
94 * when we need to put frame headers somewhere), we append to the scratch
95 * buffer, resizing if necessary, and increase this accordingly.
99 * Bytes reserved in the MaxPPL budget. We keep this number of bytes spare
100 * until reserve_allowed is set to 1. Currently this is always at most 1, as
101 * a PING frame takes up one byte and this mechanism is only used to ensure
102 * we can encode a PING frame if we have been asked to ensure a packet is
103 * ACK-eliciting and we are unusure if we are going to add any other
104 * ACK-eliciting frames before we reach our MaxPPL budget.
108 * Number of iovecs we have currently appended. This is the number of
109 * entries valid in txp->iovec.
113 * Whether we are allowed to make use of the reserve bytes in our MaxPPL
114 * budget. This is used to ensure we have room to append a PING frame later
115 * if we need to. Once we know we will not need to append a PING frame, this
118 unsigned int reserve_allowed : 1;
120 * Set to 1 if we have appended a STREAM frame with an implicit length. If
121 * this happens we should never append another frame after that frame as it
122 * cannot be validly encoded. This is just a safety check.
124 unsigned int done_implicit : 1;
127 * The fields in this structure are valid if active is set, which means
128 * that a serialization transaction is currently in progress.
132 unsigned int active : 1;
136 static void tx_helper_rollback(struct tx_helper *h);
137 static int txp_ensure_iovec(OSSL_QUIC_TX_PACKETISER *txp, size_t num);
139 /* Initialises the TX helper. */
140 static int tx_helper_init(struct tx_helper *h, OSSL_QUIC_TX_PACKETISER *txp,
141 size_t max_ppl, size_t reserve)
143 if (reserve > max_ppl)
147 h->max_ppl = max_ppl;
148 h->reserve = reserve;
150 h->bytes_appended = 0;
151 h->scratch_bytes = 0;
152 h->reserve_allowed = 0;
153 h->done_implicit = 0;
157 if (max_ppl > h->txp->scratch_len) {
158 unsigned char *scratch;
160 scratch = OPENSSL_realloc(h->txp->scratch, max_ppl);
164 h->txp->scratch = scratch;
165 h->txp->scratch_len = max_ppl;
171 static void tx_helper_cleanup(struct tx_helper *h)
174 tx_helper_rollback(h);
179 static void tx_helper_unrestrict(struct tx_helper *h)
181 h->reserve_allowed = 1;
185 * Append an extent of memory to the iovec list. The memory must remain
186 * allocated until we finish generating the packet and call the QTX.
188 * In general, the buffers passed to this function will be from one of two
191 * - Application data contained in stream buffers managed elsewhere
192 * in the QUIC stack; or
194 * - Control frame data appended into txp->scratch using tx_helper_begin and
198 static int tx_helper_append_iovec(struct tx_helper *h,
199 const unsigned char *buf,
205 if (!ossl_assert(!h->done_implicit))
208 if (!txp_ensure_iovec(h->txp, h->num_iovec + 1))
211 h->txp->iovec[h->num_iovec].buf = buf;
212 h->txp->iovec[h->num_iovec].buf_len = buf_len;
215 h->bytes_appended += buf_len;
220 * How many more bytes of space do we have left in our plaintext packet payload?
222 static size_t tx_helper_get_space_left(struct tx_helper *h)
225 - (h->reserve_allowed ? 0 : h->reserve) - h->bytes_appended;
229 * Begin a control frame serialization transaction. This allows the
230 * serialization of the control frame to be backed out if it turns out it won't
231 * fit. Write the control frame to the returned WPACKET. Ensure you always
232 * call tx_helper_rollback or tx_helper_commit (or tx_helper_cleanup). Returns
235 static WPACKET *tx_helper_begin(struct tx_helper *h)
237 size_t space_left, len;
240 if (!ossl_assert(!h->txn.active))
243 if (!ossl_assert(!h->done_implicit))
246 data = (unsigned char *)h->txp->scratch + h->scratch_bytes;
247 len = h->txp->scratch_len - h->scratch_bytes;
249 space_left = tx_helper_get_space_left(h);
250 if (!ossl_assert(space_left <= len))
253 if (!WPACKET_init_static_len(&h->txn.wpkt, data, len, 0))
256 if (!WPACKET_set_max_size(&h->txn.wpkt, space_left)) {
257 WPACKET_cleanup(&h->txn.wpkt);
266 static void tx_helper_end(struct tx_helper *h, int success)
269 WPACKET_finish(&h->txn.wpkt);
271 WPACKET_cleanup(&h->txn.wpkt);
277 /* Abort a control frame serialization transaction. */
278 static void tx_helper_rollback(struct tx_helper *h)
286 /* Commit a control frame. */
287 static int tx_helper_commit(struct tx_helper *h)
294 if (!WPACKET_get_total_written(&h->txn.wpkt, &l)) {
299 if (!tx_helper_append_iovec(h, h->txn.data, l)) {
304 h->scratch_bytes += l;
309 static QUIC_SSTREAM *get_sstream_by_id(uint64_t stream_id, uint32_t pn_space,
311 static void on_regen_notify(uint64_t frame_type, uint64_t stream_id,
312 QUIC_TXPIM_PKT *pkt, void *arg);
313 static void on_sstream_updated(uint64_t stream_id, void *arg);
314 static int sstream_is_pending(QUIC_SSTREAM *sstream);
315 static int txp_el_pending(OSSL_QUIC_TX_PACKETISER *txp, uint32_t enc_level,
318 uint32_t *conn_close_enc_level);
319 static int txp_generate_for_el(OSSL_QUIC_TX_PACKETISER *txp, uint32_t enc_level,
322 int is_last_in_dgram,
323 int dgram_contains_initial,
324 int chosen_for_conn_close,
325 int *sent_ack_eliciting);
326 static size_t txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER *txp);
327 static int txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER *txp,
332 static size_t txp_get_mdpl(OSSL_QUIC_TX_PACKETISER *txp);
333 static int txp_generate_for_el_actual(OSSL_QUIC_TX_PACKETISER *txp,
340 int chosen_for_conn_close,
341 int *sent_ack_eliciting);
343 OSSL_QUIC_TX_PACKETISER *ossl_quic_tx_packetiser_new(const OSSL_QUIC_TX_PACKETISER_ARGS *args)
345 OSSL_QUIC_TX_PACKETISER *txp;
349 || args->txpim == NULL
351 || args->ackm == NULL
353 || args->conn_txfc == NULL
354 || args->conn_rxfc == NULL
355 || args->max_streams_bidi_rxfc == NULL
356 || args->max_streams_uni_rxfc == NULL) {
357 ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER);
361 txp = OPENSSL_zalloc(sizeof(*txp));
366 txp->last_tx_time = ossl_time_zero();
368 if (!ossl_quic_fifd_init(&txp->fifd,
369 txp->args.cfq, txp->args.ackm, txp->args.txpim,
370 get_sstream_by_id, txp,
371 on_regen_notify, txp,
372 on_sstream_updated, txp)) {
380 void ossl_quic_tx_packetiser_free(OSSL_QUIC_TX_PACKETISER *txp)
385 ossl_quic_tx_packetiser_set_initial_token(txp, NULL, 0, NULL, NULL);
386 ossl_quic_fifd_cleanup(&txp->fifd);
387 OPENSSL_free(txp->iovec);
388 OPENSSL_free(txp->conn_close_frame.reason);
389 OPENSSL_free(txp->scratch);
393 void ossl_quic_tx_packetiser_set_initial_token(OSSL_QUIC_TX_PACKETISER *txp,
394 const unsigned char *token,
396 ossl_quic_initial_token_free_fn *free_cb,
399 if (txp->initial_token != NULL && txp->initial_token_free_cb != NULL)
400 txp->initial_token_free_cb(txp->initial_token, txp->initial_token_len,
401 txp->initial_token_free_cb_arg);
403 txp->initial_token = token;
404 txp->initial_token_len = token_len;
405 txp->initial_token_free_cb = free_cb;
406 txp->initial_token_free_cb_arg = free_cb_arg;
409 int ossl_quic_tx_packetiser_set_cur_dcid(OSSL_QUIC_TX_PACKETISER *txp,
410 const QUIC_CONN_ID *dcid)
413 ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER);
417 txp->args.cur_dcid = *dcid;
421 int ossl_quic_tx_packetiser_set_cur_scid(OSSL_QUIC_TX_PACKETISER *txp,
422 const QUIC_CONN_ID *scid)
425 ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER);
429 txp->args.cur_scid = *scid;
433 /* Change the destination L4 address the TXP uses to send datagrams. */
434 int ossl_quic_tx_packetiser_set_peer(OSSL_QUIC_TX_PACKETISER *txp,
435 const BIO_ADDR *peer)
438 ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER);
442 txp->args.peer = *peer;
446 int ossl_quic_tx_packetiser_discard_enc_level(OSSL_QUIC_TX_PACKETISER *txp,
449 if (enc_level >= QUIC_ENC_LEVEL_NUM) {
450 ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT);
454 if (enc_level != QUIC_ENC_LEVEL_0RTT)
455 txp->args.crypto[ossl_quic_enc_level_to_pn_space(enc_level)] = NULL;
460 void ossl_quic_tx_packetiser_notify_handshake_complete(OSSL_QUIC_TX_PACKETISER *txp)
462 txp->handshake_complete = 1;
465 void ossl_quic_tx_packetiser_schedule_handshake_done(OSSL_QUIC_TX_PACKETISER *txp)
467 txp->want_handshake_done = 1;
470 void ossl_quic_tx_packetiser_schedule_ack_eliciting(OSSL_QUIC_TX_PACKETISER *txp,
473 txp->force_ack_eliciting |= (1UL << pn_space);
476 #define TXP_ERR_INTERNAL 0 /* Internal (e.g. alloc) error */
477 #define TXP_ERR_SUCCESS 1 /* Success */
478 #define TXP_ERR_SPACE 2 /* Not enough room for another packet */
479 #define TXP_ERR_INPUT 3 /* Invalid/malformed input */
481 int ossl_quic_tx_packetiser_has_pending(OSSL_QUIC_TX_PACKETISER *txp,
485 uint32_t enc_level, conn_close_enc_level = QUIC_ENC_LEVEL_NUM;
486 int bypass_cc = ((flags & TX_PACKETISER_BYPASS_CC) != 0);
491 || txp->args.cc_method->get_tx_allowance(txp->args.cc_data) > 0);
493 for (enc_level = QUIC_ENC_LEVEL_INITIAL;
494 enc_level < QUIC_ENC_LEVEL_NUM;
496 if (txp_el_pending(txp, enc_level, archetype, cc_can_send,
497 &conn_close_enc_level))
504 * Generates a datagram by polling the various ELs to determine if they want to
505 * generate any frames, and generating a datagram which coalesces packets for
508 int ossl_quic_tx_packetiser_generate(OSSL_QUIC_TX_PACKETISER *txp,
510 int *sent_ack_eliciting)
512 uint32_t enc_level, conn_close_enc_level = QUIC_ENC_LEVEL_NUM;
513 int have_pkt_for_el[QUIC_ENC_LEVEL_NUM], is_last_in_dgram, cc_can_send;
514 size_t num_el_in_dgram = 0, pkts_done = 0;
518 * If CC says we cannot send we still may be able to send any queued probes.
520 cc_can_send = (txp->args.cc_method->get_tx_allowance(txp->args.cc_data) > 0);
522 for (enc_level = QUIC_ENC_LEVEL_INITIAL;
523 enc_level < QUIC_ENC_LEVEL_NUM;
525 have_pkt_for_el[enc_level] = txp_el_pending(txp, enc_level, archetype,
527 &conn_close_enc_level);
528 if (have_pkt_for_el[enc_level])
532 if (num_el_in_dgram == 0)
533 return TX_PACKETISER_RES_NO_PKT;
536 * Should not be needed, but a sanity check in case anyone else has been
539 ossl_qtx_finish_dgram(txp->args.qtx);
541 for (enc_level = QUIC_ENC_LEVEL_INITIAL;
542 enc_level < QUIC_ENC_LEVEL_NUM;
544 if (!have_pkt_for_el[enc_level])
547 is_last_in_dgram = (pkts_done + 1 == num_el_in_dgram);
548 rc = txp_generate_for_el(txp, enc_level, archetype, cc_can_send,
550 have_pkt_for_el[QUIC_ENC_LEVEL_INITIAL],
551 enc_level == conn_close_enc_level,
554 if (rc != TXP_ERR_SUCCESS) {
556 * If we already successfully did at least one, make sure we report
557 * this via the return code.
562 return TX_PACKETISER_RES_FAILURE;
568 ossl_qtx_finish_dgram(txp->args.qtx);
569 return TX_PACKETISER_RES_SENT_PKT;
572 struct archetype_data {
573 unsigned int allow_ack : 1;
574 unsigned int allow_ping : 1;
575 unsigned int allow_crypto : 1;
576 unsigned int allow_handshake_done : 1;
577 unsigned int allow_path_challenge : 1;
578 unsigned int allow_path_response : 1;
579 unsigned int allow_new_conn_id : 1;
580 unsigned int allow_retire_conn_id : 1;
581 unsigned int allow_stream_rel : 1;
582 unsigned int allow_conn_fc : 1;
583 unsigned int allow_conn_close : 1;
584 unsigned int allow_cfq_other : 1;
585 unsigned int allow_new_token : 1;
586 unsigned int allow_force_ack_eliciting : 1;
589 static const struct archetype_data archetypes[QUIC_ENC_LEVEL_NUM][TX_PACKETISER_ARCHETYPE_NUM] = {
592 /* EL 0(INITIAL) - Archetype 0(NORMAL) */
596 /*allow_crypto =*/ 1,
597 /*allow_handshake_done =*/ 0,
598 /*allow_path_challenge =*/ 0,
599 /*allow_path_response =*/ 0,
600 /*allow_new_conn_id =*/ 0,
601 /*allow_retire_conn_id =*/ 0,
602 /*allow_stream_rel =*/ 0,
603 /*allow_conn_fc =*/ 0,
604 /*allow_conn_close =*/ 1,
605 /*allow_cfq_other =*/ 1,
606 /*allow_new_token =*/ 0,
607 /*allow_force_ack_eliciting =*/ 1,
609 /* EL 0(INITIAL) - Archetype 1(ACK_ONLY) */
613 /*allow_crypto =*/ 0,
614 /*allow_handshake_done =*/ 0,
615 /*allow_path_challenge =*/ 0,
616 /*allow_path_response =*/ 0,
617 /*allow_new_conn_id =*/ 0,
618 /*allow_retire_conn_id =*/ 0,
619 /*allow_stream_rel =*/ 0,
620 /*allow_conn_fc =*/ 0,
621 /*allow_conn_close =*/ 0,
622 /*allow_cfq_other =*/ 0,
623 /*allow_new_token =*/ 0,
624 /*allow_force_ack_eliciting =*/ 1,
627 /* EL 1(HANDSHAKE) */
629 /* EL 1(HANDSHAKE) - Archetype 0(NORMAL) */
633 /*allow_crypto =*/ 1,
634 /*allow_handshake_done =*/ 0,
635 /*allow_path_challenge =*/ 0,
636 /*allow_path_response =*/ 0,
637 /*allow_new_conn_id =*/ 0,
638 /*allow_retire_conn_id =*/ 0,
639 /*allow_stream_rel =*/ 0,
640 /*allow_conn_fc =*/ 0,
641 /*allow_conn_close =*/ 1,
642 /*allow_cfq_other =*/ 1,
643 /*allow_new_token =*/ 0,
644 /*allow_force_ack_eliciting =*/ 1,
646 /* EL 1(HANDSHAKE) - Archetype 1(ACK_ONLY) */
650 /*allow_crypto =*/ 0,
651 /*allow_handshake_done =*/ 0,
652 /*allow_path_challenge =*/ 0,
653 /*allow_path_response =*/ 0,
654 /*allow_new_conn_id =*/ 0,
655 /*allow_retire_conn_id =*/ 0,
656 /*allow_stream_rel =*/ 0,
657 /*allow_conn_fc =*/ 0,
658 /*allow_conn_close =*/ 0,
659 /*allow_cfq_other =*/ 0,
660 /*allow_new_token =*/ 0,
661 /*allow_force_ack_eliciting =*/ 1,
666 /* EL 2(0RTT) - Archetype 0(NORMAL) */
670 /*allow_crypto =*/ 0,
671 /*allow_handshake_done =*/ 0,
672 /*allow_path_challenge =*/ 0,
673 /*allow_path_response =*/ 0,
674 /*allow_new_conn_id =*/ 1,
675 /*allow_retire_conn_id =*/ 1,
676 /*allow_stream_rel =*/ 1,
677 /*allow_conn_fc =*/ 1,
678 /*allow_conn_close =*/ 1,
679 /*allow_cfq_other =*/ 0,
680 /*allow_new_token =*/ 0,
681 /*allow_force_ack_eliciting =*/ 0,
683 /* EL 2(0RTT) - Archetype 1(ACK_ONLY) */
687 /*allow_crypto =*/ 0,
688 /*allow_handshake_done =*/ 0,
689 /*allow_path_challenge =*/ 0,
690 /*allow_path_response =*/ 0,
691 /*allow_new_conn_id =*/ 0,
692 /*allow_retire_conn_id =*/ 0,
693 /*allow_stream_rel =*/ 0,
694 /*allow_conn_fc =*/ 0,
695 /*allow_conn_close =*/ 0,
696 /*allow_cfq_other =*/ 0,
697 /*allow_new_token =*/ 0,
698 /*allow_force_ack_eliciting =*/ 0,
703 /* EL 3(1RTT) - Archetype 0(NORMAL) */
707 /*allow_crypto =*/ 1,
708 /*allow_handshake_done =*/ 1,
709 /*allow_path_challenge =*/ 0,
710 /*allow_path_response =*/ 0,
711 /*allow_new_conn_id =*/ 1,
712 /*allow_retire_conn_id =*/ 1,
713 /*allow_stream_rel =*/ 1,
714 /*allow_conn_fc =*/ 1,
715 /*allow_conn_close =*/ 1,
716 /*allow_cfq_other =*/ 1,
717 /*allow_new_token =*/ 1,
718 /*allow_force_ack_eliciting =*/ 1,
720 /* EL 3(1RTT) - Archetype 1(ACK_ONLY) */
724 /*allow_crypto =*/ 0,
725 /*allow_handshake_done =*/ 0,
726 /*allow_path_challenge =*/ 0,
727 /*allow_path_response =*/ 0,
728 /*allow_new_conn_id =*/ 0,
729 /*allow_retire_conn_id =*/ 0,
730 /*allow_stream_rel =*/ 0,
731 /*allow_conn_fc =*/ 0,
732 /*allow_conn_close =*/ 0,
733 /*allow_cfq_other =*/ 0,
734 /*allow_new_token =*/ 0,
735 /*allow_force_ack_eliciting =*/ 1,
740 static int txp_get_archetype_data(uint32_t enc_level,
742 struct archetype_data *a)
744 if (enc_level >= QUIC_ENC_LEVEL_NUM
745 || archetype >= TX_PACKETISER_ARCHETYPE_NUM)
748 /* No need to avoid copying this as it should not exceed one int in size. */
749 *a = archetypes[enc_level][archetype];
754 * Returns 1 if the given EL wants to produce one or more frames.
755 * Always returns 0 if the given EL is discarded.
757 static int txp_el_pending(OSSL_QUIC_TX_PACKETISER *txp, uint32_t enc_level,
760 uint32_t *conn_close_enc_level)
762 struct archetype_data a;
763 uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
764 QUIC_CFQ_ITEM *cfq_item;
766 if (!ossl_qtx_is_enc_level_provisioned(txp->args.qtx, enc_level))
769 if (*conn_close_enc_level > enc_level)
770 *conn_close_enc_level = enc_level;
772 if (!txp_get_archetype_data(enc_level, archetype, &a))
775 /* Do we need to send a PTO probe? */
776 if (a.allow_force_ack_eliciting) {
777 OSSL_ACKM_PROBE_INFO *probe_info
778 = ossl_ackm_get0_probe_request(txp->args.ackm);
780 if ((enc_level == QUIC_ENC_LEVEL_INITIAL
781 && probe_info->anti_deadlock_initial > 0)
782 || (enc_level == QUIC_ENC_LEVEL_HANDSHAKE
783 && probe_info->anti_deadlock_handshake > 0)
784 || probe_info->pto[pn_space] > 0)
789 /* If CC says we cannot currently send, we can only send probes. */
792 /* Does the crypto stream for this EL want to produce anything? */
793 if (a.allow_crypto && sstream_is_pending(txp->args.crypto[pn_space]))
796 /* Does the ACKM for this PN space want to produce anything? */
797 if (a.allow_ack && (ossl_ackm_is_ack_desired(txp->args.ackm, pn_space)
798 || (txp->want_ack & (1UL << pn_space)) != 0))
801 /* Do we need to force emission of an ACK-eliciting packet? */
802 if (a.allow_force_ack_eliciting
803 && (txp->force_ack_eliciting & (1UL << pn_space)) != 0)
806 /* Does the connection-level RXFC want to produce a frame? */
807 if (a.allow_conn_fc && (txp->want_max_data
808 || ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 0)))
811 /* Do we want to produce a MAX_STREAMS frame? */
813 && (txp->want_max_streams_bidi
814 || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_bidi_rxfc,
816 || txp->want_max_streams_uni
817 || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_uni_rxfc,
821 /* Do we want to produce a HANDSHAKE_DONE frame? */
822 if (a.allow_handshake_done && txp->want_handshake_done)
825 /* Do we want to produce a CONNECTION_CLOSE frame? */
826 if (a.allow_conn_close && txp->want_conn_close &&
827 *conn_close_enc_level == enc_level)
829 * This is a bit of a special case since CONNECTION_CLOSE can appear in
830 * most packet types, and when we decide we want to send it this status
831 * isn't tied to a specific EL. So if we want to send it, we send it
832 * only on the lowest non-dropped EL.
836 /* Does the CFQ have any frames queued for this PN space? */
837 if (enc_level != QUIC_ENC_LEVEL_0RTT)
838 for (cfq_item = ossl_quic_cfq_get_priority_head(txp->args.cfq, pn_space);
840 cfq_item = ossl_quic_cfq_item_get_priority_next(cfq_item, pn_space)) {
841 uint64_t frame_type = ossl_quic_cfq_item_get_frame_type(cfq_item);
843 switch (frame_type) {
844 case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID:
845 if (a.allow_new_conn_id)
848 case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID:
849 if (a.allow_retire_conn_id)
852 case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN:
853 if (a.allow_new_token)
857 if (a.allow_cfq_other)
863 if (a.allow_stream_rel && txp->handshake_complete) {
866 /* If there are any active streams, 0/1-RTT wants to produce a packet.
867 * Whether a stream is on the active list is required to be precise
868 * (i.e., a stream is never on the active list if we cannot produce a
869 * frame for it), and all stream-related frames are governed by
870 * a.allow_stream_rel (i.e., if we can send one type of stream-related
871 * frame, we can send any of them), so we don't need to inspect
872 * individual streams on the active list, just confirm that the active
875 ossl_quic_stream_iter_init(&it, txp->args.qsm, 0);
876 if (it.stream != NULL)
883 static int sstream_is_pending(QUIC_SSTREAM *sstream)
885 OSSL_QUIC_FRAME_STREAM hdr;
886 OSSL_QTX_IOVEC iov[2];
887 size_t num_iov = OSSL_NELEM(iov);
889 return ossl_quic_sstream_get_stream_frame(sstream, 0, &hdr, iov, &num_iov);
893 * Generates a packet for a given EL, coalescing it into the current datagram.
895 * is_last_in_dgram and dgram_contains_initial are used to determine padding
898 * Returns TXP_ERR_* value.
900 static int txp_generate_for_el(OSSL_QUIC_TX_PACKETISER *txp, uint32_t enc_level,
903 int is_last_in_dgram,
904 int dgram_contains_initial,
905 int chosen_for_conn_close,
906 int *sent_ack_eliciting)
908 int must_pad = dgram_contains_initial && is_last_in_dgram;
909 size_t min_dpl, min_pl, min_ppl, cmpl, cmppl, running_total;
910 size_t mdpl, hdr_len, pkt_overhead, cc_limit;
914 /* Determine the limit CC imposes on what we can send. */
917 * If we are called when we cannot send, this must be because we want
918 * to generate a probe. In this circumstance, don't clamp based on CC.
922 /* Allow CC to clamp how much we can send. */
923 cc_limit_ = txp->args.cc_method->get_tx_allowance(txp->args.cc_data);
924 cc_limit = (cc_limit_ > SIZE_MAX ? SIZE_MAX : (size_t)cc_limit_);
927 /* Assemble packet header. */
928 phdr.type = ossl_quic_enc_level_to_pkt_type(enc_level);
930 phdr.pn_len = txp_determine_pn_len(txp);
933 phdr.version = QUIC_VERSION_1;
934 phdr.dst_conn_id = txp->args.cur_dcid;
935 phdr.src_conn_id = txp->args.cur_scid;
938 * We need to know the length of the payload to get an accurate header
939 * length for non-1RTT packets, because the Length field found in
940 * Initial/Handshake/0-RTT packets uses a variable-length encoding. However,
941 * we don't have a good idea of the length of our payload, because the
942 * length of the payload depends on the room in the datagram after fitting
943 * the header, which depends on the size of the header.
945 * In general, it does not matter if a packet is slightly shorter (because
946 * e.g. we predicted use of a 2-byte length field, but ended up only needing
947 * a 1-byte length field). However this does matter for Initial packets
948 * which must be at least 1200 bytes, which is also the assumed default MTU;
949 * therefore in many cases Initial packets will be padded to 1200 bytes,
950 * which means if we overestimated the header size, we will be short by a
951 * few bytes and the server will ignore the packet for being too short. In
952 * this case, however, such packets always *will* be padded to meet 1200
953 * bytes, which requires a 2-byte length field, so we don't actually need to
954 * worry about this. Thus we estimate the header length assuming a 2-byte
955 * length field here, which should in practice work well in all cases.
957 phdr.len = OSSL_QUIC_VLINT_2B_MAX - phdr.pn_len;
959 if (enc_level == QUIC_ENC_LEVEL_INITIAL) {
960 phdr.token = txp->initial_token;
961 phdr.token_len = txp->initial_token_len;
967 hdr_len = ossl_quic_wire_get_encoded_pkt_hdr_len(phdr.dst_conn_id.id_len,
970 return TXP_ERR_INPUT;
972 /* MinDPL: Minimum total datagram payload length. */
973 min_dpl = must_pad ? QUIC_MIN_INITIAL_DGRAM_LEN : 0;
975 /* How much data is already in the current datagram? */
976 running_total = ossl_qtx_get_cur_dgram_len_bytes(txp->args.qtx);
978 /* MinPL: Minimum length of the fully encoded packet. */
979 min_pl = running_total < min_dpl ? min_dpl - running_total : 0;
980 if ((uint64_t)min_pl > cc_limit)
982 * Congestion control does not allow us to send a packet of adequate
985 return TXP_ERR_SPACE;
987 /* MinPPL: Minimum plaintext payload length needed to meet MinPL. */
988 if (!txp_determine_ppl_from_pl(txp, min_pl, enc_level, hdr_len, &min_ppl))
989 /* MinPL is less than a valid packet size, so just use a MinPPL of 0. */
992 /* MDPL: Maximum datagram payload length. */
993 mdpl = txp_get_mdpl(txp);
996 * CMPL: Maximum encoded packet size we can put into this datagram given any
997 * previous packets coalesced into it.
999 if (running_total > mdpl)
1000 /* Should not be possible, but if it happens: */
1003 cmpl = mdpl - running_total;
1005 /* Clamp CMPL based on congestion control limit. */
1006 if (cmpl > cc_limit)
1009 /* CMPPL: Maximum amount we can put into the current datagram payload. */
1010 if (!txp_determine_ppl_from_pl(txp, cmpl, enc_level, hdr_len, &cmppl))
1011 return TXP_ERR_SPACE;
1013 /* Packet overhead (size of headers, AEAD tag, etc.) */
1014 pkt_overhead = cmpl - cmppl;
1016 return txp_generate_for_el_actual(txp, enc_level, archetype, min_ppl, cmppl,
1017 pkt_overhead, &phdr,
1018 chosen_for_conn_close,
1019 sent_ack_eliciting);
1022 /* Determine how many bytes we should use for the encoded PN. */
1023 static size_t txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER *txp)
1025 return 4; /* TODO(QUIC) */
1028 /* Determine plaintext packet payload length from payload length. */
1029 static int txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER *txp,
1040 if (!ossl_qtx_calculate_plaintext_payload_len(txp->args.qtx, enc_level,
1048 static size_t txp_get_mdpl(OSSL_QUIC_TX_PACKETISER *txp)
1050 return ossl_qtx_get_mdpl(txp->args.qtx);
1053 static QUIC_SSTREAM *get_sstream_by_id(uint64_t stream_id, uint32_t pn_space,
1056 OSSL_QUIC_TX_PACKETISER *txp = arg;
1059 if (stream_id == UINT64_MAX)
1060 return txp->args.crypto[pn_space];
1062 s = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1069 static void on_regen_notify(uint64_t frame_type, uint64_t stream_id,
1070 QUIC_TXPIM_PKT *pkt, void *arg)
1072 OSSL_QUIC_TX_PACKETISER *txp = arg;
1074 switch (frame_type) {
1075 case OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE:
1076 txp->want_handshake_done = 1;
1078 case OSSL_QUIC_FRAME_TYPE_MAX_DATA:
1079 txp->want_max_data = 1;
1081 case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI:
1082 txp->want_max_streams_bidi = 1;
1084 case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_UNI:
1085 txp->want_max_streams_uni = 1;
1087 case OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN:
1088 txp->want_ack |= (1UL << pkt->ackm_pkt.pkt_space);
1090 case OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA:
1093 = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1098 s->want_max_stream_data = 1;
1099 ossl_quic_stream_map_update_state(txp->args.qsm, s);
1102 case OSSL_QUIC_FRAME_TYPE_STOP_SENDING:
1105 = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1110 s->want_stop_sending = 1;
1111 ossl_quic_stream_map_update_state(txp->args.qsm, s);
1114 case OSSL_QUIC_FRAME_TYPE_RESET_STREAM:
1117 = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1122 s->want_reset_stream = 1;
1123 ossl_quic_stream_map_update_state(txp->args.qsm, s);
1132 static void on_sstream_updated(uint64_t stream_id, void *arg)
1134 OSSL_QUIC_TX_PACKETISER *txp = arg;
1137 s = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1141 ossl_quic_stream_map_update_state(txp->args.qsm, s);
1144 static int txp_generate_pre_token(OSSL_QUIC_TX_PACKETISER *txp,
1145 struct tx_helper *h,
1146 QUIC_TXPIM_PKT *tpkt,
1148 struct archetype_data *a,
1149 int chosen_for_conn_close)
1151 const OSSL_QUIC_FRAME_ACK *ack;
1152 OSSL_QUIC_FRAME_ACK ack2;
1154 tpkt->ackm_pkt.largest_acked = QUIC_PN_INVALID;
1156 /* ACK Frames (Regenerate) */
1158 && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_ACK
1160 || ossl_ackm_is_ack_desired(txp->args.ackm, pn_space))
1161 && (ack = ossl_ackm_get_ack_frame(txp->args.ackm, pn_space)) != NULL) {
1162 WPACKET *wpkt = tx_helper_begin(h);
1167 /* We do not currently support ECN */
1169 ack2.ecn_present = 0;
1171 if (ossl_quic_wire_encode_frame_ack(wpkt,
1172 txp->args.ack_delay_exponent,
1174 if (!tx_helper_commit(h))
1177 tpkt->had_ack_frame = 1;
1179 if (ack->num_ack_ranges > 0)
1180 tpkt->ackm_pkt.largest_acked = ack->ack_ranges[0].end;
1182 tx_helper_rollback(h);
1186 /* CONNECTION_CLOSE Frames (Regenerate) */
1187 if (a->allow_conn_close && txp->want_conn_close && chosen_for_conn_close) {
1188 WPACKET *wpkt = tx_helper_begin(h);
1193 if (ossl_quic_wire_encode_frame_conn_close(wpkt,
1194 &txp->conn_close_frame)) {
1195 if (!tx_helper_commit(h))
1198 tx_helper_rollback(h);
1205 static int try_len(size_t space_left, size_t orig_len,
1206 size_t base_hdr_len, size_t lenbytes,
1207 uint64_t maxn, size_t *hdr_len, size_t *payload_len)
1210 size_t maxn_ = maxn > SIZE_MAX ? SIZE_MAX : (size_t)maxn;
1212 *hdr_len = base_hdr_len + lenbytes;
1214 if (orig_len == 0 && space_left >= *hdr_len) {
1222 if (n + *hdr_len > space_left)
1223 n = (space_left >= *hdr_len) ? space_left - *hdr_len : 0;
1229 static int determine_len(size_t space_left, size_t orig_len,
1230 size_t base_hdr_len,
1231 uint64_t *hlen, uint64_t *len)
1234 size_t chosen_payload_len = 0;
1235 size_t chosen_hdr_len = 0;
1236 size_t payload_len[4], hdr_len[4];
1237 int i, valid[4] = {0};
1239 valid[0] = try_len(space_left, orig_len, base_hdr_len,
1240 1, OSSL_QUIC_VLINT_1B_MAX,
1241 &hdr_len[0], &payload_len[0]);
1242 valid[1] = try_len(space_left, orig_len, base_hdr_len,
1243 2, OSSL_QUIC_VLINT_2B_MAX,
1244 &hdr_len[1], &payload_len[1]);
1245 valid[2] = try_len(space_left, orig_len, base_hdr_len,
1246 4, OSSL_QUIC_VLINT_4B_MAX,
1247 &hdr_len[2], &payload_len[2]);
1248 valid[3] = try_len(space_left, orig_len, base_hdr_len,
1249 8, OSSL_QUIC_VLINT_8B_MAX,
1250 &hdr_len[3], &payload_len[3]);
1252 for (i = OSSL_NELEM(valid) - 1; i >= 0; --i)
1253 if (valid[i] && payload_len[i] >= chosen_payload_len) {
1254 chosen_payload_len = payload_len[i];
1255 chosen_hdr_len = hdr_len[i];
1259 *hlen = chosen_hdr_len;
1260 *len = chosen_payload_len;
1265 * Given a CRYPTO frame header with accurate chdr->len and a budget
1266 * (space_left), try to find the optimal value of chdr->len to fill as much of
1267 * the budget as possible. This is slightly hairy because larger values of
1268 * chdr->len cause larger encoded sizes of the length field of the frame, which
1269 * in turn mean less space available for payload data. We check all possible
1270 * encodings and choose the optimal encoding.
1272 static int determine_crypto_len(struct tx_helper *h,
1273 OSSL_QUIC_FRAME_CRYPTO *chdr,
1279 size_t base_hdr_len; /* CRYPTO header length without length field */
1281 if (chdr->len > SIZE_MAX)
1284 orig_len = (size_t)chdr->len;
1287 base_hdr_len = ossl_quic_wire_get_encoded_frame_len_crypto_hdr(chdr);
1288 chdr->len = orig_len;
1289 if (base_hdr_len == 0)
1294 return determine_len(space_left, orig_len, base_hdr_len, hlen, len);
1297 static int determine_stream_len(struct tx_helper *h,
1298 OSSL_QUIC_FRAME_STREAM *shdr,
1304 size_t base_hdr_len; /* STREAM header length without length field */
1306 if (shdr->len > SIZE_MAX)
1309 orig_len = (size_t)shdr->len;
1312 base_hdr_len = ossl_quic_wire_get_encoded_frame_len_stream_hdr(shdr);
1313 shdr->len = orig_len;
1314 if (base_hdr_len == 0)
1317 if (shdr->has_explicit_len)
1320 return determine_len(space_left, orig_len, base_hdr_len, hlen, len);
1323 static int txp_generate_crypto_frames(OSSL_QUIC_TX_PACKETISER *txp,
1324 struct tx_helper *h,
1326 QUIC_TXPIM_PKT *tpkt,
1327 int *have_ack_eliciting)
1329 size_t num_stream_iovec;
1330 OSSL_QUIC_FRAME_STREAM shdr = {0};
1331 OSSL_QUIC_FRAME_CRYPTO chdr = {0};
1332 OSSL_QTX_IOVEC iov[2];
1335 QUIC_TXPIM_CHUNK chunk = {0};
1336 size_t i, space_left;
1339 space_left = tx_helper_get_space_left(h);
1341 if (space_left < MIN_FRAME_SIZE_CRYPTO)
1342 return 1; /* no point trying */
1344 /* Do we have any CRYPTO data waiting? */
1345 num_stream_iovec = OSSL_NELEM(iov);
1346 if (!ossl_quic_sstream_get_stream_frame(txp->args.crypto[pn_space],
1349 return 1; /* nothing to do */
1351 /* Convert STREAM frame header to CRYPTO frame header */
1352 chdr.offset = shdr.offset;
1353 chdr.len = shdr.len;
1356 return 1; /* nothing to do */
1358 /* Find best fit (header length, payload length) combination. */
1359 if (!determine_crypto_len(h, &chdr, space_left, &hdr_bytes,
1361 return 1; /* can't fit anything */
1364 * Truncate IOVs to match our chosen length.
1366 * The length cannot be more than SIZE_MAX because this length comes
1367 * from our send stream buffer.
1369 ossl_quic_sstream_adjust_iov((size_t)chdr.len, iov, num_stream_iovec);
1372 * Ensure we have enough iovecs allocated (1 for the header, up to 2 for
1373 * the the stream data.)
1375 if (!txp_ensure_iovec(txp, h->num_iovec + 3))
1376 return 0; /* alloc error */
1378 /* Encode the header. */
1379 wpkt = tx_helper_begin(h);
1381 return 0; /* alloc error */
1383 if (!ossl_quic_wire_encode_frame_crypto_hdr(wpkt, &chdr)) {
1384 tx_helper_rollback(h);
1385 return 1; /* can't fit */
1388 if (!tx_helper_commit(h))
1389 return 0; /* alloc error */
1391 /* Add payload iovecs to the helper (infallible). */
1392 for (i = 0; i < num_stream_iovec; ++i)
1393 tx_helper_append_iovec(h, iov[i].buf, iov[i].buf_len);
1395 *have_ack_eliciting = 1;
1396 tx_helper_unrestrict(h); /* no longer need PING */
1398 /* Log chunk to TXPIM. */
1399 chunk.stream_id = UINT64_MAX; /* crypto stream */
1400 chunk.start = chdr.offset;
1401 chunk.end = chdr.offset + chdr.len - 1;
1402 chunk.has_fin = 0; /* Crypto stream never ends */
1403 if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk))
1404 return 0; /* alloc error */
1409 OSSL_QUIC_FRAME_STREAM shdr;
1410 OSSL_QTX_IOVEC iov[2];
1411 size_t num_stream_iovec;
1415 static int txp_plan_stream_chunk(OSSL_QUIC_TX_PACKETISER *txp,
1416 struct tx_helper *h,
1417 QUIC_SSTREAM *sstream,
1418 QUIC_TXFC *stream_txfc,
1420 struct chunk_info *chunk)
1422 uint64_t fc_credit, fc_swm, fc_limit;
1424 chunk->num_stream_iovec = OSSL_NELEM(chunk->iov);
1425 chunk->valid = ossl_quic_sstream_get_stream_frame(sstream, skip,
1428 &chunk->num_stream_iovec);
1432 if (!ossl_assert(chunk->shdr.len > 0 || chunk->shdr.is_fin))
1433 /* Should only have 0-length chunk if FIN */
1436 /* Clamp according to connection and stream-level TXFC. */
1437 fc_credit = ossl_quic_txfc_get_credit(stream_txfc);
1438 fc_swm = ossl_quic_txfc_get_swm(stream_txfc);
1439 fc_limit = fc_swm + fc_credit;
1441 if (chunk->shdr.len > 0 && chunk->shdr.offset + chunk->shdr.len > fc_limit) {
1442 chunk->shdr.len = (fc_limit <= chunk->shdr.offset)
1443 ? 0 : fc_limit - chunk->shdr.offset;
1444 chunk->shdr.is_fin = 0;
1447 if (chunk->shdr.len == 0 && !chunk->shdr.is_fin) {
1449 * Nothing to do due to TXFC. Since SSTREAM returns chunks in ascending
1450 * order of offset we don't need to check any later chunks, so stop
1461 * Returns 0 on fatal error (e.g. allocation failure), 1 on success.
1462 * *packet_full is set to 1 if there is no longer enough room for another STREAM
1463 * frame, and *stream_drained is set to 1 if all stream buffers have now been
1466 static int txp_generate_stream_frames(OSSL_QUIC_TX_PACKETISER *txp,
1467 struct tx_helper *h,
1469 QUIC_TXPIM_PKT *tpkt,
1471 QUIC_SSTREAM *sstream,
1472 QUIC_TXFC *stream_txfc,
1473 QUIC_STREAM *next_stream,
1475 int *have_ack_eliciting,
1477 int *stream_drained,
1478 uint64_t *new_credit_consumed)
1481 struct chunk_info chunks[2] = {0};
1483 OSSL_QUIC_FRAME_STREAM *shdr;
1485 QUIC_TXPIM_CHUNK chunk;
1486 size_t i, j, space_left;
1487 int needs_padding_if_implicit, can_fill_payload, use_explicit_len;
1488 int could_have_following_chunk;
1490 uint64_t hdr_len_implicit, payload_len_implicit;
1491 uint64_t hdr_len_explicit, payload_len_explicit;
1492 uint64_t fc_swm, fc_new_hwm;
1494 fc_swm = ossl_quic_txfc_get_swm(stream_txfc);
1495 fc_new_hwm = fc_swm;
1498 * Load the first two chunks if any offered by the send stream. We retrieve
1499 * the next chunk in advance so we can determine if we need to send any more
1500 * chunks from the same stream after this one, which is needed when
1501 * determining when we can use an implicit length in a STREAM frame.
1503 for (i = 0; i < 2; ++i) {
1504 if (!txp_plan_stream_chunk(txp, h, sstream, stream_txfc, i, &chunks[i]))
1507 if (i == 0 && !chunks[i].valid) {
1508 /* No chunks, nothing to do. */
1509 *stream_drained = 1;
1516 space_left = tx_helper_get_space_left(h);
1518 if (!chunks[i % 2].valid) {
1519 /* Out of chunks; we're done. */
1520 *stream_drained = 1;
1525 if (space_left < MIN_FRAME_SIZE_STREAM) {
1531 if (!ossl_assert(!h->done_implicit))
1533 * Logic below should have ensured we didn't append an
1534 * implicit-length unless we filled the packet or didn't have
1535 * another stream to handle, so this should not be possible.
1539 shdr = &chunks[i % 2].shdr;
1540 orig_len = shdr->len;
1542 /* Load next chunk for lookahead. */
1543 if (!txp_plan_stream_chunk(txp, h, sstream, stream_txfc, i + 1,
1544 &chunks[(i + 1) % 2]))
1548 * Find best fit (header length, payload length) combination for if we
1549 * use an implicit length.
1551 shdr->has_explicit_len = 0;
1552 hdr_len_implicit = payload_len_implicit = 0;
1553 if (!determine_stream_len(h, shdr, space_left,
1554 &hdr_len_implicit, &payload_len_implicit)) {
1557 goto err; /* can't fit anything */
1561 * If using the implicit-length representation would need padding, we
1564 needs_padding_if_implicit = (h->bytes_appended + hdr_len_implicit
1565 + payload_len_implicit < min_ppl);
1568 * If there is a next stream, we don't use the implicit length so we can
1569 * add more STREAM frames after this one, unless there is enough data
1570 * for this STREAM frame to fill the packet.
1572 can_fill_payload = (hdr_len_implicit + payload_len_implicit
1576 * Is there is a stream after this one, or another chunk pending
1577 * transmission in this stream?
1579 could_have_following_chunk
1580 = (next_stream != NULL || chunks[(i + 1) % 2].valid);
1582 /* Choose between explicit or implicit length representations. */
1583 use_explicit_len = !((can_fill_payload || !could_have_following_chunk)
1584 && !needs_padding_if_implicit);
1586 if (use_explicit_len) {
1588 * Find best fit (header length, payload length) combination for if
1589 * we use an explicit length.
1591 shdr->has_explicit_len = 1;
1592 hdr_len_explicit = payload_len_explicit = 0;
1593 if (!determine_stream_len(h, shdr, space_left,
1594 &hdr_len_explicit, &payload_len_explicit)) {
1597 goto err; /* can't fit anything */
1600 shdr->len = payload_len_explicit;
1602 shdr->has_explicit_len = 0;
1603 shdr->len = payload_len_implicit;
1606 /* If this is a FIN, don't keep filling the packet with more FINs. */
1608 chunks[(i + 1) % 2].valid = 0;
1610 /* Truncate IOVs to match our chosen length. */
1611 ossl_quic_sstream_adjust_iov((size_t)shdr->len, chunks[i % 2].iov,
1612 chunks[i % 2].num_stream_iovec);
1615 * Ensure we have enough iovecs allocated (1 for the header, up to 2 for
1616 * the the stream data.)
1618 if (!txp_ensure_iovec(txp, h->num_iovec + 3))
1619 goto err; /* alloc error */
1621 /* Encode the header. */
1622 wpkt = tx_helper_begin(h);
1624 goto err; /* alloc error */
1626 shdr->stream_id = id;
1627 if (!ossl_assert(ossl_quic_wire_encode_frame_stream_hdr(wpkt, shdr))) {
1628 /* (Should not be possible.) */
1629 tx_helper_rollback(h);
1632 goto err; /* can't fit */
1635 if (!tx_helper_commit(h))
1636 goto err; /* alloc error */
1638 /* Add payload iovecs to the helper (infallible). */
1639 for (j = 0; j < chunks[i % 2].num_stream_iovec; ++j)
1640 tx_helper_append_iovec(h, chunks[i % 2].iov[j].buf,
1641 chunks[i % 2].iov[j].buf_len);
1643 *have_ack_eliciting = 1;
1644 tx_helper_unrestrict(h); /* no longer need PING */
1645 if (!shdr->has_explicit_len)
1646 h->done_implicit = 1;
1648 /* Log new TXFC credit which was consumed. */
1649 if (shdr->len > 0 && shdr->offset + shdr->len > fc_new_hwm)
1650 fc_new_hwm = shdr->offset + shdr->len;
1652 /* Log chunk to TXPIM. */
1653 chunk.stream_id = shdr->stream_id;
1654 chunk.start = shdr->offset;
1655 chunk.end = shdr->offset + shdr->len - 1;
1656 chunk.has_fin = shdr->is_fin;
1657 chunk.has_stop_sending = 0;
1658 chunk.has_reset_stream = 0;
1659 if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk))
1660 goto err; /* alloc error */
1662 if (shdr->len < orig_len) {
1664 * If we did not serialize all of this chunk we definitely do not
1665 * want to try the next chunk (and we must not mark the stream
1674 *new_credit_consumed = fc_new_hwm - fc_swm;
1678 static void txp_enlink_tmp(QUIC_STREAM **tmp_head, QUIC_STREAM *stream)
1680 stream->txp_next = *tmp_head;
1684 static int txp_generate_stream_related(OSSL_QUIC_TX_PACKETISER *txp,
1685 struct tx_helper *h,
1687 QUIC_TXPIM_PKT *tpkt,
1689 int *have_ack_eliciting,
1690 QUIC_STREAM **tmp_head)
1692 QUIC_STREAM_ITER it;
1696 QUIC_STREAM *stream, *snext;
1698 for (ossl_quic_stream_iter_init(&it, txp->args.qsm, 1);
1699 it.stream != NULL;) {
1702 ossl_quic_stream_iter_next(&it);
1705 stream->txp_sent_fc = 0;
1706 stream->txp_sent_stop_sending = 0;
1707 stream->txp_sent_reset_stream = 0;
1708 stream->txp_drained = 0;
1709 stream->txp_blocked = 0;
1710 stream->txp_txfc_new_credit_consumed = 0;
1712 rstream = stream->rstream;
1714 /* Stream Abort Frames (STOP_SENDING, RESET_STREAM) */
1715 if (stream->want_stop_sending) {
1716 OSSL_QUIC_FRAME_STOP_SENDING f;
1718 wpkt = tx_helper_begin(h);
1720 return 0; /* alloc error */
1722 f.stream_id = stream->id;
1723 f.app_error_code = stream->stop_sending_aec;
1724 if (!ossl_quic_wire_encode_frame_stop_sending(wpkt, &f)) {
1725 tx_helper_rollback(h); /* can't fit */
1726 txp_enlink_tmp(tmp_head, stream);
1730 if (!tx_helper_commit(h))
1731 return 0; /* alloc error */
1733 *have_ack_eliciting = 1;
1734 tx_helper_unrestrict(h); /* no longer need PING */
1735 stream->txp_sent_stop_sending = 1;
1738 if (stream->want_reset_stream) {
1739 OSSL_QUIC_FRAME_RESET_STREAM f;
1741 wpkt = tx_helper_begin(h);
1743 return 0; /* alloc error */
1745 f.stream_id = stream->id;
1746 f.app_error_code = stream->reset_stream_aec;
1747 f.final_size = ossl_quic_sstream_get_cur_size(stream->sstream);
1748 if (!ossl_quic_wire_encode_frame_reset_stream(wpkt, &f)) {
1749 tx_helper_rollback(h); /* can't fit */
1750 txp_enlink_tmp(tmp_head, stream);
1754 if (!tx_helper_commit(h))
1755 return 0; /* alloc error */
1757 *have_ack_eliciting = 1;
1758 tx_helper_unrestrict(h); /* no longer need PING */
1759 stream->txp_sent_reset_stream = 1;
1762 /* Stream Flow Control Frames (MAX_STREAM_DATA) */
1764 && (stream->want_max_stream_data
1765 || ossl_quic_rxfc_has_cwm_changed(&stream->rxfc, 0))) {
1767 wpkt = tx_helper_begin(h);
1769 return 0; /* alloc error */
1771 cwm = ossl_quic_rxfc_get_cwm(&stream->rxfc);
1773 if (!ossl_quic_wire_encode_frame_max_stream_data(wpkt, stream->id,
1775 tx_helper_rollback(h); /* can't fit */
1776 txp_enlink_tmp(tmp_head, stream);
1780 if (!tx_helper_commit(h))
1781 return 0; /* alloc error */
1783 *have_ack_eliciting = 1;
1784 tx_helper_unrestrict(h); /* no longer need PING */
1785 stream->txp_sent_fc = 1;
1788 /* Stream Data Frames (STREAM) */
1789 if (stream->sstream != NULL) {
1790 int packet_full = 0, stream_drained = 0;
1792 if (!txp_generate_stream_frames(txp, h, pn_space, tpkt,
1793 stream->id, stream->sstream,
1799 &stream->txp_txfc_new_credit_consumed)) {
1800 /* Fatal error (allocation, etc.) */
1801 txp_enlink_tmp(tmp_head, stream);
1806 stream->txp_drained = 1;
1809 txp_enlink_tmp(tmp_head, stream);
1814 txp_enlink_tmp(tmp_head, stream);
1821 * Generates a packet for a given EL with the given minimum and maximum
1822 * plaintext packet payload lengths. Returns TXP_ERR_* value.
1824 static int txp_generate_for_el_actual(OSSL_QUIC_TX_PACKETISER *txp,
1829 size_t pkt_overhead,
1831 int chosen_for_conn_close,
1832 int *sent_ack_eliciting)
1834 int rc = TXP_ERR_SUCCESS;
1835 struct archetype_data a;
1836 uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
1838 int have_helper = 0, have_ack_eliciting = 0, done_pre_token = 0;
1839 int require_ack_eliciting = 0;
1840 QUIC_CFQ_ITEM *cfq_item;
1841 QUIC_TXPIM_PKT *tpkt = NULL;
1843 QUIC_STREAM *tmp_head = NULL, *stream;
1844 OSSL_ACKM_PROBE_INFO *probe_info
1845 = ossl_ackm_get0_probe_request(txp->args.ackm);
1847 if (!txp_get_archetype_data(enc_level, archetype, &a))
1850 if (a.allow_force_ack_eliciting) {
1852 * Make this packet ACK-eliciting if it has been explicitly requested,
1853 * or if ACKM has requested a probe for this PN space.
1855 if ((txp->force_ack_eliciting & (1UL << pn_space)) != 0
1856 || (enc_level == QUIC_ENC_LEVEL_INITIAL
1857 && probe_info->anti_deadlock_initial > 0)
1858 || (enc_level == QUIC_ENC_LEVEL_HANDSHAKE
1859 && probe_info->anti_deadlock_handshake > 0)
1860 || probe_info->pto[pn_space] > 0)
1861 require_ack_eliciting = 1;
1864 /* Minimum cannot be bigger than maximum. */
1865 if (min_ppl > max_ppl)
1868 /* Maximum PN reached? */
1869 if (txp->next_pn[pn_space] >= (((QUIC_PN)1) << 62))
1872 if ((tpkt = ossl_quic_txpim_pkt_alloc(txp->args.txpim)) == NULL)
1876 * Initialise TX helper. If we must be ACK eliciting, reserve 1 byte for
1879 if (!tx_helper_init(&h, txp, max_ppl, require_ack_eliciting ? 1 : 0))
1885 * Frame Serialization
1886 * ===================
1888 * We now serialize frames into the packet in descending order of priority.
1891 /* HANDSHAKE_DONE (Regenerate) */
1892 if (a.allow_handshake_done && txp->want_handshake_done
1893 && tx_helper_get_space_left(&h) >= MIN_FRAME_SIZE_HANDSHAKE_DONE) {
1894 WPACKET *wpkt = tx_helper_begin(&h);
1899 if (ossl_quic_wire_encode_frame_handshake_done(wpkt)) {
1900 tpkt->had_handshake_done_frame = 1;
1901 have_ack_eliciting = 1;
1903 if (!tx_helper_commit(&h))
1906 tx_helper_unrestrict(&h); /* no longer need PING */
1908 tx_helper_rollback(&h);
1912 /* MAX_DATA (Regenerate) */
1914 && (txp->want_max_data
1915 || ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 0))
1916 && tx_helper_get_space_left(&h) >= MIN_FRAME_SIZE_MAX_DATA) {
1917 WPACKET *wpkt = tx_helper_begin(&h);
1918 uint64_t cwm = ossl_quic_rxfc_get_cwm(txp->args.conn_rxfc);
1923 if (ossl_quic_wire_encode_frame_max_data(wpkt, cwm)) {
1924 tpkt->had_max_data_frame = 1;
1925 have_ack_eliciting = 1;
1927 if (!tx_helper_commit(&h))
1930 tx_helper_unrestrict(&h); /* no longer need PING */
1932 tx_helper_rollback(&h);
1936 /* MAX_STREAMS_BIDI (Regenerate) */
1938 && (txp->want_max_streams_bidi
1939 || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_bidi_rxfc, 0))
1940 && tx_helper_get_space_left(&h) >= MIN_FRAME_SIZE_MAX_STREAMS_BIDI) {
1941 WPACKET *wpkt = tx_helper_begin(&h);
1942 uint64_t max_streams
1943 = ossl_quic_rxfc_get_cwm(txp->args.max_streams_bidi_rxfc);
1948 if (ossl_quic_wire_encode_frame_max_streams(wpkt, /*is_uni=*/0,
1950 tpkt->had_max_streams_bidi_frame = 1;
1951 have_ack_eliciting = 1;
1953 if (!tx_helper_commit(&h))
1956 tx_helper_unrestrict(&h); /* no longer need PING */
1958 tx_helper_rollback(&h);
1962 /* MAX_STREAMS_UNI (Regenerate) */
1964 && (txp->want_max_streams_uni
1965 || ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_uni_rxfc, 0))
1966 && tx_helper_get_space_left(&h) >= MIN_FRAME_SIZE_MAX_STREAMS_UNI) {
1967 WPACKET *wpkt = tx_helper_begin(&h);
1968 uint64_t max_streams
1969 = ossl_quic_rxfc_get_cwm(txp->args.max_streams_uni_rxfc);
1974 if (ossl_quic_wire_encode_frame_max_streams(wpkt, /*is_uni=*/1,
1976 tpkt->had_max_streams_uni_frame = 1;
1977 have_ack_eliciting = 1;
1979 if (!tx_helper_commit(&h))
1982 tx_helper_unrestrict(&h); /* no longer need PING */
1984 tx_helper_rollback(&h);
1989 for (cfq_item = ossl_quic_cfq_get_priority_head(txp->args.cfq, pn_space);
1991 cfq_item = ossl_quic_cfq_item_get_priority_next(cfq_item, pn_space)) {
1992 uint64_t frame_type = ossl_quic_cfq_item_get_frame_type(cfq_item);
1993 const unsigned char *encoded = ossl_quic_cfq_item_get_encoded(cfq_item);
1994 size_t encoded_len = ossl_quic_cfq_item_get_encoded_len(cfq_item);
1996 switch (frame_type) {
1997 case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID:
1998 if (!a.allow_new_conn_id)
2001 case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID:
2002 if (!a.allow_retire_conn_id)
2005 case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN:
2006 if (!a.allow_new_token)
2010 * NEW_TOKEN frames are handled via GCR, but some
2011 * Regenerate-strategy frames should come before them (namely
2012 * ACK, CONNECTION_CLOSE, PATH_CHALLENGE and PATH_RESPONSE). If
2013 * we find a NEW_TOKEN frame, do these now. If there are no
2014 * NEW_TOKEN frames in the GCR queue we will handle these below.
2016 if (!done_pre_token)
2017 if (txp_generate_pre_token(txp, &h, tpkt, pn_space, &a,
2018 chosen_for_conn_close))
2023 if (!a.allow_cfq_other)
2029 * If the frame is too big, don't try to schedule any more GCR frames in
2030 * this packet rather than sending subsequent ones out of order.
2032 if (encoded_len > tx_helper_get_space_left(&h))
2035 if (!tx_helper_append_iovec(&h, encoded, encoded_len))
2038 ossl_quic_txpim_pkt_add_cfq_item(tpkt, cfq_item);
2040 if (ossl_quic_frame_type_is_ack_eliciting(frame_type)) {
2041 have_ack_eliciting = 1;
2042 tx_helper_unrestrict(&h); /* no longer need PING */
2047 * If we didn't generate ACK, CONNECTION_CLOSE, PATH_CHALLENGE or
2048 * PATH_RESPONSE (as desired) before, do so now.
2050 if (!done_pre_token)
2051 if (txp_generate_pre_token(txp, &h, tpkt, pn_space, &a,
2052 chosen_for_conn_close))
2057 if (!txp_generate_crypto_frames(txp, &h, pn_space, tpkt,
2058 &have_ack_eliciting))
2061 /* Stream-specific frames */
2062 if (a.allow_stream_rel && txp->handshake_complete)
2063 if (!txp_generate_stream_related(txp, &h, pn_space, tpkt, min_ppl,
2064 &have_ack_eliciting,
2069 tx_helper_unrestrict(&h);
2071 if (require_ack_eliciting && !have_ack_eliciting && a.allow_ping) {
2074 wpkt = tx_helper_begin(&h);
2078 if (!ossl_quic_wire_encode_frame_ping(wpkt)
2079 || !tx_helper_commit(&h))
2081 * We treat a request to be ACK-eliciting as a requirement, so this
2086 have_ack_eliciting = 1;
2090 if (h.bytes_appended < min_ppl) {
2091 WPACKET *wpkt = tx_helper_begin(&h);
2095 if (!ossl_quic_wire_encode_padding(wpkt, min_ppl - h.bytes_appended)
2096 || !tx_helper_commit(&h))
2105 tpkt->ackm_pkt.num_bytes = h.bytes_appended + pkt_overhead;
2106 tpkt->ackm_pkt.pkt_num = txp->next_pn[pn_space];
2107 /* largest_acked is set in txp_generate_pre_token */
2108 tpkt->ackm_pkt.pkt_space = pn_space;
2109 tpkt->ackm_pkt.is_inflight = 1;
2110 tpkt->ackm_pkt.is_ack_eliciting = have_ack_eliciting;
2111 tpkt->ackm_pkt.is_pto_probe = 0;
2112 tpkt->ackm_pkt.is_mtu_probe = 0;
2113 tpkt->ackm_pkt.time = ossl_time_now();
2115 /* Packet Information for QTX */
2117 pkt.iovec = txp->iovec;
2118 pkt.num_iovec = h.num_iovec;
2120 pkt.peer = BIO_ADDR_family(&txp->args.peer) == AF_UNSPEC
2121 ? NULL : &txp->args.peer;
2122 pkt.pn = txp->next_pn[pn_space];
2123 pkt.flags = OSSL_QTX_PKT_FLAG_COALESCE; /* always try to coalesce */
2125 /* Do TX key update if needed. */
2126 if (enc_level == QUIC_ENC_LEVEL_1RTT) {
2127 uint64_t cur_pkt_count, max_pkt_count;
2129 cur_pkt_count = ossl_qtx_get_cur_epoch_pkt_count(txp->args.qtx, enc_level);
2130 max_pkt_count = ossl_qtx_get_max_epoch_pkt_count(txp->args.qtx, enc_level);
2132 if (cur_pkt_count >= max_pkt_count / 2)
2133 if (!ossl_qtx_trigger_key_update(txp->args.qtx))
2137 if (!ossl_assert(h.bytes_appended > 0))
2140 /* Generate TXPIM chunks representing STOP_SENDING and RESET_STREAM frames. */
2141 for (stream = tmp_head; stream != NULL; stream = stream->txp_next)
2142 if (stream->txp_sent_stop_sending || stream->txp_sent_reset_stream) {
2143 /* Log STOP_SENDING chunk to TXPIM. */
2144 QUIC_TXPIM_CHUNK chunk;
2146 chunk.stream_id = stream->id;
2147 chunk.start = UINT64_MAX;
2150 chunk.has_stop_sending = stream->txp_sent_stop_sending;
2151 chunk.has_reset_stream = stream->txp_sent_reset_stream;
2152 if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk))
2153 return 0; /* alloc error */
2156 /* Dispatch to FIFD. */
2157 if (!ossl_quic_fifd_pkt_commit(&txp->fifd, tpkt))
2160 /* Send the packet. */
2161 if (!ossl_qtx_write_pkt(txp->args.qtx, &pkt))
2164 ++txp->next_pn[pn_space];
2167 * Record FC and stream abort frames as sent; deactivate streams which no
2168 * longer have anything to do.
2170 for (stream = tmp_head; stream != NULL; stream = stream->txp_next) {
2171 if (stream->txp_sent_fc) {
2172 stream->want_max_stream_data = 0;
2173 ossl_quic_rxfc_has_cwm_changed(&stream->rxfc, 1);
2176 if (stream->txp_sent_stop_sending)
2177 stream->want_stop_sending = 0;
2179 if (stream->txp_sent_reset_stream)
2180 stream->want_reset_stream = 0;
2182 if (stream->txp_txfc_new_credit_consumed > 0) {
2183 if (!ossl_assert(ossl_quic_txfc_consume_credit(&stream->txfc,
2184 stream->txp_txfc_new_credit_consumed)))
2186 * Should not be possible, but we should continue with our
2187 * bookkeeping as we have already committed the packet to the
2188 * FIFD. Just change the value we return.
2190 rc = TXP_ERR_INTERNAL;
2192 stream->txp_txfc_new_credit_consumed = 0;
2196 * If we no longer need to generate any flow control (MAX_STREAM_DATA),
2197 * STOP_SENDING or RESET_STREAM frames, nor any STREAM frames (because
2198 * the stream is drained of data or TXFC-blocked), we can mark the
2199 * stream as inactive.
2201 ossl_quic_stream_map_update_state(txp->args.qsm, stream);
2203 if (stream->txp_drained)
2204 assert(!ossl_quic_sstream_has_pending(stream->sstream));
2207 /* We have now sent the packet, so update state accordingly. */
2208 if (have_ack_eliciting)
2209 txp->force_ack_eliciting &= ~(1UL << pn_space);
2211 if (tpkt->had_handshake_done_frame)
2212 txp->want_handshake_done = 0;
2214 if (tpkt->had_max_data_frame) {
2215 txp->want_max_data = 0;
2216 ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 1);
2219 if (tpkt->had_max_streams_bidi_frame) {
2220 txp->want_max_streams_bidi = 0;
2221 ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_bidi_rxfc, 1);
2224 if (tpkt->had_max_streams_uni_frame) {
2225 txp->want_max_streams_uni = 0;
2226 ossl_quic_rxfc_has_cwm_changed(txp->args.max_streams_uni_rxfc, 1);
2229 if (tpkt->had_ack_frame)
2230 txp->want_ack &= ~(1UL << pn_space);
2233 * Decrement probe request counts if we have sent a packet that meets
2234 * the requirement of a probe, namely being ACK-eliciting.
2236 if (have_ack_eliciting) {
2237 if (enc_level == QUIC_ENC_LEVEL_INITIAL
2238 && probe_info->anti_deadlock_initial > 0)
2239 --probe_info->anti_deadlock_initial;
2241 if (enc_level == QUIC_ENC_LEVEL_HANDSHAKE
2242 && probe_info->anti_deadlock_handshake > 0)
2243 --probe_info->anti_deadlock_handshake;
2245 if (a.allow_force_ack_eliciting /* (i.e., not for 0-RTT) */
2246 && probe_info->pto[pn_space] > 0)
2247 --probe_info->pto[pn_space];
2250 if (have_ack_eliciting)
2251 *sent_ack_eliciting = 1;
2254 tx_helper_cleanup(&h);
2259 * Handler for fatal errors, i.e. errors causing us to abort the entire
2260 * packet rather than just one frame. Examples of such errors include
2261 * allocation errors.
2264 tx_helper_cleanup(&h);
2266 ossl_quic_txpim_pkt_release(txp->args.txpim, tpkt);
2267 return TXP_ERR_INTERNAL;
2270 /* Ensure the iovec array is at least num elements long. */
2271 static int txp_ensure_iovec(OSSL_QUIC_TX_PACKETISER *txp, size_t num)
2273 OSSL_QTX_IOVEC *iovec;
2275 if (txp->alloc_iovec >= num)
2278 num = txp->alloc_iovec != 0 ? txp->alloc_iovec * 2 : 8;
2280 iovec = OPENSSL_realloc(txp->iovec, sizeof(OSSL_QTX_IOVEC) * num);
2285 txp->alloc_iovec = num;
2289 int ossl_quic_tx_packetiser_schedule_conn_close(OSSL_QUIC_TX_PACKETISER *txp,
2290 const OSSL_QUIC_FRAME_CONN_CLOSE *f)
2292 char *reason = NULL;
2293 size_t reason_len = f->reason_len;
2294 size_t max_reason_len = txp_get_mdpl(txp) / 2;
2296 if (txp->want_conn_close)
2300 * Arbitrarily limit the length of the reason length string to half of the
2303 if (reason_len > max_reason_len)
2304 reason_len = max_reason_len;
2306 if (reason_len > 0) {
2307 reason = OPENSSL_memdup(f->reason, reason_len);
2312 txp->conn_close_frame = *f;
2313 txp->conn_close_frame.reason = reason;
2314 txp->conn_close_frame.reason_len = reason_len;
2315 txp->want_conn_close = 1;