fix: bound streamable HTTP memory usage (#970)
* fix: bound streamable HTTP memory usage * fix: handle SSE comment lines safely * refactor: narrow SSE limit public API surface
This commit is contained in:
parent
1543a1afee
commit
24ba5265e2
1 changed files with 811 additions and 39 deletions
|
|
@ -5,13 +5,166 @@ use std::{
|
|||
time::Duration,
|
||||
};
|
||||
|
||||
use futures::{Stream, stream::BoxStream};
|
||||
use sse_stream::{Error as SseError, Sse};
|
||||
use bytes::Bytes;
|
||||
use futures::{Stream, StreamExt, stream::BoxStream};
|
||||
use sse_stream::{Error as SseError, Sse, SseStream};
|
||||
use thiserror::Error;
|
||||
|
||||
use crate::model::ServerJsonRpcMessage;
|
||||
|
||||
pub type BoxedSseResponse = BoxStream<'static, Result<Sse, SseError>>;
|
||||
|
||||
/// Maximum raw size of one SSE event accepted from a remote server.
|
||||
pub(crate) const DEFAULT_MAX_SSE_EVENT_SIZE: usize = 16 * 1024 * 1024;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
enum BoundedSseStreamError {
|
||||
#[error(transparent)]
|
||||
Source(Box<dyn std::error::Error + Send + Sync>),
|
||||
#[error("SSE event exceeded the maximum size of {max_size} bytes")]
|
||||
EventTooLarge { max_size: usize },
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct SseEventSizeLimiter {
|
||||
max_size: usize,
|
||||
retained_size: usize,
|
||||
line_size: usize,
|
||||
line_is_comment: bool,
|
||||
previous_was_cr: bool,
|
||||
}
|
||||
|
||||
impl SseEventSizeLimiter {
|
||||
fn new(max_size: usize) -> Self {
|
||||
Self {
|
||||
max_size,
|
||||
retained_size: 0,
|
||||
line_size: 0,
|
||||
line_is_comment: false,
|
||||
previous_was_cr: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn observe(&mut self, chunk: &[u8]) -> Result<(), ()> {
|
||||
for &byte in chunk {
|
||||
if self.previous_was_cr {
|
||||
self.previous_was_cr = false;
|
||||
if byte == b'\n' {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
match byte {
|
||||
b'\r' => {
|
||||
self.finish_line()?;
|
||||
self.previous_was_cr = true;
|
||||
}
|
||||
b'\n' => self.finish_line()?,
|
||||
_ => {
|
||||
if self.line_size == 0 {
|
||||
self.line_is_comment = byte == b':';
|
||||
}
|
||||
self.line_size = self.line_size.saturating_add(1);
|
||||
self.check_limit()?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish_line(&mut self) -> Result<(), ()> {
|
||||
if self.line_size == 0 {
|
||||
self.retained_size = 0;
|
||||
} else if !self.line_is_comment {
|
||||
// The SSE parser inserts a newline when joining multiple data fields.
|
||||
self.retained_size = self
|
||||
.retained_size
|
||||
.saturating_add(self.line_size)
|
||||
.saturating_add(1);
|
||||
}
|
||||
self.line_size = 0;
|
||||
self.line_is_comment = false;
|
||||
self.check_limit()
|
||||
}
|
||||
|
||||
fn check_limit(&self) -> Result<(), ()> {
|
||||
if self.retained_size.saturating_add(self.line_size) > self.max_size {
|
||||
Err(())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pin_project_lite::pin_project! {
|
||||
struct BoundedSseByteStream<S> {
|
||||
#[pin]
|
||||
inner: S,
|
||||
limiter: SseEventSizeLimiter,
|
||||
failed: bool,
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, E> Stream for BoundedSseByteStream<S>
|
||||
where
|
||||
S: Stream<Item = Result<Bytes, E>>,
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
type Item = Result<Bytes, BoundedSseStreamError>;
|
||||
|
||||
fn poll_next(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
) -> Poll<Option<Self::Item>> {
|
||||
let mut this = self.project();
|
||||
if *this.failed {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
|
||||
match ready!(this.inner.as_mut().poll_next(cx)) {
|
||||
Some(Ok(chunk)) => {
|
||||
if this.limiter.observe(&chunk).is_err() {
|
||||
*this.failed = true;
|
||||
Poll::Ready(Some(Err(BoundedSseStreamError::EventTooLarge {
|
||||
max_size: this.limiter.max_size,
|
||||
})))
|
||||
} else {
|
||||
Poll::Ready(Some(Ok(chunk)))
|
||||
}
|
||||
}
|
||||
Some(Err(error)) => {
|
||||
*this.failed = true;
|
||||
Poll::Ready(Some(Err(BoundedSseStreamError::Source(Box::new(error)))))
|
||||
}
|
||||
None => Poll::Ready(None),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn bounded_sse_stream<S, E>(stream: S, max_event_size: usize) -> BoxedSseResponse
|
||||
where
|
||||
S: Stream<Item = Result<Bytes, E>> + Send + 'static,
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
let stream = BoundedSseByteStream {
|
||||
inner: stream,
|
||||
limiter: SseEventSizeLimiter::new(max_event_size),
|
||||
failed: false,
|
||||
};
|
||||
SseStream::from_bytes_stream(stream).boxed()
|
||||
}
|
||||
|
||||
fn is_event_too_large_error(error: &SseError) -> bool {
|
||||
matches!(
|
||||
error,
|
||||
SseError::Body(error)
|
||||
if matches!(
|
||||
error.downcast_ref::<BoundedSseStreamError>(),
|
||||
Some(BoundedSseStreamError::EventTooLarge { .. })
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
pub trait SseRetryPolicy: std::fmt::Debug + Send + Sync {
|
||||
fn retry(&self, current_times: usize) -> Option<Duration>;
|
||||
}
|
||||
|
|
@ -124,6 +277,10 @@ pub(crate) trait SseStreamReconnect {
|
|||
tracing::warn!("sse stream error: {error}");
|
||||
}
|
||||
}
|
||||
fn map_fatal_stream_error(&mut self, error: SseError) -> Option<Self::Error> {
|
||||
tracing::warn!("fatal sse stream error: {error}");
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pin_project_lite::pin_project! {
|
||||
|
|
@ -248,6 +405,10 @@ where
|
|||
}
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
if is_event_too_large_error(&e) {
|
||||
this.state.set(SseAutoReconnectStreamState::Terminated);
|
||||
return Poll::Ready(this.connector.map_fatal_stream_error(e).map(Err));
|
||||
}
|
||||
this.connector
|
||||
.handle_stream_error(&e, this.last_event_id.as_deref());
|
||||
let retrying = this
|
||||
|
|
@ -327,3 +488,202 @@ where
|
|||
self.poll_next(cx)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
enum TestReconnectError {
|
||||
#[error("SSE stream error: {0}")]
|
||||
Sse(SseError),
|
||||
#[error("unexpected reconnect")]
|
||||
Reconnect,
|
||||
}
|
||||
|
||||
struct CountingReconnect {
|
||||
attempts: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl SseStreamReconnect for CountingReconnect {
|
||||
type Error = TestReconnectError;
|
||||
type Future = futures::future::Ready<Result<BoxedSseResponse, Self::Error>>;
|
||||
|
||||
fn retry_connection(&mut self, _last_event_id: Option<&str>) -> Self::Future {
|
||||
self.attempts.fetch_add(1, Ordering::Relaxed);
|
||||
futures::future::ready(Err(TestReconnectError::Reconnect))
|
||||
}
|
||||
|
||||
fn map_fatal_stream_error(&mut self, error: SseError) -> Option<Self::Error> {
|
||||
Some(TestReconnectError::Sse(error))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_sse_stream_rejects_unterminated_event_over_limit() {
|
||||
let source = futures::stream::iter([
|
||||
Ok::<_, std::io::Error>(Bytes::from_static(b"data: aaaaa")),
|
||||
Ok(Bytes::from_static(b"aaaaaa")),
|
||||
]);
|
||||
let mut stream = bounded_sse_stream(source, 16);
|
||||
|
||||
let error = stream.next().await.unwrap().unwrap_err();
|
||||
|
||||
assert!(
|
||||
error.to_string().contains("maximum size of 16 bytes"),
|
||||
"unexpected error: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_sse_stream_resets_limit_after_event_terminator() {
|
||||
let source = futures::stream::iter([
|
||||
Ok::<_, std::io::Error>(Bytes::from_static(b"data: a\r")),
|
||||
Ok(Bytes::from_static(b"\n\r\ndata: b\n\n")),
|
||||
]);
|
||||
let mut stream = bounded_sse_stream(source, 8);
|
||||
|
||||
let first = stream.next().await.unwrap().unwrap();
|
||||
let second = stream.next().await.unwrap().unwrap();
|
||||
|
||||
assert_eq!(
|
||||
(first.data.as_deref(), second.data.as_deref()),
|
||||
(Some("a"), Some("b"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_sse_stream_passes_event_at_exact_limit() {
|
||||
let source =
|
||||
futures::stream::iter([Ok::<_, std::io::Error>(Bytes::from_static(b"data: ab\n\n"))]);
|
||||
let mut stream = bounded_sse_stream(source, 9);
|
||||
|
||||
let event = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(event.data.as_deref(), Some("ab"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_sse_stream_rejects_oversize_split_across_many_chunks() {
|
||||
let source = futures::stream::iter(
|
||||
std::iter::repeat_with(|| Ok::<_, std::io::Error>(Bytes::from_static(b"data: x")))
|
||||
.take(20),
|
||||
);
|
||||
let mut stream = bounded_sse_stream(source, 32);
|
||||
|
||||
let mut found_error = false;
|
||||
while let Some(item) = stream.next().await {
|
||||
if item.is_err() {
|
||||
found_error = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(found_error, "expected oversize error");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_sse_stream_handles_crlf_split_across_chunks() {
|
||||
let source = futures::stream::iter([
|
||||
Ok::<_, std::io::Error>(Bytes::from_static(b"data: hello\r")),
|
||||
Ok(Bytes::from_static(b"\n\ndata: world\n\n")),
|
||||
]);
|
||||
let mut stream = bounded_sse_stream(source, 64);
|
||||
|
||||
let first = stream.next().await.unwrap().unwrap();
|
||||
let second = stream.next().await.unwrap().unwrap();
|
||||
|
||||
assert_eq!(first.data.as_deref(), Some("hello"));
|
||||
assert_eq!(second.data.as_deref(), Some("world"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_sse_stream_discards_completed_comment_lines_from_limit() {
|
||||
let source = futures::stream::iter([Ok::<_, std::io::Error>(Bytes::from_static(
|
||||
b": ping\n: ping\n: ping\n",
|
||||
))]);
|
||||
let mut stream = bounded_sse_stream(source, 6);
|
||||
|
||||
assert!(stream.next().await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_sse_stream_comments_do_not_reset_accumulated_data() {
|
||||
let source = futures::stream::iter([Ok::<_, std::io::Error>(Bytes::from_static(
|
||||
b"data: a\n: ping\ndata: b\n",
|
||||
))]);
|
||||
let mut stream = bounded_sse_stream(source, 14);
|
||||
|
||||
assert!(stream.next().await.unwrap().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_sse_stream_counts_multiline_data_join_newlines() {
|
||||
let source = futures::stream::iter([Ok::<_, std::io::Error>(Bytes::from_static(
|
||||
b"data: aaa\ndata: bbb\n\n",
|
||||
))]);
|
||||
let mut stream = bounded_sse_stream(source, 18);
|
||||
|
||||
let error = stream.next().await.unwrap().unwrap_err();
|
||||
assert!(
|
||||
error.to_string().contains("maximum size"),
|
||||
"unexpected error: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_sse_stream_propagates_source_error() {
|
||||
let source = futures::stream::iter([Err::<Bytes, _>(std::io::Error::new(
|
||||
std::io::ErrorKind::ConnectionReset,
|
||||
"connection reset",
|
||||
))]);
|
||||
let mut stream = bounded_sse_stream(source, 1024);
|
||||
|
||||
let error = stream.next().await.unwrap().unwrap_err();
|
||||
assert!(
|
||||
error.to_string().contains("connection reset"),
|
||||
"unexpected error: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn is_event_too_large_error_detects_oversize() {
|
||||
let source = futures::stream::iter([Ok::<_, std::io::Error>(Bytes::from(vec![b'A'; 100]))]);
|
||||
let mut stream = bounded_sse_stream(source, 8);
|
||||
|
||||
let error = stream.next().await.unwrap().unwrap_err();
|
||||
assert!(is_event_too_large_error(&error));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn is_event_too_large_error_rejects_other_errors() {
|
||||
let source =
|
||||
futures::stream::iter([Err::<Bytes, _>(std::io::Error::other("something else"))]);
|
||||
let mut stream = bounded_sse_stream(source, 1024);
|
||||
|
||||
let error = stream.next().await.unwrap().unwrap_err();
|
||||
assert!(!is_event_too_large_error(&error));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oversized_event_returns_error_without_reconnecting() {
|
||||
let source = futures::stream::iter([Ok::<_, std::io::Error>(Bytes::from(vec![b'A'; 100]))]);
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
let connector = CountingReconnect {
|
||||
attempts: attempts.clone(),
|
||||
};
|
||||
let stream = SseAutoReconnectStream::new(
|
||||
bounded_sse_stream(source, 8),
|
||||
connector,
|
||||
Arc::new(NeverRetry),
|
||||
);
|
||||
let mut stream = std::pin::pin!(stream);
|
||||
|
||||
let result = stream.next().await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Some(Err(TestReconnectError::Sse(_))))
|
||||
&& attempts.load(Ordering::Relaxed) == 0
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue