diff --git a/Cargo.lock b/Cargo.lock index b06a8c5262..3c0ae9feca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8945,6 +8945,25 @@ dependencies = [ "rand_core", ] +[[package]] +name = "range-requests" +version = "0.1.0" +dependencies = [ + "bytes", + "dropshot", + "futures", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "http-range", + "hyper 1.4.1", + "omicron-workspace-hack", + "proptest", + "thiserror", + "tokio", + "tokio-util", +] + [[package]] name = "ratatui" version = "0.28.1" diff --git a/Cargo.toml b/Cargo.toml index ab82cbef54..e8fa7f42fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,6 +97,7 @@ members = [ "oximeter/types", "package", "passwords", + "range-requests", "rpaths", "sled-agent", "sled-agent/api", @@ -224,6 +225,7 @@ default-members = [ "oximeter/types", "package", "passwords", + "range-requests", "rpaths", "sled-agent", "sled-agent/api", @@ -393,7 +395,9 @@ hickory-server = "0.24.1" highway = "1.2.0" hkdf = "0.12.4" http = "1.1.0" +http-body = "1.0.1" http-body-util = "0.1.2" +http-range = "0.1.5" httpmock = "0.8.0-alpha.1" httptest = "0.16.1" hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "main" } @@ -530,6 +534,7 @@ rand = "0.8.5" rand_core = "0.6.4" rand_distr = "0.4.3" rand_seeder = "0.3.0" +range-requests = { path = "range-requests" } ratatui = "0.28.1" rayon = "1.10" rcgen = "0.12.1" diff --git a/range-requests/Cargo.toml b/range-requests/Cargo.toml new file mode 100644 index 0000000000..ab9972180d --- /dev/null +++ b/range-requests/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "range-requests" +description = "Helpers for making and receiving range requests" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[lints] +workspace = true + +[dependencies] +bytes.workspace = true +dropshot.workspace = true +futures.workspace = true +http.workspace = true +http-range.workspace = true +http-body-util.workspace = true +hyper.workspace = true +thiserror.workspace = true +omicron-workspace-hack.workspace = true + +[dev-dependencies] +http-body.workspace = true +proptest.workspace = true +tokio.workspace = true +tokio-util.workspace = true diff --git a/range-requests/src/lib.rs b/range-requests/src/lib.rs new file mode 100644 index 0000000000..ccd250d949 --- /dev/null +++ b/range-requests/src/lib.rs @@ -0,0 +1,532 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use dropshot::Body; +use futures::TryStreamExt; +use http::HeaderValue; +use hyper::{ + header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE}, + Response, StatusCode, +}; + +const ACCEPT_RANGES_BYTES: http::HeaderValue = + http::HeaderValue::from_static("bytes"); +const CONTENT_TYPE_OCTET_STREAM: http::HeaderValue = + http::HeaderValue::from_static("application/octet-stream"); + +/// Errors which may be returned when processing range requests +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Using multiple ranges is not supported")] + MultipleRangesUnsupported, + + #[error("Range would overflow (start + length is too large)")] + RangeOverflow, + + #[error("Range would underflow (total content length < start)")] + RangeUnderflow, + + #[error("Empty Range")] + EmptyRange, + + #[error("Failed to parse range: {0:?}")] + Parse(http_range::HttpRangeParseError), + + #[error(transparent)] + Http(#[from] http::Error), +} + +// TODO(https://github.com/oxidecomputer/dropshot/issues/39): Return a dropshot +// type here (HttpError?) to e.g. include the RequestID in the response. +// +// Same for the other functions returning "Response" below - we're doing +// this so the "RANGE_NOT_SATISFIABLE" response can attach extra info, but it's +// currently happening at the expense of headers that Dropshot wants to supply. + +fn bad_request_response() -> Response { + hyper::Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(Body::empty()) + .expect("'bad request response' creation should be infallible") +} + +fn internal_error_response() -> Response { + hyper::Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::empty()) + .expect("'internal error response' creation should be infallible") +} + +fn not_satisfiable_response(file_size: u64) -> Response { + hyper::Response::builder() + .status(StatusCode::RANGE_NOT_SATISFIABLE) + .header(ACCEPT_RANGES, ACCEPT_RANGES_BYTES) + .header(CONTENT_RANGE, format!("bytes */{file_size}")) + .body(Body::empty()) + .expect("'not satisfiable response' creation should be infallible") +} + +/// Generate a GET response, optionally for a HTTP range request. The total +/// file length should be provided, whether or not the expected Content-Length +/// for a range request is shorter. +/// +/// It is the responsibility of the caller to ensure that `rx` is a stream of +/// data matching the requested range in the `range` argument, if it is +/// supplied. +pub fn make_get_response( + range: Option, + file_length: u64, + content_type: Option>, + rx: S, +) -> Result, Error> +where + E: Send + Sync + std::error::Error + 'static, + D: Into, + S: Send + Sync + futures::stream::Stream> + 'static, +{ + Ok(make_response_common(range, file_length, content_type).body( + Body::wrap(http_body_util::StreamBody::new( + rx.map_ok(|b| hyper::body::Frame::data(b.into())), + )), + )?) +} + +/// Generate a HEAD response, optionally for a HTTP range request. The total +/// file length should be provided, whether or not the expected Content-Length +/// for a range request is shorter. +pub fn make_head_response( + range: Option, + file_length: u64, + content_type: Option>, +) -> Result, Error> { + Ok(make_response_common(range, file_length, content_type) + .body(Body::empty())?) +} + +fn make_response_common( + range: Option, + file_length: u64, + content_type: Option>, +) -> hyper::http::response::Builder { + let mut res = Response::builder(); + res = res.header(ACCEPT_RANGES, ACCEPT_RANGES_BYTES); + res = res.header( + CONTENT_TYPE, + content_type.map(|t| t.into()).unwrap_or(CONTENT_TYPE_OCTET_STREAM), + ); + + if let Some(range) = range { + res = res.header(CONTENT_LENGTH, range.content_length().to_string()); + res = res.header(CONTENT_RANGE, range.to_content_range()); + res = res.status(StatusCode::PARTIAL_CONTENT); + } else { + res = res.header(CONTENT_LENGTH, file_length.to_string()); + res = res.status(StatusCode::OK); + } + + res +} + +/// Represents the raw, unparsed values of "range" from a request header. +pub struct PotentialRange(Vec); + +impl PotentialRange { + /// Parses a single range request out of the range request. + /// + /// `len` is the total length of the document, for the range request being made. + /// + /// On failure, returns a range response with the appropriate headers + /// to inform the caller how to make a correct range request. + pub fn parse(&self, len: u64) -> Result> { + self.single_range(len).map_err(|err| match err { + Error::MultipleRangesUnsupported | Error::Parse(_) => { + bad_request_response() + } + Error::RangeOverflow + | Error::RangeUnderflow + | Error::EmptyRange => not_satisfiable_response(len), + Error::Http(_err) => internal_error_response(), + }) + } + + fn single_range(&self, len: u64) -> Result { + match http_range::HttpRange::parse_bytes(&self.0, len) { + Ok(ranges) => { + if ranges.len() != 1 || ranges[0].length < 1 { + // Right now, we don't want to deal with encoding a + // response that has multiple ranges. + Err(Error::MultipleRangesUnsupported) + } else { + Ok(SingleRange::new(ranges[0], len)?) + } + } + Err(err) => Err(Error::Parse(err)), + } + } +} + +/// A parsed range request, and associated "total document length". +#[derive(Clone, Debug)] +pub struct SingleRange { + range: http_range::HttpRange, + total: u64, +} + +#[cfg(test)] +impl PartialEq for SingleRange { + fn eq(&self, other: &Self) -> bool { + self.range.start == other.range.start + && self.range.length == other.range.length + && self.total == other.total + } +} + +impl SingleRange { + fn new(range: http_range::HttpRange, total: u64) -> Result { + let http_range::HttpRange { start, mut length } = range; + + // Clip the length to avoid going beyond the end of the total range + if start.checked_add(length).ok_or(Error::RangeOverflow)? >= total { + length = total.checked_sub(start).ok_or(Error::RangeUnderflow)?; + } + // If the length is zero, we cannot satisfy the range request + if length == 0 { + return Err(Error::EmptyRange); + } + + Ok(Self { range: http_range::HttpRange { start, length }, total }) + } + + /// Return the first byte in this range for use in inclusive ranges. + pub fn start(&self) -> u64 { + self.range.start + } + + /// Return the last byte in this range for use in inclusive ranges. + pub fn end_inclusive(&self) -> u64 { + assert!(self.range.length > 0); + + self.range + .start + .checked_add(self.range.length) + .expect("start + length overflowed, but should have been checked in 'SingleRange::new'") + .checked_sub(1) + .expect("start + length underflowed, but should have been checked in 'SingleRange::new'") + } + + /// Generate the Content-Range header for inclusion in a HTTP 206 partial + /// content response using this range. + pub fn to_content_range(&self) -> HeaderValue { + HeaderValue::from_str(&format!( + "bytes {}-{}/{}", + self.range.start, + self.end_inclusive(), + self.total + )) + .expect("Content-Range value should have been ASCII string") + } + + /// Generate a Range header for inclusion in another HTTP request; e.g., + /// to a backend object store. + pub fn to_range(&self) -> HeaderValue { + HeaderValue::from_str(&format!( + "bytes={}-{}", + self.range.start, + self.end_inclusive() + )) + .expect("Range bounds should have been ASCII string") + } + + /// Returns the content length for this range + pub fn content_length(&self) -> std::num::NonZeroU64 { + self.range.length.try_into().expect( + "Length should be more than zero, validated in SingleRange::new", + ) + } +} + +/// A trait, implemented for [dropshot::RequestContext], to pull a range header +/// out of the request headers. +pub trait RequestContextEx { + fn range(&self) -> Option; +} + +impl RequestContextEx for dropshot::RequestContext +where + T: Send + Sync + 'static, +{ + /// If there is a Range header, return it for processing during response + /// generation. + fn range(&self) -> Option { + self.request + .headers() + .get(hyper::header::RANGE) + .map(|hv| PotentialRange(hv.as_bytes().to_vec())) + } +} + +#[cfg(test)] +mod test { + use super::*; + + use bytes::Bytes; + use futures::stream::once; + use http_body_util::BodyExt; + use proptest::prelude::*; + use std::convert::Infallible; + use tokio_util::io::ReaderStream; + + proptest! { + #[test] + fn potential_range_parsing_does_not_crash( + bytes: Vec, + len in 0_u64..=u64::MAX, + ) { + let result = PotentialRange(bytes).parse(len); + let Ok(range) = result else { return Ok(()); }; + let _ = range.start(); + let _ = range.end_inclusive(); + let _ = range.to_content_range(); + let _ = range.to_range(); + } + + #[test] + fn single_range_parsing_does_not_crash( + start in 0_u64..=u64::MAX, + length in 0_u64..=u64::MAX, + total in 0_u64..=u64::MAX + ) { + let result = SingleRange::new(http_range::HttpRange { + start, length + }, total); + + let Ok(range) = result else { return Ok(()); }; + + assert_eq!(range.start(), start); + let _ = range.end_inclusive(); + let _ = range.to_content_range(); + let _ = range.to_range(); + } + } + + #[test] + fn invalid_ranges() { + assert!(matches!( + SingleRange::new( + http_range::HttpRange { start: u64::MAX, length: 1 }, + 1 + ), + Err(Error::RangeOverflow) + )); + + assert!(matches!( + SingleRange::new( + http_range::HttpRange { start: 100, length: 0 }, + 10 + ), + Err(Error::RangeUnderflow) + )); + + assert!(matches!( + SingleRange::new(http_range::HttpRange { start: 0, length: 0 }, 1), + Err(Error::EmptyRange) + )); + } + + #[test] + fn parse_range_valid() { + // Whole range + let pr = PotentialRange(b"bytes=0-100".to_vec()); + assert_eq!( + pr.single_range(100).unwrap(), + SingleRange { + range: http_range::HttpRange { start: 0, length: 100 }, + total: 100 + } + ); + + // Clipped + let pr = PotentialRange(b"bytes=0-100".to_vec()); + assert_eq!( + pr.single_range(50).unwrap(), + SingleRange { + range: http_range::HttpRange { start: 0, length: 50 }, + total: 50 + } + ); + + // Single byte + let pr = PotentialRange(b"bytes=49-49".to_vec()); + assert_eq!( + pr.single_range(50).unwrap(), + SingleRange { + range: http_range::HttpRange { start: 49, length: 1 }, + total: 50 + } + ); + } + + #[test] + fn parse_range_invalid() { + let pr = PotentialRange(b"bytes=50-50".to_vec()); + assert!(matches!( + pr.single_range(50).expect_err("Range should be invalid"), + Error::Parse(http_range::HttpRangeParseError::NoOverlap), + )); + + let pr = PotentialRange(b"bytes=20-1".to_vec()); + assert!(matches!( + pr.single_range(50).expect_err("Range should be invalid"), + Error::Parse(http_range::HttpRangeParseError::InvalidRange), + )); + } + + #[test] + fn get_response_no_range() { + let bytes = b"Hello world"; + + let response = make_get_response( + None, + bytes.len() as u64, + None::, + ReaderStream::new(bytes.as_slice()), + ) + .expect("Should have made response"); + + assert_eq!(response.status(), StatusCode::OK); + + expect_headers( + response.headers(), + &[ + (ACCEPT_RANGES, "bytes"), + (CONTENT_TYPE, "application/octet-stream"), + (CONTENT_LENGTH, &bytes.len().to_string()), + ], + ); + } + + // Makes a get response with a Vec of bytes that counts from zero. + // + // The u8s aren't normal bounds on the length, but they make the mapping + // of "the data is the index" easy. + fn ranged_get_request( + start: u8, + length: u8, + total_length: u8, + ) -> Response { + let range = SingleRange::new( + http_range::HttpRange { + start: start.into(), + length: length.into(), + }, + total_length.into(), + ) + .unwrap(); + + let b: Vec<_> = (u8::try_from(range.start()).unwrap() + ..=u8::try_from(range.end_inclusive()).unwrap()) + .collect(); + + let response = make_get_response( + Some(range.clone()), + total_length.into(), + None::, + once(async move { Ok::<_, Infallible>(b) }), + ) + .expect("Should have made response"); + + response + } + + // Validates the headers exactly match the map + fn expect_headers( + headers: &http::HeaderMap, + expected: &[(http::HeaderName, &str)], + ) { + println!("Headers: {headers:#?}"); + assert_eq!(headers.len(), expected.len()); + for (k, v) in expected { + assert_eq!(headers.get(k).unwrap(), v); + } + } + + // Validates the data matches an incrementing Vec of u8 values + async fn expect_data( + body: &mut (dyn http_body::Body< + Data = Bytes, + Error = Box, + > + Unpin), + start: u8, + length: u8, + ) { + println!("Checking data from {start}, with length {length}"); + let frame = body + .frame() + .await + .expect("Error reading frame") + .expect("Should have one frame") + .into_data() + .expect("Should be a DATA frame"); + assert_eq!(frame.len(), usize::from(length),); + + for i in 0..length { + assert_eq!(frame[i as usize], i + start); + } + } + + #[tokio::test] + async fn get_response_with_range() { + // First half + let mut response = ranged_get_request(0, 32, 64); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + expect_data(response.body_mut(), 0, 32).await; + expect_headers( + response.headers(), + &[ + (ACCEPT_RANGES, "bytes"), + (CONTENT_TYPE, "application/octet-stream"), + (CONTENT_LENGTH, "32"), + (CONTENT_RANGE, "bytes 0-31/64"), + ], + ); + + // Second half + let mut response = ranged_get_request(32, 32, 64); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + expect_data(response.body_mut(), 32, 32).await; + expect_headers( + response.headers(), + &[ + (ACCEPT_RANGES, "bytes"), + (CONTENT_TYPE, "application/octet-stream"), + (CONTENT_LENGTH, "32"), + (CONTENT_RANGE, "bytes 32-63/64"), + ], + ); + + // Partially out of bounds + let mut response = ranged_get_request(60, 32, 64); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + expect_data(response.body_mut(), 60, 4).await; + expect_headers( + response.headers(), + &[ + (ACCEPT_RANGES, "bytes"), + (CONTENT_TYPE, "application/octet-stream"), + (CONTENT_LENGTH, "4"), + (CONTENT_RANGE, "bytes 60-63/64"), + ], + ); + + // Fully out of bounds + assert!(matches!( + SingleRange::new( + http_range::HttpRange { start: 64, length: 32 }, + 64 + ) + .expect_err("Should have thrown an error"), + Error::EmptyRange, + )); + } +}