From 99becd8c9750b651654a9f0bb00761782fdcc52d Mon Sep 17 00:00:00 2001 From: Dustin Carlino Date: Sun, 1 Sep 2024 13:39:02 +0100 Subject: [PATCH] Fix serialization, changing it to wrap MapModel (which has the additional census/POI fields) instead. #17 Not yet updating the CLI... --- backend/src/amenity.rs | 5 ++- backend/src/lib.rs | 72 +++++++++++++++------------------- backend/src/main.rs | 51 ++++++++++++------------ backend/src/zone.rs | 13 ++---- graph/src/lib.rs | 4 +- web/src/RouteMode.svelte | 2 +- web/src/UploadRouteMode.svelte | 2 +- web/src/title/MapLoader.svelte | 12 +++--- web/src/worker.ts | 12 +----- 9 files changed, 79 insertions(+), 94 deletions(-) diff --git a/backend/src/amenity.rs b/backend/src/amenity.rs index 8a0a3c9..b775287 100644 --- a/backend/src/amenity.rs +++ b/backend/src/amenity.rs @@ -6,8 +6,10 @@ use geo::{Coord, Point}; use geojson::{Feature, GeoJson, Geometry}; use graph::{Graph, Mode, Timer}; use osm_reader::OsmID; +use serde::{Deserialize, Serialize}; use utils::{Mercator, Tags}; +#[derive(Serialize, Deserialize)] pub struct Amenities { pub amenities: Vec, // Indexed by RoadID. These're broken down this way because the 3 graphs look different and @@ -15,6 +17,7 @@ pub struct Amenities { pub per_road: Vec>>, } +#[derive(Serialize, Deserialize)] pub struct Amenity { pub id: AmenityID, pub osm_id: OsmID, @@ -27,7 +30,7 @@ pub struct Amenity { pub cuisine: Option, } -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord, Serialize, Deserialize)] pub struct AmenityID(pub usize); impl Amenities { diff --git a/backend/src/lib.rs b/backend/src/lib.rs index 16ec1eb..1692342 100644 --- a/backend/src/lib.rs +++ b/backend/src/lib.rs @@ -9,7 +9,7 @@ use chrono::NaiveTime; use geo::{Coord, LineString}; use geojson::{de::deserialize_geometry, Feature, GeoJson, Geometry}; use graph::{Graph, Mode, Timer}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use wasm_bindgen::prelude::*; use crate::amenity::Amenities; @@ -25,6 +25,7 @@ static START: Once = Once::new(); // TODO Rename #[wasm_bindgen] +#[derive(Serialize, Deserialize)] pub struct MapModel { graph: Graph, zones: Zones, @@ -33,12 +34,9 @@ pub struct MapModel { #[wasm_bindgen] impl MapModel { - /// If is_osm is true, expect bytes of an osm.pbf or osm.xml string. Otherwise, expect a - /// bincoded graph #[wasm_bindgen(constructor)] pub async fn new( input_bytes: &[u8], - is_osm: bool, gtfs_url: Option, population_url: Option, progress_cb: Option, @@ -50,40 +48,17 @@ impl MapModel { }); let mut timer = Timer::new("build graph", progress_cb); - let mut amenities = Amenities::new(); - let modify_roads = |_roads: &mut Vec| {}; - let graph = if is_osm { - let mut graph = Graph::new(input_bytes, &mut amenities, modify_roads, &mut timer) - .map_err(err_to_js)?; - - graph - .setup_gtfs( - match gtfs_url { - Some(url) => graph::GtfsSource::Geomedea(url), - None => graph::GtfsSource::None, - }, - &mut timer, - ) - .await - .map_err(err_to_js)?; - - graph - } else { - bincode::deserialize_from(input_bytes).map_err(err_to_js)? - }; - amenities.finalize(&graph, &mut timer); - // TODO Serialize this too - let zones = Zones::load(population_url, &graph.mercator, &mut timer) + let model = MapModel::create(input_bytes, gtfs_url, population_url, &mut timer) .await .map_err(err_to_js)?; - timer.done(); - Ok(MapModel { - graph, - zones, - amenities, - }) + Ok(model) + } + + #[wasm_bindgen(js_name = loadFile)] + pub fn load_file(input_bytes: &[u8]) -> Result { + bincode::deserialize_from(input_bytes).map_err(err_to_js) } /// Returns a GeoJSON string. Just shows the full network @@ -242,16 +217,33 @@ impl MapModel { } } -// Non WASM methods -// TODO Reconsider these. Benchmark should use Graph. MapModel should just be a thin WASM layer. +// Non WASM methods, also used by the CLI impl MapModel { - pub fn from_graph_bytes(input_bytes: &[u8]) -> Result { - let graph = bincode::deserialize_from(input_bytes).map_err(err_to_js)?; + pub async fn create( + input_bytes: &[u8], + gtfs_url: Option, + population_url: Option, + timer: &mut Timer, + ) -> anyhow::Result { let mut amenities = Amenities::new(); - amenities.finalize(&graph, &mut Timer::new("deserialize graph", None)); + let modify_roads = |_roads: &mut Vec| {}; + let mut graph = Graph::new(input_bytes, &mut amenities, modify_roads, timer)?; + + graph + .setup_gtfs( + match gtfs_url { + Some(url) => graph::GtfsSource::Geomedea(url), + None => graph::GtfsSource::None, + }, + timer, + ) + .await?; + amenities.finalize(&graph, timer); + let zones = Zones::load(population_url, &graph.mercator, timer).await?; + Ok(MapModel { graph, - zones: Zones::empty(), + zones, amenities, }) } diff --git a/backend/src/main.rs b/backend/src/main.rs index 8ae87d3..bd231f1 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -3,48 +3,51 @@ use std::io::BufWriter; use anyhow::Result; -use backend::{Graph, GtfsSource, Timer}; +use backend::MapModel; +use graph::{GtfsModel, Timer}; // TODO Don't need tokio multithreading, but fighting config to get single working -/// This is a CLI tool to build a Graph file with GTFS data, for later use in the web app. This -/// gives a perf benefit (faster to load a pre-built graph) and allows GTFS use (no practical way -/// to read a huge GTFS file or clip it from web). The downside is having to manually manage these -/// prebuilt files as the format changes -- which is why, unlike in A/B Street, this'll just be for -/// manual testing for now. +/// This is a CLI tool to build a MapModel file, for later use in the web app or CLI. This gives a +/// perf benefit (faster to load a pre-built graph), but manually managing these prebuilt files as +/// the format changes is tedious. That's why, unlike in A/B Street, this'll just be for manual +/// testing for now. #[tokio::main] async fn main() -> Result<()> { simple_logger::init_with_level(log::Level::Info).unwrap(); let args: Vec = std::env::args().collect(); - if args.len() < 3 || (args[1] != "graph" && args[1] != "gmd") { + if args.len() != 3 || (args[1] != "graph" && args[1] != "gmd") { println!("Usage: one of these:"); - println!("To make a graph.bin: graph osm.pbf [gtfs_directory]"); + println!("To make a graph.bin: graph osm.pbf"); println!("To make a gtfs.gmd: gmd gtfs_directory"); std::process::exit(1); } - // TODO Hardcoded for now - let population_url = Some("https://assets.od2net.org/population.fgb".to_string()); - - if args[1] == "graph" { - let timer = Timer::new("build graph", None); - let osm_bytes = std::fs::read(&args[2])?; - let gtfs = match args.get(3) { - Some(dir) => GtfsSource::Dir(dir.to_string()), - None => GtfsSource::None, - }; - - let graph = Graph::new(&osm_bytes, gtfs, population_url, timer).await?; - let writer = BufWriter::new(File::create("graph.bin")?); - bincode::serialize_into(writer, &graph)?; - } else if args[1] == "gmd" { + if args[1] == "gmd" { let mut timer = Timer::new("build geomedea from gtfs", None); timer.step("parse GTFS"); - let model = backend::GtfsModel::parse(&args[2], None)?; + let model = GtfsModel::parse(&args[2], None)?; timer.step("turn into geomedea"); model.to_geomedea("gtfs.gmd")?; timer.done(); + return Ok(()); } + let mut timer = Timer::new("build model", None); + let osm_bytes = std::fs::read(&args[2])?; + let model = MapModel::create( + &osm_bytes, + // TODO Hardcoded, or could we read from local files at least? + Some("https://assets.od2net.org/gtfs.gmd".to_string()), + Some("https://assets.od2net.org/population.fgb".to_string()), + &mut timer, + ) + .await?; + + timer.step("Writing"); + let writer = BufWriter::new(File::create("model.bin")?); + bincode::serialize_into(writer, &model)?; + + timer.done(); Ok(()) } diff --git a/backend/src/zone.rs b/backend/src/zone.rs index a0d380c..55887fe 100644 --- a/backend/src/zone.rs +++ b/backend/src/zone.rs @@ -3,18 +3,21 @@ use flatgeobuf::{FeatureProperties, FgbFeature, GeozeroGeometry, HttpFgbReader}; use geo::{Area, MultiPolygon, Polygon}; use geojson::{Feature, FeatureCollection, Geometry}; use rstar::{primitives::GeomWithData, RTree}; +use serde::{Deserialize, Serialize}; use utils::Mercator; use crate::Timer; +#[derive(Serialize, Deserialize)] pub struct Zones { pub zones: Vec, pub rtree: RTree>, } -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord, Serialize, Deserialize)] pub struct ZoneID(pub usize); +#[derive(Serialize, Deserialize)] pub struct Zone { // TODO Maybe split these upfront, including area and population, and just store in the RTree? // TODO Do these need to be Mercator? @@ -25,14 +28,6 @@ pub struct Zone { } impl Zones { - // TODO Remove when benchmarks don't use WASM layer - pub fn empty() -> Self { - Self { - zones: Vec::new(), - rtree: RTree::new(), - } - } - pub async fn load( population_url: Option, mercator: &Mercator, diff --git a/graph/src/lib.rs b/graph/src/lib.rs index b3eba45..1a5e599 100644 --- a/graph/src/lib.rs +++ b/graph/src/lib.rs @@ -23,8 +23,8 @@ use utils::{Mercator, Tags}; pub use self::route::Route; use self::route::Router; pub use self::timer::Timer; -use crate::gtfs::TripID; -use crate::gtfs::{GtfsModel, StopID}; +pub use crate::gtfs::GtfsModel; +use crate::gtfs::{StopID, TripID}; #[derive(Serialize, Deserialize)] pub struct Graph { diff --git a/web/src/RouteMode.svelte b/web/src/RouteMode.svelte index 3c839b2..ca75919 100644 --- a/web/src/RouteMode.svelte +++ b/web/src/RouteMode.svelte @@ -139,7 +139,7 @@ {#if $showRouteBuffer}