Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Don't use the file system to pass export results #6087

Merged
merged 2 commits into from
Feb 10, 2025
+139 −129
Merged

Don't use the file system to pass export results #6087

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b017817
better
robertbastian Feb 7, 2025
f3cc168
gen
robertbastian Feb 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 31 additions & 36 deletions provider/baked/src/export.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,27 +286,6 @@ impl BakedExporter {
.map_err(|e| DataError::from(e).with_path_context(&path))
}

fn print_deps(&mut self) {
let mut deps = core::mem::take(&mut self.dependencies)
.into_iter()
.collect::<BTreeSet<_>>();
if !self.use_separate_crates {
deps.retain(|&krate| {
!krate.starts_with("icu_")
|| krate == "icu_provider"
|| krate == "icu_locale_core"
|| krate == "icu_pattern"
});
deps.insert("icu");
}
deps.insert("icu_provider");

log::info!("The generated module requires the following crates:");
for crate_name in deps {
log::info!("{}", crate_name);
}
}

fn write_impl_macros(
&self,
marker: DataMarkerInfo,
Expand Down Expand Up @@ -791,26 +770,42 @@ impl DataExporter for BakedExporter {
},
)?;

// TODO: Return the statistics instead of writing them out.
let mut file = crlify::BufWriterWithLineEndingFix::new(std::fs::File::create(
self.mod_directory.join("fingerprints.csv"),
)?);
for (marker, (_, stats)) in data {
if !marker.is_singleton {
writeln!(
&mut file,
"{marker:?}, <lookup>, {}B, {} identifiers",
stats.lookup_struct_size, stats.identifiers_count
)?;
}
}
let statistics = data
.into_iter()
.map(|(marker, (_, stats))| (marker, stats))
.collect();

self.print_deps();
let mut required_crates = core::mem::take(&mut self.dependencies)
.into_iter()
.collect::<BTreeSet<_>>();
if !self.use_separate_crates {
required_crates.retain(|&krate| {
!krate.starts_with("icu_")
|| krate == "icu_provider"
|| krate == "icu_locale_core"
|| krate == "icu_pattern"
});
required_crates.insert("icu");
}
required_crates.insert("icu_provider");

Ok(Default::default())
Ok(ExporterCloseMetadata(Some(Box::new(
BakedExporterCloseMetadata {
statistics,
required_crates,
},
))))
}
}

/// TODO
pub struct BakedExporterCloseMetadata {
/// TODO
pub statistics: BTreeMap<DataMarkerInfo, Statistics>,
/// TODO
pub required_crates: BTreeSet<&'static str>,
}

macro_rules! cb {
($($marker_ty:ty:$marker:ident,)+ #[experimental] $($emarker_ty:ty:$emarker:ident,)+) => {
fn bake_marker(marker: DataMarkerInfo) -> databake::TokenStream {
Expand Down
1 change: 0 additions & 1 deletion provider/baked/tests/data/fingerprints.csv
View file
Open in desktop

This file was deleted.

18 changes: 10 additions & 8 deletions provider/core/src/export/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,14 @@ pub trait DataExporter: Sync {
/// markers have been fully dumped). This conceptually takes ownership, so
/// clients *may not* interact with this object after close has been called.
fn close(&mut self) -> Result<ExporterCloseMetadata, DataError> {
Ok(ExporterCloseMetadata)
Ok(ExporterCloseMetadata::default())
}
}

#[non_exhaustive]
#[derive(Debug, Clone, Default)]
#[derive(Debug, Default)]
#[allow(clippy::exhaustive_structs)] // newtype
/// Contains information about a successful export.
pub struct ExporterCloseMetadata;
pub struct ExporterCloseMetadata(pub Option<Box<dyn core::any::Any>>);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thought: dyn Display or Debug? Could be paired with Any.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think there's much of a point just printing everything that's in there


/// Metadata for [`DataExporter::flush`]
#[non_exhaustive]
Expand Down Expand Up @@ -201,9 +201,11 @@ impl DataExporter for MultiExporter {
}

fn close(&mut self) -> Result<ExporterCloseMetadata, DataError> {
self.0.iter_mut().try_fold(ExporterCloseMetadata, |m, e| {
e.close()?;
Ok(m)
})
Ok(ExporterCloseMetadata(Some(Box::new(
self.0.iter_mut().try_fold(vec![], |mut m, e| {
m.push(e.close()?.0);
Ok::<_, DataError>(m)
})?,
))))
}
}
2 changes: 1 addition & 1 deletion provider/export/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ impl ExportDriver {
}

#[non_exhaustive]
#[derive(Debug, Clone)]
#[derive(Debug)]
/// Contains information about a successful export.
pub struct ExportMetadata {
/// The metadata coming from the [`DataExporter`].
Expand Down
180 changes: 97 additions & 83 deletions tools/make/bakeddata/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ use icu_provider::prelude::*;
use icu_provider_export::baked_exporter;
use icu_provider_export::prelude::*;
use icu_provider_source::{CoverageLevel, SourceDataProvider};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::collections::BTreeMap;
use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io::Write;
use std::path::Path;
Expand Down Expand Up @@ -162,14 +163,9 @@ fn main() {
let stub_exporter = StubExporter(
baked_exporter::BakedExporter::new(path.join("stubdata"), options).unwrap(),
);
let fingerprinter = StatisticsExporter {
data: Default::default(),
fingerprints: crlify::BufWriterWithLineEndingFix::new(
File::create(path.join("fingerprints.csv")).unwrap(),
),
};
let fingerprinter = StatisticsExporter::default();

driver
let export_metdata = driver
.clone()
.with_markers(markers.iter().copied())
.export(
Expand All @@ -182,18 +178,92 @@ fn main() {
)
.unwrap();

// Stitch
let struct_fingerprints = std::fs::read_to_string(path.join("fingerprints.csv")).unwrap();
let lookup_fingerprints =
std::fs::read_to_string(path.join("data").join("fingerprints.csv")).unwrap();
std::fs::remove_file(path.join("stubdata").join("fingerprints.csv")).unwrap();
std::fs::remove_file(path.join("fingerprints.csv")).unwrap();
std::fs::remove_file(path.join("data").join("fingerprints.csv")).unwrap();
let mut export_metadatas = export_metdata
.exporter
.0
.unwrap()
.downcast::<Vec<Option<Box<dyn core::any::Any>>>>()
.unwrap();

let baked_metadata = export_metadatas[0]
.take()
.unwrap()
.downcast::<icu_provider_export::baked_exporter::BakedExporterCloseMetadata>()
.unwrap();

let fingerprint_metadata = export_metadatas[2]
.take()
.unwrap()
.downcast::<HashMap<DataMarkerInfo, Statistics>>()
.unwrap();

let mut lines = Vec::new();

for (marker, data) in fingerprint_metadata.into_iter() {
if marker.is_singleton {
let ((baked_struct_size, postcard_struct_size), hash) =
data.size_hash[&Default::default()];
lines.push(format!(
"{marker:?}, <singleton>, {baked_struct_size}B, {postcard_struct_size}B, {hash:x}"
));
} else {
let postcard_structs_size = data
.struct_sizes
.values()
.map(|(_, postcard)| postcard)
.sum::<usize>();
let baked_structs_size = data
.struct_sizes
.values()
.map(|(baked, _)| baked)
.sum::<usize>();

let baked_exporter::Statistics {
structs_count,
lookup_struct_size,
identifiers_count,
..
} = &baked_metadata.statistics[&marker];

lines.push(format!(
"{marker:?}, <total>, {baked_structs_size}B, {postcard_structs_size}B, {structs_count} unique payloads",
));
lines.push(format!(
"{marker:?}, <lookup>, {lookup_struct_size}B, {identifiers_count} identifiers",
));

let mut seen = HashMap::new();
for (id, ((baked_size, postcard_size), hash)) in data
.size_hash
.into_iter()
.map(|(id, v)| {
(
if !id.marker_attributes.is_empty() {
format!(
"{locale}/{marker_attributes}",
locale = id.locale,
marker_attributes = id.marker_attributes.as_str(),
)
} else {
id.locale.to_string()
},
v,
)
})
.collect::<BTreeMap<_, _>>()
{
if let Some(deduped_req) = seen.get(&hash) {
lines.push(format!("{marker:?}, {id}, -> {deduped_req}",));
} else {
lines.push(format!(
"{marker:?}, {id}, {baked_size}B, {postcard_size}B, {hash:x}",
));
seen.insert(hash, id.clone());
}
}
}
}

let mut lines = [&struct_fingerprints, &lookup_fingerprints]
.into_iter()
.flat_map(|f| f.lines())
.collect::<Vec<_>>();
lines.sort();
let mut out = crlify::BufWriterWithLineEndingFix::new(
File::create(path.join("fingerprints.csv")).unwrap(),
Expand Down Expand Up @@ -234,19 +304,19 @@ impl<E: DataExporter> DataExporter for StubExporter<E> {
}
}

struct StatisticsExporter<F> {
data: Mutex<HashMap<DataMarkerInfo, Data>>,
fingerprints: F,
#[derive(Default)]
struct StatisticsExporter {
data: Mutex<HashMap<DataMarkerInfo, Statistics>>,
}

#[derive(Default)]
struct Data {
struct Statistics {
size_hash: HashMap<DataIdentifierCow<'static>, ((usize, usize), u64)>,
struct_sizes: HashMap<u64, (usize, usize)>,
identifiers: HashSet<DataIdentifierCow<'static>>,
}

impl<F: Write + Send + Sync> DataExporter for StatisticsExporter<F> {
impl DataExporter for StatisticsExporter {
fn put_payload(
&self,
marker: DataMarkerInfo,
Expand Down Expand Up @@ -274,65 +344,9 @@ impl<F: Write + Send + Sync> DataExporter for StatisticsExporter<F> {
Ok(())
}

fn flush(&self, _marker: DataMarkerInfo, _metadata: FlushMetadata) -> Result<(), DataError> {
Ok(())
}

fn close(&mut self) -> Result<ExporterCloseMetadata, DataError> {
let data = core::mem::take(self.data.get_mut().expect("poison"));

let mut lines = Vec::new();

for (marker, data) in data.into_iter() {
if !marker.is_singleton {
let structs_count = data.struct_sizes.len();
let baked_structs_size = data.struct_sizes.values().map(|(x, _)| x).sum::<usize>();
let postcard_structs_size =
data.struct_sizes.values().map(|(_, x)| x).sum::<usize>();
lines.push(format!(
"{marker:?}, <total>, {baked_structs_size}B, {postcard_structs_size}B, {structs_count} unique payloads",
));
}

let sorted = data
.size_hash
.into_iter()
.map(|(id, (size, hash))| {
(
if marker.is_singleton && id.locale.is_default() {
"<singleton>".to_string()
} else if !id.marker_attributes.is_empty() {
format!(
"{locale}/{marker_attributes}",
locale = id.locale,
marker_attributes = id.marker_attributes.as_str(),
)
} else {
id.locale.to_string()
},
(size, hash),
)
})
.collect::<BTreeMap<_, _>>();

let mut seen = HashMap::new();
for (id, ((baked_size, postcard_size), hash)) in sorted.into_iter() {
if let Some(deduped_req) = seen.get(&hash) {
lines.push(format!("{marker:?}, {id}, -> {deduped_req}",));
} else {
lines.push(format!(
"{marker:?}, {id}, {baked_size}B, {postcard_size}B, {hash:x}",
));
seen.insert(hash, id.clone());
}
}
}

lines.sort();
for line in lines {
writeln!(&mut self.fingerprints, "{line}")?;
}

Ok(Default::default())
Ok(ExporterCloseMetadata(Some(Box::new(core::mem::take(
self.data.get_mut().expect("poison"),
)))))
}
}
Loading