fix doc build and PathBuf on windows

README.md

@@ -13,7 +13,7 @@
 </p>
 
 <p align="center">
-    <a href="https://docs.rs/amadeus/0.2.0">📖 Docs</a> | <a href="https://constellation.rs/amadeus">🌐 Home</a> | <a href="https://constellation.zulipchat.com/#narrow/stream/213231-amadeus">💬 Chat</a>
+    <a href="https://docs.rs/amadeus/0.2.0/amadeus/">📖 Docs</a> | <a href="https://constellation.rs/amadeus">🌐 Home</a> | <a href="https://constellation.zulipchat.com/#narrow/stream/213231-amadeus">💬 Chat</a>
 </p>
 
 ## Amadeus provides:

amadeus-aws/src/cloudfront.rs

@@ -10,7 +10,7 @@ use std::{
 };
 
 use amadeus_core::{
-	into_par_stream::IntoDistributedStream, par_stream::{DistributedStream, ParallelStream}, util::{DistParStream, ResultExpandIter}, Source
+	into_par_stream::IntoDistributedStream, par_stream::DistributedStream, util::{DistParStream, ResultExpandIter}, Source
 };
 use amadeus_types::{DateTime, IpAddr, Url};
 
@@ -55,9 +55,11 @@ impl Source for Cloudfront {
 	type Error = AwsError;
 
 	#[cfg(not(feature = "doc"))]
-	type ParStream = impl ParallelStream<Item = Result<Self::Item, Self::Error>>;
+	type ParStream =
+		impl amadeus_core::par_stream::ParallelStream<Item = Result<Self::Item, Self::Error>>;
 	#[cfg(feature = "doc")]
-	type ParStream = amadeus_core::util::ImplParallelStream<Result<Self::Item, Self::Error>>;
+	type ParStream =
+		DistParStream<amadeus_core::util::ImplDistributedStream<Result<Self::Item, Self::Error>>>;
 	#[cfg(not(feature = "doc"))]
 	type DistStream = impl DistributedStream<Item = Result<Self::Item, Self::Error>>;
 	#[cfg(feature = "doc")]

amadeus-aws/src/file.rs

@@ -73,11 +73,10 @@ impl Directory for S3Directory {
 				let file_name = path.pop().unwrap();
 				skip = skip
 					&& path.len() >= current_path.depth()
-					&& path
+					&& current_path.iter().eq(path
 						.iter()
 						.take(current_path.depth())
-						.copied()
-						.eq(current_path.iter());
+						.copied());
 				if skip {
 					return false;
 				}

amadeus-commoncrawl/src/lib.rs

@@ -11,7 +11,7 @@ use serde_closure::*;
 use std::{io, time};
 
 use amadeus_core::{
-	into_par_stream::IntoDistributedStream, par_stream::{DistributedStream, ParallelStream}, util::DistParStream, Source
+	into_par_stream::IntoDistributedStream, par_stream::DistributedStream, util::DistParStream, Source
 };
 use amadeus_types::Webpage;
 
@@ -59,9 +59,11 @@ impl Source for CommonCrawl {
 	type Error = io::Error;
 
 	#[cfg(not(feature = "doc"))]
-	type ParStream = impl ParallelStream<Item = Result<Self::Item, Self::Error>>;
+	type ParStream =
+		impl amadeus_core::par_stream::ParallelStream<Item = Result<Self::Item, Self::Error>>;
 	#[cfg(feature = "doc")]
-	type ParStream = amadeus_core::util::ImplParallelStream<Result<Self::Item, Self::Error>>;
+	type ParStream =
+		DistParStream<amadeus_core::util::ImplDistributedStream<Result<Self::Item, Self::Error>>>;
 	#[cfg(not(feature = "doc"))]
 	type DistStream = impl DistributedStream<Item = Result<Self::Item, Self::Error>>;
 	#[cfg(feature = "doc")]

amadeus-core/Cargo.toml

@@ -36,3 +36,4 @@ serde_closure = "0.2"
 streaming_algorithms = "0.2"
 sum = { version = "0.1", features = ["serde"] }
 walkdir = "2.2"
+widestring = "0.4"

amadeus-core/src/file.rs

@@ -1,3 +1,5 @@
+// TODO: Use WTF-8 rather than UTF-16
+
 #![allow(clippy::type_complexity)]
 
 mod local;
@@ -6,109 +8,179 @@ use async_trait::async_trait;
 use futures::{future::BoxFuture, ready};
 use pin_project::pin_project;
 use std::{
-	convert::TryFrom, error::Error, fmt, future::Future, io, pin::Pin, sync::Arc, task::{Context, Poll}
+	convert::TryFrom, error::Error, ffi, fmt, future::Future, io, pin::Pin, sync::Arc, task::{Context, Poll}
 };
+use widestring::U16String;
 
 use crate::pool::ProcessSend;
 
 pub use local::LocalFile;
 
 const PAGE_SIZE: usize = 10 * 1024 * 1024; // `Reader` reads this many bytes at a time
 
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct PathBuf {
-	wide: bool, // for if the Vec<u8> is actually utf16
-	components: Vec<Vec<u8>>,
-	file_name: Option<Vec<u8>>,
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
+pub struct OsString {
+	buf: U16String,
 }
-impl PathBuf {
+impl OsString {
 	pub fn new() -> Self {
 		Self {
-			wide: false,
-			components: Vec::new(),
-			file_name: None,
+			buf: U16String::new(),
+		}
+	}
+	pub fn to_string_lossy(&self) -> String {
+		self.buf.to_string_lossy()
+	}
+	pub fn display<'a>(&'a self) -> impl fmt::Display + 'a {
+		struct Display<'a>(&'a OsString);
+		impl<'a> fmt::Display for Display<'a> {
+			fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+				self.0.to_string_lossy().fmt(f)
+			}
 		}
+		Display(self)
 	}
-	pub fn new_wide() -> Self {
+}
+impl From<Vec<u8>> for OsString {
+	fn from(from: Vec<u8>) -> Self {
+		Self {
+			buf: String::from_utf8(from)
+				.expect("Not yet imlemented: Handling non-UTF-8")
+				.into(),
+		} // TODO
+	}
+}
+impl From<String> for OsString {
+	fn from(from: String) -> Self {
+		Self { buf: from.into() }
+	}
+}
+impl From<&str> for OsString {
+	fn from(from: &str) -> Self {
+		Self {
+			buf: U16String::from_str(from),
+		}
+	}
+}
+impl From<ffi::OsString> for OsString {
+	fn from(from: ffi::OsString) -> Self {
+		Self {
+			buf: U16String::from_os_str(&from),
+		}
+	}
+}
+impl From<&ffi::OsStr> for OsString {
+	fn from(from: &ffi::OsStr) -> Self {
+		Self {
+			buf: U16String::from_os_str(from),
+		}
+	}
+}
+pub struct InvalidOsString;
+impl TryFrom<OsString> for ffi::OsString {
+	type Error = InvalidOsString;
+
+	fn try_from(from: OsString) -> Result<Self, Self::Error> {
+		Ok(from.buf.to_os_string()) // TODO: this is lossy but it should error
+	}
+}
+impl PartialEq<Vec<u8>> for OsString {
+	fn eq(&self, other: &Vec<u8>) -> bool {
+		self == &OsString::from(other.clone())
+	}
+}
+impl PartialEq<String> for OsString {
+	fn eq(&self, other: &String) -> bool {
+		self == &OsString::from(other.clone())
+	}
+}
+impl PartialEq<str> for OsString {
+	fn eq(&self, other: &str) -> bool {
+		self == &OsString::from(other)
+	}
+}
+impl PartialEq<ffi::OsString> for OsString {
+	fn eq(&self, other: &ffi::OsString) -> bool {
+		self == &OsString::from(other.clone())
+	}
+}
+impl PartialEq<ffi::OsStr> for OsString {
+	fn eq(&self, other: &ffi::OsStr) -> bool {
+		self == &OsString::from(other)
+	}
+}
+impl fmt::Debug for OsString {
+	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+		write!(f, "{}", self.display())
+	}
+}
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
+pub struct PathBuf {
+	components: Vec<OsString>,
+	file_name: Option<OsString>,
+}
+impl PathBuf {
+	pub fn new() -> Self {
 		Self {
-			wide: true,
 			components: Vec::new(),
 			file_name: None,
 		}
 	}
 	pub fn push<S>(&mut self, component: S)
 	where
-		S: Into<Vec<u8>>,
+		S: Into<OsString>,
 	{
 		assert!(self.file_name.is_none());
 		self.components.push(component.into());
 	}
-	pub fn pop(&mut self) -> Option<Vec<u8>> {
+	pub fn pop(&mut self) -> Option<OsString> {
 		assert!(self.file_name.is_none());
 		self.components.pop()
 	}
-	pub fn last(&self) -> Option<String> {
+	pub fn last(&self) -> Option<&OsString> {
 		assert!(self.file_name.is_none());
-		self.components
-			.last()
-			.map(|bytes| String::from_utf8_lossy(bytes).into_owned())
+		self.components.last()
 	}
 	pub fn set_file_name<S>(&mut self, file_name: Option<S>)
 	where
-		S: Into<Vec<u8>>,
+		S: Into<OsString>,
 	{
 		self.file_name = file_name.map(Into::into);
 	}
 	pub fn is_file(&self) -> bool {
 		self.file_name.is_some()
 	}
-	pub fn file_name(&self) -> Option<String> {
-		self.file_name
-			.as_ref()
-			.map(|file_name| String::from_utf8_lossy(file_name).into_owned())
+	pub fn file_name(&self) -> Option<&OsString> {
+		self.file_name.as_ref()
 	}
 	pub fn depth(&self) -> usize {
 		self.components.len()
 	}
-	pub fn iter<'a>(&'a self) -> impl Iterator<Item = String> + 'a {
-		self.components
-			.iter()
-			.map(|bytes| String::from_utf8_lossy(bytes).into_owned())
-	}
-}
-impl Default for PathBuf {
-	fn default() -> Self {
-		Self::new()
+	pub fn iter<'a>(&'a self) -> impl Iterator<Item = &OsString> + 'a {
+		self.components.iter()
 	}
-}
-impl fmt::Display for PathBuf {
-	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		let mut res: fmt::Result = self
-			.iter()
-			.map(|component| write!(f, "{}/", component))
-			.collect();
-		if let Some(file_name) = self.file_name() {
-			res = res.and_then(|()| write!(f, "{}", file_name));
+	pub fn display<'a>(&'a self) -> impl fmt::Display + 'a {
+		struct Display<'a>(&'a PathBuf);
+		impl<'a> fmt::Display for Display<'a> {
+			fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+				let mut res: fmt::Result = self
+					.0
+					.iter()
+					.map(|component| write!(f, "{}/", component.to_string_lossy()))
+					.collect();
+				if let Some(file_name) = self.0.file_name() {
+					res = res.and_then(|()| write!(f, "{}", file_name.to_string_lossy()));
+				}
+				res
+			}
 		}
-		res
+		Display(self)
 	}
 }
 impl fmt::Debug for PathBuf {
 	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-		fmt::Debug::fmt(&self.to_string(), f)
-	}
-}
-impl PartialEq<str> for PathBuf {
-	fn eq(&self, other: &str) -> bool {
-		let mut vec: Vec<u8> = Vec::new();
-		for component in self.components.iter() {
-			vec.extend(component);
-			vec.push(b'/');
-		}
-		if let Some(file_name) = &self.file_name {
-			vec.extend(file_name);
-		}
-		&*vec == other.as_bytes()
+		write!(f, "{}", self.display())
 	}
 }
 

amadeus-core/src/file/local.rs

@@ -99,34 +99,16 @@ impl Directory for &Path {
 					return true;
 				}
 				let mut path = path.strip_prefix(self).unwrap();
-				let mut path_buf = if !cfg!(windows) {
-					super::PathBuf::new()
-				} else {
-					super::PathBuf::new_wide()
-				};
+				let mut path_buf = super::PathBuf::new();
 				let mut file_name = None;
-				#[cfg(unix)]
-				let into = |osstr: &OsStr| -> Vec<u8> {
-					std::os::unix::ffi::OsStrExt::as_bytes(osstr).to_owned()
-				};
-				#[cfg(windows)]
-				let into = |osstr: &OsStr| -> Vec<u8> {
-					std::os::windows::ffi::OsStrExt::encode_wide(osstr)
-						.flat_map(|char| {
-							let char = char.to_be();
-							std::iter::once(u8::try_from(char >> 8).unwrap())
-								.chain(std::iter::once(u8::try_from(char & 0xff).unwrap()))
-						})
-						.collect()
-				};
 				if !is_dir {
 					file_name = Some(path.file_name().unwrap());
 					path = path.parent().unwrap();
 				}
 				for component in path {
-					path_buf.push(into(component));
+					path_buf.push(component);
 				}
-				path_buf.set_file_name(file_name.map(into));
+				path_buf.set_file_name(file_name);
 				f(&path_buf)
 			})
 			.filter_map(|e| match e {