Replace reqwest with hyper (#8099)

* Very primitive test of the Client API

* [WIP] getting rid of request

* Add support for redirects.

* Remove CpuPool from `fetch::Client`.

* Adapt code to API changes and fix tests.

* Use reference counter to stop background thread.

On `clone` the counter is incremented, on `drop` decremented. Once 0 we
send `None` over the channel, expecting the background thread to end.

* Fix tests.

* Comment.

* Change expect messages.

* Use local test server for testing fetch client.

* Ensure max_size also in BodyReader.

* Replace `Condvar` with `sync_channel`.

* Re-export `url::Url` from `fetch` crate.

* Remove spaces.

* Use random ports in local test server.

dapps/src/apps/fetcher/installers.rs

@@ -19,9 +19,10 @@ use std::{fs, fmt};
 use std::io::{self, Read, Write};
 use std::path::PathBuf;
 use ethereum_types::H256;
-use fetch::{self, Mime};
+use fetch;
 use futures_cpupool::CpuPool;
 use hash::keccak_buffer;
+use mime_guess::Mime;
 
 use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest, serialize_manifest, Manifest};
 use handlers::{ContentValidator, ValidatorResponse};
@@ -53,7 +54,7 @@ fn write_response_and_check_hash(
 
 	// Now write the response
 	let mut file = io::BufWriter::new(fs::File::create(&content_path)?);
-	let mut reader = io::BufReader::new(response);
+	let mut reader = io::BufReader::new(fetch::BodyReader::new(response));
 	io::copy(&mut reader, &mut file)?;
 	file.flush()?;
 

dapps/src/apps/fetcher/mod.rs

@@ -216,6 +216,7 @@ impl<R: URLHint + 'static, F: Fetch> Endpoint for ContentFetcher<F, R> {
 										),
 										self.embeddable_on.clone(),
 										self.fetch.clone(),
+										self.pool.clone(),
 									)
 								},
 							    URLHintResult::GithubDapp(content) => {
@@ -232,6 +233,7 @@ impl<R: URLHint + 'static, F: Fetch> Endpoint for ContentFetcher<F, R> {
 										),
 										self.embeddable_on.clone(),
 										self.fetch.clone(),
+										self.pool.clone(),
 									)
 							    },
 								URLHintResult::Content(content) => {
@@ -248,6 +250,7 @@ impl<R: URLHint + 'static, F: Fetch> Endpoint for ContentFetcher<F, R> {
 										),
 										self.embeddable_on.clone(),
 										self.fetch.clone(),
+										self.pool.clone(),
 									)
 								},
 							};
@@ -280,7 +283,7 @@ impl<R: URLHint + 'static, F: Fetch> Endpoint for ContentFetcher<F, R> {
 mod tests {
 	use std::env;
 	use std::sync::Arc;
-	use fetch::{Fetch, Client};
+	use fetch::Client;
 	use futures::{future, Future};
 	use hash_fetch::urlhint::{URLHint, URLHintResult};
 	use ethereum_types::H256;

dapps/src/apps/mod.rs

@@ -81,7 +81,7 @@ pub fn all_endpoints<F: Fetch>(
 	insert::<parity_ui::old::App>(&mut pages, "v1", Embeddable::Yes(embeddable.clone()), pool.clone());
 
 	pages.insert("proxy".into(), ProxyPac::boxed(embeddable.clone(), dapps_domain.to_owned()));
-	pages.insert(WEB_PATH.into(), Web::boxed(embeddable.clone(), web_proxy_tokens.clone(), fetch.clone()));
+	pages.insert(WEB_PATH.into(), Web::boxed(embeddable.clone(), web_proxy_tokens.clone(), fetch.clone(), pool.clone()));
 
 	(local_endpoints, pages)
 }

dapps/src/handlers/fetch.rs

@@ -23,6 +23,7 @@ use std::time::{Instant, Duration};
 use fetch::{self, Fetch};
 use futures::sync::oneshot;
 use futures::{self, Future};
+use futures_cpupool::CpuPool;
 use hyper::{self, Method, StatusCode};
 use parking_lot::Mutex;
 
@@ -35,7 +36,7 @@ const FETCH_TIMEOUT: u64 = 300;
 
 pub enum ValidatorResponse {
 	Local(local::Dapp),
-	Streaming(StreamingHandler<fetch::Response>),
+	Streaming(StreamingHandler<fetch::BodyReader>),
 }
 
 pub trait ContentValidator: Sized + Send + 'static {
@@ -252,6 +253,7 @@ impl ContentFetcherHandler {
 		installer: H,
 		embeddable_on: Embeddable,
 		fetch: F,
+		pool: CpuPool,
 	) -> Self {
 		let fetch_control = FetchControl::default();
 		let errors = Errors { embeddable_on };
@@ -262,6 +264,7 @@ impl ContentFetcherHandler {
 			Method::Get => {
 				trace!(target: "dapps", "Fetching content from: {:?}", url);
 				FetchState::InProgress(Self::fetch_content(
+						pool,
 						fetch,
 						url,
 						fetch_control.abort.clone(),
@@ -282,6 +285,7 @@ impl ContentFetcherHandler {
 	}
 
 	fn fetch_content<H: ContentValidator, F: Fetch>(
+		pool: CpuPool,
 		fetch: F,
 		url: &str,
 		abort: Arc<AtomicBool>,
@@ -290,8 +294,8 @@ impl ContentFetcherHandler {
 		installer: H,
 	) -> Box<Future<Item=FetchState, Error=()> + Send> {
 		// Start fetching the content
-		let fetch2 = fetch.clone();
-		let future = fetch.fetch_with_abort(url, abort.into()).then(move |result| {
+		let pool2 = pool.clone();
+		let future = fetch.fetch(url, abort.into()).then(move |result| {
 			trace!(target: "dapps", "Fetching content finished. Starting validation: {:?}", result);
 			Ok(match result {
 				Ok(response) => match installer.validate_and_install(response) {
@@ -303,7 +307,7 @@ impl ContentFetcherHandler {
 					Ok(ValidatorResponse::Streaming(stream)) => {
 						trace!(target: "dapps", "Validation OK. Streaming response.");
 						let (reading, response) = stream.into_response();
-						fetch2.process_and_forget(reading);
+						pool.spawn(reading).forget();
 						FetchState::Streaming(response)
 					},
 					Err(e) => {
@@ -319,7 +323,7 @@ impl ContentFetcherHandler {
 		});
 
 		// make sure to run within fetch thread pool.
-		fetch.process(future)
+		Box::new(pool2.spawn(future))
 	}
 }
 

dapps/src/tests/helpers/fetch.rs

@@ -14,12 +14,13 @@
 // You should have received a copy of the GNU General Public License
 // along with Parity.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::{io, thread, time};
+use std::{thread, time};
 use std::sync::{atomic, mpsc, Arc};
 use parking_lot::Mutex;
+use hyper;
 
 use futures::{self, Future};
-use fetch::{self, Fetch};
+use fetch::{self, Fetch, Url};
 
 pub struct FetchControl {
 	sender: mpsc::Sender<()>,
@@ -96,11 +97,8 @@ impl FakeFetch {
 impl Fetch for FakeFetch {
 	type Result = Box<Future<Item = fetch::Response, Error = fetch::Error> + Send>;
 
-	fn new() -> Result<Self, fetch::Error> where Self: Sized {
-		Ok(FakeFetch::default())
-	}
-
-	fn fetch_with_abort(&self, url: &str, _abort: fetch::Abort) -> Self::Result {
+	fn fetch(&self, url: &str, abort: fetch::Abort) -> Self::Result {
+		let u = Url::parse(url).unwrap();
 		self.requested.lock().push(url.into());
 		let manual = self.manual.clone();
 		let response = self.response.clone();
@@ -111,23 +109,10 @@ impl Fetch for FakeFetch {
 				// wait for manual resume
 				let _ = rx.recv();
 			}
-
 			let data = response.lock().take().unwrap_or(b"Some content");
-			let cursor = io::Cursor::new(data);
-			tx.send(fetch::Response::from_reader(cursor)).unwrap();
+			tx.send(fetch::Response::new(u, hyper::Response::new().with_body(data), abort)).unwrap();
 		});
 
 		Box::new(rx.map_err(|_| fetch::Error::Aborted))
 	}
-
-	fn process_and_forget<F, I, E>(&self, f: F) where
-		F: Future<Item=I, Error=E> + Send + 'static,
-		I: Send + 'static,
-		E: Send + 'static,
-	{
-		// Spawn the task in a separate thread.
-		thread::spawn(|| {
-			let _ = f.wait();
-		});
-	}
 }

dapps/src/tests/helpers/mod.rs

@@ -63,7 +63,7 @@ pub fn init_server<F, B>(process: F, io: IoHandler) -> (Server, Arc<FakeRegistra
 	let mut dapps_path = env::temp_dir();
 	dapps_path.push("non-existent-dir-to-prevent-fs-files-from-loading");
 
-	let mut builder = ServerBuilder::new(&dapps_path, registrar.clone());
+	let mut builder = ServerBuilder::new(FetchClient::new().unwrap(), &dapps_path, registrar.clone());
 	builder.signer_address = Some(("127.0.0.1".into(), SIGNER_PORT));
 	let server = process(builder).start_unsecured_http(&"127.0.0.1:0".parse().unwrap(), io).unwrap();
 	(
@@ -149,21 +149,21 @@ pub struct ServerBuilder<T: Fetch = FetchClient> {
 	web_proxy_tokens: Arc<WebProxyTokens>,
 	signer_address: Option<(String, u16)>,
 	allowed_hosts: DomainsValidation<Host>,
-	fetch: Option<T>,
+	fetch: T,
 	serve_ui: bool,
 }
 
 impl ServerBuilder {
 	/// Construct new dapps server
-	pub fn new<P: AsRef<Path>>(dapps_path: P, registrar: Arc<RegistrarClient<Call=Asynchronous>>) -> Self {
+	pub fn new<P: AsRef<Path>>(fetch: FetchClient, dapps_path: P, registrar: Arc<RegistrarClient<Call=Asynchronous>>) -> Self {
 		ServerBuilder {
 			dapps_path: dapps_path.as_ref().to_owned(),
 			registrar: registrar,
 			sync_status: Arc::new(FakeSync(false)),
 			web_proxy_tokens: Arc::new(|_| None),
 			signer_address: None,
 			allowed_hosts: DomainsValidation::Disabled,
-			fetch: None,
+			fetch: fetch,
 			serve_ui: false,
 		}
 	}
@@ -179,15 +179,14 @@ impl<T: Fetch> ServerBuilder<T> {
 			web_proxy_tokens: self.web_proxy_tokens,
 			signer_address: self.signer_address,
 			allowed_hosts: self.allowed_hosts,
-			fetch: Some(fetch),
+			fetch: fetch,
 			serve_ui: self.serve_ui,
 		}
 	}
 
 	/// Asynchronously start server with no authentication,
 	/// returns result with `Server` handle on success or an error.
 	pub fn start_unsecured_http(self, addr: &SocketAddr, io: IoHandler) -> io::Result<Server> {
-		let fetch = self.fetch_client();
 		Server::start_http(
 			addr,
 			io,
@@ -199,17 +198,10 @@ impl<T: Fetch> ServerBuilder<T> {
 			self.sync_status,
 			self.web_proxy_tokens,
 			Remote::new_sync(),
-			fetch,
+			self.fetch,
 			self.serve_ui,
 		)
 	}
-
-	fn fetch_client(&self) -> T {
-		match self.fetch.clone() {
-			Some(fetch) => fetch,
-			None => T::new().unwrap(),
-		}
-	}
 }
 
 const DAPPS_DOMAIN: &'static str = "web3.site";

dapps/src/web.rs

@@ -25,6 +25,7 @@ use hyper::{mime, StatusCode};
 use apps;
 use endpoint::{Endpoint, EndpointPath, Request, Response};
 use futures::future;
+use futures_cpupool::CpuPool;
 use handlers::{
 	ContentFetcherHandler, ContentHandler, ContentValidator, ValidatorResponse,
 	StreamingHandler,
@@ -35,18 +36,21 @@ pub struct Web<F> {
 	embeddable_on: Embeddable,
 	web_proxy_tokens: Arc<WebProxyTokens>,
 	fetch: F,
+	pool: CpuPool,
 }
 
 impl<F: Fetch> Web<F> {
 	pub fn boxed(
 		embeddable_on: Embeddable,
 		web_proxy_tokens: Arc<WebProxyTokens>,
 		fetch: F,
+		pool: CpuPool,
 	) -> Box<Endpoint> {
 		Box::new(Web {
 			embeddable_on,
 			web_proxy_tokens,
 			fetch,
+			pool,
 		})
 	}
 
@@ -129,6 +133,7 @@ impl<F: Fetch> Endpoint for Web<F> {
 			},
 			self.embeddable_on.clone(),
 			self.fetch.clone(),
+			self.pool.clone(),
 		))
 	}
 }
@@ -146,7 +151,7 @@ impl ContentValidator for WebInstaller {
 		let is_html = response.is_html();
 		let mime = response.content_type().unwrap_or(mime::TEXT_HTML);
 		let mut handler = StreamingHandler::new(
-			response,
+			fetch::BodyReader::new(response),
 			status,
 			mime,
 			self.embeddable_on,

ethcore/Cargo.toml

@@ -33,6 +33,7 @@ ethjson = { path = "../json" }
 ethkey = { path = "../ethkey" }
 ethstore = { path = "../ethstore" }
 evm = { path = "evm" }
+futures-cpupool = "0.1"
 hardware-wallet = { path = "../hw" }
 heapsize = "0.4"
 itertools = "0.5"

ethcore/src/lib.rs

@@ -71,6 +71,7 @@ extern crate ethcore_transaction as transaction;
 extern crate ethereum_types;
 extern crate ethjson;
 extern crate ethkey;
+extern crate futures_cpupool;
 extern crate hardware_wallet;
 extern crate hashdb;
 extern crate itertools;

ethcore/src/miner/miner.rs

@@ -35,6 +35,7 @@ use ethcore_miner::transaction_queue::{
 	AccountDetails,
 	TransactionOrigin,
 };
+use futures_cpupool::CpuPool;
 use ethcore_miner::work_notify::{WorkPoster, NotifyWork};
 use miner::service_transaction_checker::ServiceTransactionChecker;
 use miner::{MinerService, MinerStatus};
@@ -218,11 +219,11 @@ pub enum GasPricer {
 
 impl GasPricer {
 	/// Create a new Calibrated `GasPricer`.
-	pub fn new_calibrated(options: GasPriceCalibratorOptions, fetch: FetchClient) -> GasPricer {
+	pub fn new_calibrated(options: GasPriceCalibratorOptions, fetch: FetchClient, p: CpuPool) -> GasPricer {
 		GasPricer::Calibrated(GasPriceCalibrator {
 			options: options,
 			next_calibration: Instant::now(),
-			price_info: PriceInfoClient::new(fetch),
+			price_info: PriceInfoClient::new(fetch, p),
 		})
 	}
 

hash-fetch/Cargo.toml

@@ -8,6 +8,7 @@ authors = ["Parity Technologies <[email protected]>"]
 
 [dependencies]
 futures = "0.1"
+futures-cpupool = "0.1"
 log = "0.3"
 mime = "0.3"
 mime_guess = "2.0.0-alpha.2"
@@ -25,4 +26,5 @@ ethabi-derive = "5.0"
 ethabi-contract = "5.0"
 
 [dev-dependencies]
+hyper = "0.11"
 parking_lot = "0.5"