From 62c62f7ec6474e902427b3a2f31237d3ee847c97 Mon Sep 17 00:00:00 2001 From: Gaius Date: Tue, 26 Nov 2024 18:07:50 +0800 Subject: [PATCH] feat: add sse4.2 target feature for building (#867) Signed-off-by: Gaius --- ci/Dockerfile | 2 +- ci/Dockerfile.dfinit | 2 +- dragonfly-client-backend/src/hdfs.rs | 20 ++++----- .../src/object_storage.rs | 44 +++++++++---------- dragonfly-client-config/build.rs | 21 ++++++++- dragonfly-client-core/src/error/mod.rs | 2 +- dragonfly-client/src/bin/dfget/main.rs | 8 ++-- dragonfly-client/src/proxy/mod.rs | 2 +- .../src/resource/persistent_cache_task.rs | 6 +-- dragonfly-client/src/resource/piece.rs | 13 +++--- dragonfly-client/src/resource/task.rs | 13 +++--- rust-toolchain.toml | 2 +- 12 files changed, 73 insertions(+), 62 deletions(-) diff --git a/ci/Dockerfile b/ci/Dockerfile index 05e5a474bcc..f6b3e222ded 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.80.0 AS builder +FROM rust:1.82.0 AS builder WORKDIR /app/client diff --git a/ci/Dockerfile.dfinit b/ci/Dockerfile.dfinit index 3f4bb79e7ab..64943bacbf3 100644 --- a/ci/Dockerfile.dfinit +++ b/ci/Dockerfile.dfinit @@ -1,4 +1,4 @@ -FROM rust:1.80.0 AS builder +FROM rust:1.82.0 AS builder RUN apt-get update && apt-get install -y \ openssl libclang-dev pkg-config protobuf-compiler \ diff --git a/dragonfly-client-backend/src/hdfs.rs b/dragonfly-client-backend/src/hdfs.rs index 5bd3c815d22..83518a073a9 100644 --- a/dragonfly-client-backend/src/hdfs.rs +++ b/dragonfly-client-backend/src/hdfs.rs @@ -119,11 +119,11 @@ impl super::Backend for Hdfs { "list request failed {} {}: {}", request.task_id, request.url, err ); - ClientError::BackendError(BackendError { + ClientError::BackendError(Box::new(BackendError { message: err.to_string(), status_code: None, header: None, - }) + })) })? .into_iter() .map(|entry| { @@ -150,11 +150,11 @@ impl super::Backend for Hdfs { "stat request failed {} {}: {}", request.task_id, request.url, err ); - ClientError::BackendError(BackendError { + ClientError::BackendError(Box::new(BackendError { message: err.to_string(), status_code: None, header: None, - }) + })) })?; info!( @@ -202,11 +202,11 @@ impl super::Backend for Hdfs { "get request failed {} {}: {}", request.piece_id, request.url, err ); - ClientError::BackendError(BackendError { + ClientError::BackendError(Box::new(BackendError { message: err.to_string(), status_code: None, header: None, - }) + })) })?; let stream = match request.range { @@ -218,22 +218,22 @@ impl super::Backend for Hdfs { "get request failed {} {}: {}", request.piece_id, request.url, err ); - ClientError::BackendError(BackendError { + ClientError::BackendError(Box::new(BackendError { message: err.to_string(), status_code: None, header: None, - }) + })) })?, None => operator_reader.into_bytes_stream(..).await.map_err(|err| { error!( "get request failed {} {}: {}", request.piece_id, request.url, err ); - ClientError::BackendError(BackendError { + ClientError::BackendError(Box::new(BackendError { message: err.to_string(), status_code: None, header: None, - }) + })) })?, }; diff --git a/dragonfly-client-backend/src/object_storage.rs b/dragonfly-client-backend/src/object_storage.rs index feb04188535..b24a6308ff2 100644 --- a/dragonfly-client-backend/src/object_storage.rs +++ b/dragonfly-client-backend/src/object_storage.rs @@ -201,11 +201,11 @@ impl ObjectStorage { ) -> ClientResult { // If download backend is object storage, object_storage parameter is required. let Some(object_storage) = object_storage else { - return Err(ClientError::BackendError(BackendError { + return Err(ClientError::BackendError(Box::new(BackendError { message: format!("{} need object_storage parameter", self.scheme), status_code: None, header: None, - })); + }))); }; match self.scheme { @@ -232,7 +232,7 @@ impl ObjectStorage { &object_storage.access_key_secret, &object_storage.region, ) else { - return Err(ClientError::BackendError(BackendError { + return Err(ClientError::BackendError(Box::new(BackendError { message: format!( "{} {}", self.scheme, @@ -244,7 +244,7 @@ impl ObjectStorage { ), status_code: None, header: None, - })); + }))); }; // Initialize the S3 operator with the object storage. @@ -320,7 +320,7 @@ impl ObjectStorage { &object_storage.access_key_secret, &object_storage.endpoint, ) else { - return Err(ClientError::BackendError(BackendError { + return Err(ClientError::BackendError(Box::new(BackendError { message: format!( "{} {}", self.scheme, @@ -332,7 +332,7 @@ impl ObjectStorage { ), status_code: None, header: None, - })); + }))); }; // Initialize the ABS operator with the object storage. @@ -363,7 +363,7 @@ impl ObjectStorage { &object_storage.access_key_secret, &object_storage.endpoint, ) else { - return Err(ClientError::BackendError(BackendError { + return Err(ClientError::BackendError(Box::new(BackendError { message: format!( "{} {}", self.scheme, @@ -375,7 +375,7 @@ impl ObjectStorage { ), status_code: None, header: None, - })); + }))); }; // Initialize the OSS operator with the object storage. @@ -407,7 +407,7 @@ impl ObjectStorage { &object_storage.access_key_secret, &object_storage.endpoint, ) else { - return Err(ClientError::BackendError(BackendError { + return Err(ClientError::BackendError(Box::new(BackendError { message: format!( "{} {}", self.scheme, @@ -419,7 +419,7 @@ impl ObjectStorage { ), status_code: None, header: None, - })); + }))); }; // Initialize the OBS operator with the object storage. @@ -449,7 +449,7 @@ impl ObjectStorage { &object_storage.access_key_secret, &object_storage.endpoint, ) else { - return Err(ClientError::BackendError(BackendError { + return Err(ClientError::BackendError(Box::new(BackendError { message: format!( "{} {}", self.scheme, @@ -461,7 +461,7 @@ impl ObjectStorage { ), status_code: None, header: None, - })); + }))); }; // Initialize the COS operator with the object storage. @@ -524,11 +524,11 @@ impl crate::Backend for ObjectStorage { "list request failed {} {}: {}", request.task_id, request.url, err ); - ClientError::BackendError(BackendError { + ClientError::BackendError(Box::new(BackendError { message: err.to_string(), status_code: None, header: None, - }) + })) })? .into_iter() .map(|entry| { @@ -550,11 +550,11 @@ impl crate::Backend for ObjectStorage { "stat request failed {} {}: {}", request.task_id, request.url, err ); - ClientError::BackendError(BackendError { + ClientError::BackendError(Box::new(BackendError { message: err.to_string(), status_code: None, header: None, - }) + })) })?; debug!( @@ -608,11 +608,11 @@ impl crate::Backend for ObjectStorage { "get request failed {} {}: {}", request.piece_id, request.url, err ); - ClientError::BackendError(BackendError { + ClientError::BackendError(Box::new(BackendError { message: err.to_string(), status_code: None, header: None, - }) + })) })?; let stream = match request.range { @@ -624,22 +624,22 @@ impl crate::Backend for ObjectStorage { "get request failed {} {}: {}", request.piece_id, request.url, err ); - ClientError::BackendError(BackendError { + ClientError::BackendError(Box::new(BackendError { message: err.to_string(), status_code: None, header: None, - }) + })) })?, None => operator_reader.into_bytes_stream(..).await.map_err(|err| { error!( "get request failed {} {}: {}", request.piece_id, request.url, err ); - ClientError::BackendError(BackendError { + ClientError::BackendError(Box::new(BackendError { message: err.to_string(), status_code: None, header: None, - }) + })) })?, }; diff --git a/dragonfly-client-config/build.rs b/dragonfly-client-config/build.rs index 2f1f1a147a4..55af8951fbd 100644 --- a/dragonfly-client-config/build.rs +++ b/dragonfly-client-config/build.rs @@ -61,8 +61,25 @@ fn get_commit_from_git() -> Option { fn main() { // Set the environment variables for the build platform. - if let Ok(target) = env::var("TARGET") { - println!("cargo:rustc-env=BUILD_PLATFORM={}", target); + let target = env::var("TARGET").unwrap_or_default(); + println!("cargo:rustc-env=BUILD_PLATFORM={}", target); + + // Get the RUSTFLAGS environment variable. + let mut rustflags = env::var("RUSTFLAGS").unwrap_or_default(); + + // Set the environment variables for the RUSTFLAGS. + let additional_rustflags = if target.contains("x86_64") { + "-C target-feature=+sse4.2" + } else { + "" + }; + + if !additional_rustflags.is_empty() { + if !rustflags.is_empty() { + rustflags.push(' '); + } + rustflags.push_str(additional_rustflags); + println!("cargo:rustc-env=RUSTFLAGS={}", rustflags); } // Set the environment variables for the build time. diff --git a/dragonfly-client-core/src/error/mod.rs b/dragonfly-client-core/src/error/mod.rs index a54d05d7d7d..9efeb40d8c8 100644 --- a/dragonfly-client-core/src/error/mod.rs +++ b/dragonfly-client-core/src/error/mod.rs @@ -183,7 +183,7 @@ pub enum DFError { /// BackendError is the error for backend. #[error(transparent)] - BackendError(BackendError), + BackendError(Box), /// HyperUtilClientLegacyError is the error for hyper util client legacy. #[error(transparent)] diff --git a/dragonfly-client/src/bin/dfget/main.rs b/dragonfly-client/src/bin/dfget/main.rs index d11fb0c3edf..912b1155875 100644 --- a/dragonfly-client/src/bin/dfget/main.rs +++ b/dragonfly-client/src/bin/dfget/main.rs @@ -820,14 +820,12 @@ async fn get_entries( hdfs, }) .await - .map_err(|err| { + .inspect_err(|_err| { // Collect backend request failure metrics. collect_backend_request_failure_metrics( backend.scheme().as_str(), http::Method::HEAD.as_str(), ); - - err })?; // Return error when response is failed. @@ -838,11 +836,11 @@ async fn get_entries( http::Method::HEAD.as_str(), ); - return Err(Error::BackendError(BackendError { + return Err(Error::BackendError(Box::new(BackendError { message: response.error_message.unwrap_or_default(), status_code: Some(response.http_status_code.unwrap_or_default()), header: Some(response.http_header.unwrap_or_default()), - })); + }))); } // Collect backend request finished metrics. diff --git a/dragonfly-client/src/proxy/mod.rs b/dragonfly-client/src/proxy/mod.rs index c9f568ba953..06040d3e819 100644 --- a/dragonfly-client/src/proxy/mod.rs +++ b/dragonfly-client/src/proxy/mod.rs @@ -661,7 +661,7 @@ async fn proxy_by_dfdaemon( }; // Write the task data to the reader. - let (reader, mut writer) = tokio::io::duplex(64 * 1024); + let (reader, mut writer) = tokio::io::duplex(256 * 1024); // Write the status code to the writer. let (sender, mut receiver) = mpsc::channel(10 * 1024); diff --git a/dragonfly-client/src/resource/persistent_cache_task.rs b/dragonfly-client/src/resource/persistent_cache_task.rs index 95ae17ce16b..5db6a171d07 100644 --- a/dragonfly-client/src/resource/persistent_cache_task.rs +++ b/dragonfly-client/src/resource/persistent_cache_task.rs @@ -54,7 +54,7 @@ use tokio::task::JoinSet; use tokio::time::sleep; use tokio_stream::{wrappers::ReceiverStream, StreamExt}; use tonic::{Request, Status}; -use tracing::{error, info, instrument, Instrument}; +use tracing::{debug, error, info, instrument, Instrument}; use super::*; @@ -385,7 +385,7 @@ impl PersistentCacheTask { })?; // Download the pieces from the local peer. - info!("download the pieces from local peer"); + debug!("download the pieces from local peer"); let finished_pieces = match self .download_partial_from_local_peer( task.clone(), @@ -425,7 +425,7 @@ impl PersistentCacheTask { info!("all pieces are downloaded from local peer"); return Ok(()); }; - info!("download the pieces with scheduler"); + debug!("download the pieces with scheduler"); // Download the pieces with scheduler. let finished_pieces = match self diff --git a/dragonfly-client/src/resource/piece.rs b/dragonfly-client/src/resource/piece.rs index 7f75cbf8d02..e4430ff82dd 100644 --- a/dragonfly-client/src/resource/piece.rs +++ b/dragonfly-client/src/resource/piece.rs @@ -325,12 +325,11 @@ impl Piece { self.storage .upload_piece(piece_id, task_id, range) .await - .map(|reader| { + .inspect(|_reader| { collect_upload_piece_traffic_metrics( self.id_generator.task_type(task_id) as i32, length, ); - reader }) } @@ -489,13 +488,12 @@ impl Piece { error!("piece not found"); Error::PieceNotFound(number.to_string()) }) - .map(|piece| { + .inspect(|_piece| { collect_download_piece_traffic_metrics( &TrafficType::RemotePeer, self.id_generator.task_type(task_id) as i32, length, ); - piece }) } @@ -610,11 +608,11 @@ impl Piece { error!("backend get failed: {} {}", error_message, buffer.as_str()); self.storage.download_piece_failed(piece_id)?; - return Err(Error::BackendError(BackendError { + return Err(Error::BackendError(Box::new(BackendError { message: error_message, status_code: Some(response.http_status_code.unwrap_or_default()), header: Some(response.http_header.unwrap_or_default()), - })); + }))); } // Collect the backend request finished metrics. @@ -651,13 +649,12 @@ impl Piece { error!("piece not found"); Error::PieceNotFound(number.to_string()) }) - .map(|piece| { + .inspect(|_piece| { collect_download_piece_traffic_metrics( &TrafficType::BackToSource, self.id_generator.task_type(task_id) as i32, length, ); - piece }) } } diff --git a/dragonfly-client/src/resource/task.rs b/dragonfly-client/src/resource/task.rs index 624b2579d6d..81af6e7a00e 100644 --- a/dragonfly-client/src/resource/task.rs +++ b/dragonfly-client/src/resource/task.rs @@ -161,13 +161,12 @@ impl Task { hdfs: request.hdfs, }) .await - .map_err(|err| { + .inspect_err(|_err| { // Collect the backend request failure metrics. collect_backend_request_failure_metrics( backend.scheme().as_str(), http::Method::HEAD.as_str(), ); - err })?; // Check if the status code is success. @@ -178,11 +177,11 @@ impl Task { http::Method::HEAD.as_str(), ); - return Err(Error::BackendError(BackendError { + return Err(Error::BackendError(Box::new(BackendError { message: response.error_message.unwrap_or_default(), status_code: Some(response.http_status_code.unwrap_or_default()), header: Some(response.http_header.unwrap_or_default()), - })); + }))); } // Collect the backend request finished metrics. @@ -336,7 +335,7 @@ impl Task { })?; // Download the pieces from the local peer. - info!("download the pieces from local peer"); + debug!("download the pieces from local peer"); let finished_pieces = match self .download_partial_from_local_peer( task.clone(), @@ -371,7 +370,7 @@ impl Task { info!("all pieces are downloaded from local peer"); return Ok(()); }; - info!("download the pieces with scheduler"); + debug!("download the pieces with scheduler"); // Download the pieces with scheduler. let finished_pieces = match self @@ -942,7 +941,7 @@ impl Task { while let Some(collect_piece) = piece_collector_rx.recv().await { if interrupt.load(Ordering::SeqCst) { // If the interrupt is true, break the collector loop. - info!("interrupt the piece collector"); + debug!("interrupt the piece collector"); drop(piece_collector_rx); break; } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 4d2dee853e4..2e2b8c8521e 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "1.80.0" +channel = "1.82.0"