Skip to content

Commit

Permalink
feat: add sse4.2 target feature for building (dragonflyoss#867)
Browse files Browse the repository at this point in the history 
Signed-off-by: Gaius <[email protected]>
  • Loading branch information
@gaius-qi
gaius-qi authored Nov 26, 2024
1 parent f827084 commit 62c62f7
Show file tree
Hide file tree
Showing 12 changed files with 73 additions and 62 deletions.
2 changes: 1 addition & 1 deletion ci/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM rust:1.80.0 AS builder
FROM rust:1.82.0 AS builder

WORKDIR /app/client

Expand Down
2 changes: 1 addition & 1 deletion ci/Dockerfile.dfinit
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM rust:1.80.0 AS builder
FROM rust:1.82.0 AS builder

RUN apt-get update && apt-get install -y \
openssl libclang-dev pkg-config protobuf-compiler \
Expand Down
20 changes: 10 additions & 10 deletions dragonfly-client-backend/src/hdfs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,11 @@ impl super::Backend for Hdfs {
"list request failed {} {}: {}",
request.task_id, request.url, err
);
ClientError::BackendError(BackendError {
ClientError::BackendError(Box::new(BackendError {
message: err.to_string(),
status_code: None,
header: None,
})
}))
})?
.into_iter()
.map(|entry| {
Expand All @@ -150,11 +150,11 @@ impl super::Backend for Hdfs {
"stat request failed {} {}: {}",
request.task_id, request.url, err
);
ClientError::BackendError(BackendError {
ClientError::BackendError(Box::new(BackendError {
message: err.to_string(),
status_code: None,
header: None,
})
}))
})?;

info!(
Expand Down Expand Up @@ -202,11 +202,11 @@ impl super::Backend for Hdfs {
"get request failed {} {}: {}",
request.piece_id, request.url, err
);
ClientError::BackendError(BackendError {
ClientError::BackendError(Box::new(BackendError {
message: err.to_string(),
status_code: None,
header: None,
})
}))
})?;

let stream = match request.range {
Expand All @@ -218,22 +218,22 @@ impl super::Backend for Hdfs {
"get request failed {} {}: {}",
request.piece_id, request.url, err
);
ClientError::BackendError(BackendError {
ClientError::BackendError(Box::new(BackendError {
message: err.to_string(),
status_code: None,
header: None,
})
}))
})?,
None => operator_reader.into_bytes_stream(..).await.map_err(|err| {
error!(
"get request failed {} {}: {}",
request.piece_id, request.url, err
);
ClientError::BackendError(BackendError {
ClientError::BackendError(Box::new(BackendError {
message: err.to_string(),
status_code: None,
header: None,
})
}))
})?,
};

Expand Down
44 changes: 22 additions & 22 deletions dragonfly-client-backend/src/object_storage.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,11 +201,11 @@ impl ObjectStorage {
) -> ClientResult<Operator> {
// If download backend is object storage, object_storage parameter is required.
let Some(object_storage) = object_storage else {
return Err(ClientError::BackendError(BackendError {
return Err(ClientError::BackendError(Box::new(BackendError {
message: format!("{} need object_storage parameter", self.scheme),
status_code: None,
header: None,
}));
})));
};

match self.scheme {
Expand All @@ -232,7 +232,7 @@ impl ObjectStorage {
&object_storage.access_key_secret,
&object_storage.region,
) else {
return Err(ClientError::BackendError(BackendError {
return Err(ClientError::BackendError(Box::new(BackendError {
message: format!(
"{} {}",
self.scheme,
Expand All @@ -244,7 +244,7 @@ impl ObjectStorage {
),
status_code: None,
header: None,
}));
})));
};

// Initialize the S3 operator with the object storage.
Expand Down Expand Up @@ -320,7 +320,7 @@ impl ObjectStorage {
&object_storage.access_key_secret,
&object_storage.endpoint,
) else {
return Err(ClientError::BackendError(BackendError {
return Err(ClientError::BackendError(Box::new(BackendError {
message: format!(
"{} {}",
self.scheme,
Expand All @@ -332,7 +332,7 @@ impl ObjectStorage {
),
status_code: None,
header: None,
}));
})));
};

// Initialize the ABS operator with the object storage.
Expand Down Expand Up @@ -363,7 +363,7 @@ impl ObjectStorage {
&object_storage.access_key_secret,
&object_storage.endpoint,
) else {
return Err(ClientError::BackendError(BackendError {
return Err(ClientError::BackendError(Box::new(BackendError {
message: format!(
"{} {}",
self.scheme,
Expand All @@ -375,7 +375,7 @@ impl ObjectStorage {
),
status_code: None,
header: None,
}));
})));
};

// Initialize the OSS operator with the object storage.
Expand Down Expand Up @@ -407,7 +407,7 @@ impl ObjectStorage {
&object_storage.access_key_secret,
&object_storage.endpoint,
) else {
return Err(ClientError::BackendError(BackendError {
return Err(ClientError::BackendError(Box::new(BackendError {
message: format!(
"{} {}",
self.scheme,
Expand All @@ -419,7 +419,7 @@ impl ObjectStorage {
),
status_code: None,
header: None,
}));
})));
};

// Initialize the OBS operator with the object storage.
Expand Down Expand Up @@ -449,7 +449,7 @@ impl ObjectStorage {
&object_storage.access_key_secret,
&object_storage.endpoint,
) else {
return Err(ClientError::BackendError(BackendError {
return Err(ClientError::BackendError(Box::new(BackendError {
message: format!(
"{} {}",
self.scheme,
Expand All @@ -461,7 +461,7 @@ impl ObjectStorage {
),
status_code: None,
header: None,
}));
})));
};

// Initialize the COS operator with the object storage.
Expand Down Expand Up @@ -524,11 +524,11 @@ impl crate::Backend for ObjectStorage {
"list request failed {} {}: {}",
request.task_id, request.url, err
);
ClientError::BackendError(BackendError {
ClientError::BackendError(Box::new(BackendError {
message: err.to_string(),
status_code: None,
header: None,
})
}))
})?
.into_iter()
.map(|entry| {
Expand All @@ -550,11 +550,11 @@ impl crate::Backend for ObjectStorage {
"stat request failed {} {}: {}",
request.task_id, request.url, err
);
ClientError::BackendError(BackendError {
ClientError::BackendError(Box::new(BackendError {
message: err.to_string(),
status_code: None,
header: None,
})
}))
})?;

debug!(
Expand Down Expand Up @@ -608,11 +608,11 @@ impl crate::Backend for ObjectStorage {
"get request failed {} {}: {}",
request.piece_id, request.url, err
);
ClientError::BackendError(BackendError {
ClientError::BackendError(Box::new(BackendError {
message: err.to_string(),
status_code: None,
header: None,
})
}))
})?;

let stream = match request.range {
Expand All @@ -624,22 +624,22 @@ impl crate::Backend for ObjectStorage {
"get request failed {} {}: {}",
request.piece_id, request.url, err
);
ClientError::BackendError(BackendError {
ClientError::BackendError(Box::new(BackendError {
message: err.to_string(),
status_code: None,
header: None,
})
}))
})?,
None => operator_reader.into_bytes_stream(..).await.map_err(|err| {
error!(
"get request failed {} {}: {}",
request.piece_id, request.url, err
);
ClientError::BackendError(BackendError {
ClientError::BackendError(Box::new(BackendError {
message: err.to_string(),
status_code: None,
header: None,
})
}))
})?,
};

Expand Down
21 changes: 19 additions & 2 deletions dragonfly-client-config/build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,25 @@ fn get_commit_from_git() -> Option<Commit> {

fn main() {
// Set the environment variables for the build platform.
if let Ok(target) = env::var("TARGET") {
println!("cargo:rustc-env=BUILD_PLATFORM={}", target);
let target = env::var("TARGET").unwrap_or_default();
println!("cargo:rustc-env=BUILD_PLATFORM={}", target);

// Get the RUSTFLAGS environment variable.
let mut rustflags = env::var("RUSTFLAGS").unwrap_or_default();

// Set the environment variables for the RUSTFLAGS.
let additional_rustflags = if target.contains("x86_64") {
"-C target-feature=+sse4.2"
} else {
""
};

if !additional_rustflags.is_empty() {
if !rustflags.is_empty() {
rustflags.push(' ');
}
rustflags.push_str(additional_rustflags);
println!("cargo:rustc-env=RUSTFLAGS={}", rustflags);
}

// Set the environment variables for the build time.
Expand Down
2 changes: 1 addition & 1 deletion dragonfly-client-core/src/error/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ pub enum DFError {

/// BackendError is the error for backend.
#[error(transparent)]
BackendError(BackendError),
BackendError(Box<BackendError>),

/// HyperUtilClientLegacyError is the error for hyper util client legacy.
#[error(transparent)]
Expand Down
8 changes: 3 additions & 5 deletions dragonfly-client/src/bin/dfget/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -820,14 +820,12 @@ async fn get_entries(
hdfs,
})
.await
.map_err(|err| {
.inspect_err(|_err| {
// Collect backend request failure metrics.
collect_backend_request_failure_metrics(
backend.scheme().as_str(),
http::Method::HEAD.as_str(),
);

err
})?;

// Return error when response is failed.
Expand All @@ -838,11 +836,11 @@ async fn get_entries(
http::Method::HEAD.as_str(),
);

return Err(Error::BackendError(BackendError {
return Err(Error::BackendError(Box::new(BackendError {
message: response.error_message.unwrap_or_default(),
status_code: Some(response.http_status_code.unwrap_or_default()),
header: Some(response.http_header.unwrap_or_default()),
}));
})));
}

// Collect backend request finished metrics.
Expand Down
2 changes: 1 addition & 1 deletion dragonfly-client/src/proxy/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -661,7 +661,7 @@ async fn proxy_by_dfdaemon(
};

// Write the task data to the reader.
let (reader, mut writer) = tokio::io::duplex(64 * 1024);
let (reader, mut writer) = tokio::io::duplex(256 * 1024);

// Write the status code to the writer.
let (sender, mut receiver) = mpsc::channel(10 * 1024);
Expand Down
6 changes: 3 additions & 3 deletions dragonfly-client/src/resource/persistent_cache_task.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ use tokio::task::JoinSet;
use tokio::time::sleep;
use tokio_stream::{wrappers::ReceiverStream, StreamExt};
use tonic::{Request, Status};
use tracing::{error, info, instrument, Instrument};
use tracing::{debug, error, info, instrument, Instrument};

use super::*;

Expand Down Expand Up @@ -385,7 +385,7 @@ impl PersistentCacheTask {
})?;

// Download the pieces from the local peer.
info!("download the pieces from local peer");
debug!("download the pieces from local peer");
let finished_pieces = match self
.download_partial_from_local_peer(
task.clone(),
Expand Down Expand Up @@ -425,7 +425,7 @@ impl PersistentCacheTask {
info!("all pieces are downloaded from local peer");
return Ok(());
};
info!("download the pieces with scheduler");
debug!("download the pieces with scheduler");

// Download the pieces with scheduler.
let finished_pieces = match self
Expand Down
Loading

0 comments on commit 62c62f7

Please sign in to comment.