From 89ca30e51d61f98a54bb5da1223dc62e6b66bcd1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 1 Sep 2024 00:56:45 +0000
Subject: [PATCH 01/12] chore(deps): bump crate-ci/typos from 1.23.6 to 1.24.3

Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.23.6 to 1.24.3.
- [Release notes](https://github.com/crate-ci/typos/releases)
- [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md)
- [Commits](https://github.com/crate-ci/typos/compare/v1.23.6...v1.24.3)

---
updated-dependencies:
- dependency-name: crate-ci/typos
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/ci_check.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml
index 03558adcf54e..2f2362221273 100644
--- a/.github/workflows/ci_check.yml
+++ b/.github/workflows/ci_check.yml
@@ -41,7 +41,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Check typos
-        uses: crate-ci/typos@v1.23.6
+        uses: crate-ci/typos@v1.24.3
 
   licenses:
     runs-on: ubuntu-latest

From 4cfb6f06a21405f0988bf4b3eb444d2515d2482c Mon Sep 17 00:00:00 2001
From: liugddx <liugddx@gmail.com>
Date: Sun, 1 Sep 2024 13:00:47 +0800
Subject: [PATCH 02/12] 1

---
 core/src/services/lakefs/backend.rs | 336 ++++++++++++++++++++++++++++
 core/src/services/lakefs/core.rs    | 167 ++++++++++++++
 core/src/services/lakefs/docs.md    |  62 +++++
 core/src/services/lakefs/error.rs   |  94 ++++++++
 core/src/services/lakefs/mod.rs     |  23 ++
 core/src/services/mod.rs            |   3 +
 core/src/types/operator/builder.rs  |   2 +
 core/src/types/scheme.rs            |   6 +
 8 files changed, 693 insertions(+)
 create mode 100644 core/src/services/lakefs/backend.rs
 create mode 100644 core/src/services/lakefs/core.rs
 create mode 100644 core/src/services/lakefs/docs.md
 create mode 100644 core/src/services/lakefs/error.rs
 create mode 100644 core/src/services/lakefs/mod.rs

diff --git a/core/src/services/lakefs/backend.rs b/core/src/services/lakefs/backend.rs
new file mode 100644
index 000000000000..c72cf9406121
--- /dev/null
+++ b/core/src/services/lakefs/backend.rs
@@ -0,0 +1,336 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::Response;
+use http::StatusCode;
+use log::debug;
+use serde::Deserialize;
+use serde::Serialize;
+
+use super::core::LakefsCore;
+use super::core::LakefsStatus;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+/// Configuration for Lakefs service support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct LakefsConfig {
+    /// Base url.
+    ///
+    /// This is required.
+    pub endpoint: Option<String>,
+    /// Username for HTTP basic authentication.
+    ///
+    /// This is required.
+    pub username: Option<String>,
+    /// Password for HTTP basic authentication.
+    ///
+    /// This is required.
+    pub password: Option<String>,
+    /// Root of this backend. Can be "/path/to/dir".
+    ///
+    /// Default is "/".
+    pub root: Option<String>,
+
+    /// The repository name
+    ///
+    /// This is required.
+    pub repository_id: Option<String>,
+    /// Name of the branch or a commit ID. Default is main.
+    ///
+    /// This is optional.
+    pub branch: Option<String>,
+}
+
+impl Debug for LakefsConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("LakefsConfig");
+
+        if let Some(endpoint) = &self.endpoint {
+            ds.field("endpoint", &endpoint);
+        }
+        if let Some(username) = &self.username {
+            ds.field("username", &"<redacted>");
+        }
+        if let Some(password) = &self.password {
+            ds.field("password", &"<redacted>");
+        }
+        if let Some(root) = &self.root {
+            ds.field("root", &root);
+        }
+        if let Some(repository_id) = &self.repository_id {
+            ds.field("repository_id", &repository_id);
+        }
+        if let Some(branch) = &self.branch {
+            ds.field("branch", &branch);
+        }
+
+        ds.finish()
+    }
+}
+
+impl Configurator for LakefsConfig {
+    type Builder = LakefsBuilder;
+    fn into_builder(self) -> Self::Builder {
+        LakefsBuilder { config: self }
+    }
+}
+
+/// [Lakefs](https://docs.lakefs.io/reference/api.html#/)'s API support.
+#[doc = include_str!("docs.md")]
+#[derive(Default, Clone)]
+pub struct LakefsBuilder {
+    config: LakefsConfig,
+}
+
+impl Debug for LakefsBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("Builder");
+
+        ds.field("config", &self.config);
+        ds.finish()
+    }
+}
+
+impl LakefsBuilder {
+   /// Set the endpoint of this backend.
+   ///
+   /// endpoint must be full uri.
+   ///
+   /// This is required.
+   /// - `http://127.0.0.1:8000` (lakefs daemon in local)
+   /// - `https://my-lakefs.example.com` (lakefs server)
+    pub fn endpoint(mut self, endpoint: &str) -> Self {
+        if !endpoint.is_empty() {
+            self.config.endpoint = Some(endpoint.to_string());
+        }
+        self
+    }
+
+    /// Set username of this backend. This is required.
+    pub fn username(mut self, username: &str) -> Self {
+        if !username.is_empty() {
+            self.config.username = Some(username.to_string());
+        }
+        self
+    }
+
+    /// Set password of this backend. This is required.
+    pub fn password(mut self, password: &str) -> Self {
+        if !password.is_empty() {
+            self.config.password = Some(password.to_string());
+        }
+        self
+    }
+
+    /// Set branch of this backend or a commit ID. Default is main.
+    ///
+    /// Branch can be a branch name.
+    ///
+    /// For example, branch can be:
+    /// - main
+    /// - 1d0c4eb
+    pub fn branch(mut self, branch: &str) -> Self {
+        if !branch.is_empty() {
+            self.config.branch = Some(branch.to_string());
+        }
+        self
+    }
+
+    /// Set root of this backend.
+    ///
+    /// All operations will happen under this root.
+    pub fn root(mut self, root: &str) -> Self {
+        if !root.is_empty() {
+            self.config.root = Some(root.to_string());
+        }
+        self
+    }
+
+    /// Set the repository_id of this backend.
+    ///
+    /// This is required.
+    pub fn repository_id(mut self, repository_id: &str) -> Self {
+        if !repository_id.is_empty() {
+            self.config.repository_id = Some(repository_id.to_string());
+        }
+        self
+    }
+}
+
+impl Builder for LakefsBuilder {
+    const SCHEME: Scheme = Scheme::Lakefs;
+    type Config = LakefsConfig;
+
+    /// Build a LakefsBackend.
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {:?}", &self);
+
+        let endpoint = match self.config.endpoint {
+            Some(endpoint) => Ok(endpoint.clone()),
+            None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
+                .with_operation("Builder::build")
+                .with_context("service", Scheme::Lakefs)),
+        }?;
+        debug!("backend use endpoint: {:?}", &endpoint);
+
+        let repository_id = match &self.config.repository_id {
+            Some(repository_id) => Ok(repository_id.clone()),
+            None => Err(Error::new(ErrorKind::ConfigInvalid, "repository_id is empty")
+                .with_operation("Builder::build")
+                .with_context("service", Scheme::Lakefs)),
+        }?;
+        debug!("backend use repository_id: {}", &repository_id);
+
+        let branch = match &self.config.branch {
+            Some(branch) => branch.clone(),
+            None => "main".to_string(),
+        };
+        debug!("backend use branch: {}", &branch);
+
+        let root = normalize_root(&self.config.root.unwrap_or_default());
+        debug!("backend use root: {}", &root);
+
+        let username = match &self.config.username {
+            Some(username) => Ok(username.clone()),
+            None => Err(Error::new(ErrorKind::ConfigInvalid, "username is empty")
+                .with_operation("Builder::build")
+                .with_context("service", Scheme::Lakefs)),
+        }?;
+
+        let password = match &self.config.password {
+            Some(password) => Ok(password.clone()),
+            None => Err(Error::new(ErrorKind::ConfigInvalid, "password is empty")
+                .with_operation("Builder::build")
+                .with_context("service", Scheme::Lakefs)),
+        }?;
+
+        let client = HttpClient::new()?;
+
+        Ok(LakefsBackend {
+            core: Arc::new(LakefsCore {
+                endpoint,
+                repository_id,
+                branch,
+                root,
+                username,
+                password,
+                client,
+            }),
+        })
+    }
+}
+
+/// Backend for Lakefs service
+#[derive(Debug, Clone)]
+pub struct LakefsBackend {
+    core: Arc<LakefsCore>,
+}
+
+impl Access for LakefsBackend {
+    type Reader = HttpBody;
+    type Writer = ();
+    type Lister = ();
+    type BlockingReader = ();
+    type BlockingWriter = ();
+    type BlockingLister = ();
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        let mut am = AccessorInfo::default();
+        am.set_scheme(Scheme::Lakefs)
+            .set_native_capability(Capability {
+                stat: true,
+
+                read: true,
+
+                list: true,
+                list_with_recursive: true,
+
+                ..Default::default()
+            });
+        am.into()
+    }
+
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        // Stat root always returns a DIR.
+        if path == "/" {
+            return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
+        }
+
+        let resp = self.core.get_file_meta(path).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let mut meta = parse_into_metadata(path, resp.headers())?;
+                let bs = resp.into_body();
+
+                let decoded_response: Vec<LakefsStatus> =
+                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+                // NOTE: if the file is not found, the server will return 200 with an empty array
+                if let Some(status) = decoded_response.first() {
+                    if let Some(commit_info) = status.last_commit.as_ref() {
+                        meta.set_last_modified(parse_datetime_from_rfc3339(
+                            commit_info.date.as_str(),
+                        )?);
+                    }
+
+                    meta.set_content_length(status.size);
+
+                    match status.type_.as_str() {
+                        "directory" => meta.set_mode(EntryMode::DIR),
+                        "file" => meta.set_mode(EntryMode::FILE),
+                        _ => return Err(Error::new(ErrorKind::Unexpected, "unknown status type")),
+                    };
+                } else {
+                    return Err(Error::new(ErrorKind::NotFound, "path not found"));
+                }
+
+                Ok(RpStat::new(meta))
+            }
+            _ => Err(parse_error(resp).await?),
+        }
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let resp = self.core.get_file(path, args.range(), &args).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+                Ok((RpRead::default(), resp.into_body()))
+            }
+            _ => {
+                let (part, mut body) = resp.into_parts();
+                let buf = body.to_buffer().await?;
+                Err(parse_error(Response::from_parts(part, buf)).await?)
+            }
+        }
+    }
+
+}
diff --git a/core/src/services/lakefs/core.rs b/core/src/services/lakefs/core.rs
new file mode 100644
index 000000000000..256bb07ab2df
--- /dev/null
+++ b/core/src/services/lakefs/core.rs
@@ -0,0 +1,167 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use bytes::Bytes;
+use http::header;
+use http::Request;
+use http::Response;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+pub struct LakefsCore {
+    pub endpoint: String,
+    pub repository_id: String,
+    pub branch: String,
+    pub root: String,
+    pub username: String,
+    pub password: String,
+    pub client: HttpClient,
+}
+
+impl Debug for LakefsCore {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("LakefsCore")
+            .field("endpoint", &self.endpoint)
+            .field("username", &self.username)
+            .field("password", &self.password)
+            .field("root", &self.root)
+            .field("repository_id", &self.repository_id)
+            .field("branch", &self.branch)
+            .finish_non_exhaustive()
+    }
+}
+
+impl LakefsCore {
+    pub async fn get_file_meta(&self, path: &str) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path)
+            .trim_end_matches('/')
+            .to_string();
+
+        let url = format!("{}/api/v1/repositories/{}/refs/{}/objects/stat?path={}"
+                          , self.endpoint, self.repository_id, self.branch, percent_encode_path(&p));
+
+        let mut req = Request::get(&url);
+
+        let auth_header_content = format_authorization_by_basic(&self.username,&self.password)?;
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+
+        let req = req
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.client.send(req).await
+    }
+
+    pub async fn get_file(
+        &self,
+        path: &str,
+        range: BytesRange,
+        _args: &OpRead,
+    ) -> Result<Response<HttpBody>> {
+        let p = build_abs_path(&self.root, path)
+            .trim_end_matches('/')
+            .to_string();
+
+        let url = format!("{}/api/v1/repositories/{}/refs/{}/objects?path={}"
+                          , self.endpoint, self.repository_id, self.branch, percent_encode_path(&p));
+
+        let mut req = Request::get(&url);
+
+        let auth_header_content = format_authorization_by_basic(&self.username,&self.password)?;
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+
+        if !range.is_full() {
+            req = req.header(header::RANGE, range.to_header());
+        }
+
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.client.fetch(req).await
+    }
+}
+
+#[derive(Deserialize, Eq, PartialEq, Debug)]
+#[serde(rename_all = "camelCase")]
+#[allow(dead_code)]
+pub(super) struct LakefsStatus {
+    #[serde(rename = "type")]
+    pub type_: String,
+    pub oid: String,
+    pub size: u64,
+    pub lfs: Option<LakefsLfs>,
+    pub path: String,
+    pub last_commit: Option<LakefsLastCommit>,
+    pub security: Option<LakefsSecurity>,
+}
+
+#[derive(Deserialize, Eq, PartialEq, Debug)]
+#[serde(rename_all = "camelCase")]
+#[allow(dead_code)]
+pub(super) struct LakefsLfs {
+    pub oid: String,
+    pub size: u64,
+    pub pointer_size: u64,
+}
+
+#[derive(Deserialize, Eq, PartialEq, Debug)]
+#[serde(rename_all = "camelCase")]
+#[allow(dead_code)]
+pub(super) struct LakefsLastCommit {
+    pub id: String,
+    pub title: String,
+    pub date: String,
+}
+
+#[derive(Deserialize, Eq, PartialEq, Debug)]
+#[serde(rename_all = "camelCase")]
+#[allow(dead_code)]
+pub(super) struct LakefsSecurity {
+    pub blob_id: String,
+    pub name: String,
+    pub safe: bool,
+    pub av_scan: Option<LakefsAvScan>,
+    pub pickle_import_scan: Option<LakefsPickleImportScan>,
+}
+
+#[derive(Deserialize, Eq, PartialEq, Debug)]
+#[allow(dead_code)]
+#[serde(rename_all = "camelCase")]
+pub(super) struct LakefsAvScan {
+    pub virus_found: bool,
+    pub virus_names: Option<Vec<String>>,
+}
+
+#[derive(Deserialize, Eq, PartialEq, Debug)]
+#[serde(rename_all = "camelCase")]
+#[allow(dead_code)]
+pub(super) struct LakefsPickleImportScan {
+    pub highest_safety_level: String,
+    pub imports: Vec<LakefsImport>,
+}
+
+#[derive(Deserialize, Eq, PartialEq, Debug)]
+#[allow(dead_code)]
+pub(super) struct LakefsImport {
+    pub module: String,
+    pub name: String,
+    pub safety: String,
+}
+
diff --git a/core/src/services/lakefs/docs.md b/core/src/services/lakefs/docs.md
new file mode 100644
index 000000000000..498cdf0a4c65
--- /dev/null
+++ b/core/src/services/lakefs/docs.md
@@ -0,0 +1,62 @@
+This service will visit the [Lakefs API](https://Lakefs.co/docs/Lakefs_hub/package_reference/hf_api) to access the Lakefs File System.
+Currently, we only support the `model` and `dataset` types of repositories, and operations are limited to reading and listing/stating.
+
+Lakefs doesn't host official HTTP API docs. Detailed HTTP request API information can be found on the [`Lakefs_hub` Source Code](https://github.com/Lakefs/Lakefs_hub).
+
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [ ] write
+- [ ] create_dir
+- [ ] delete
+- [ ] copy
+- [ ] rename
+- [ ] list
+- [ ] ~~presign~~
+- [ ] blocking
+
+## Configurations
+
+- `endpoint`: The endpoint of the Lakefs repository.
+- `repository_id`: The id of the repository.
+- `branch`: The branch of the repository.
+- `root`: Set the work directory for backend.
+- `username`: The username for accessing the repository.
+- `password`: The password for accessing the repository.
+
+Refer to [`LakefsBuilder`]'s public API docs for more information.
+
+## Examples
+
+### Via Builder
+
+```rust,no_run
+use opendal::Operator;
+use opendal::services::Lakefs;
+use anyhow::Result;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // Create Lakefs backend builder
+    let mut builder = Lakefs::default()
+        // set the type of Lakefs endpoint
+        .endpoint("https://whole-llama-mh6mux.us-east-1.lakefscloud.io")
+        // set the id of Lakefs repository
+        .repository_id("sample-repo")
+        // set the branch of Lakefs repository
+        .branch("main")
+        // set the username for accessing the repository
+        .username("xxx")
+        // set the password for accessing the repository
+        .password("xxx");
+
+    let op: Operator = Operator::new(builder)?.finish();
+
+    let stat = op.stat("README.md").await?;
+    println!("{:?}", stat);
+    Ok(())
+}
+```
diff --git a/core/src/services/lakefs/error.rs b/core/src/services/lakefs/error.rs
new file mode 100644
index 000000000000..38679926a89b
--- /dev/null
+++ b/core/src/services/lakefs/error.rs
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use bytes::Buf;
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+/// LakefsError is the error returned by Lakefs File System.
+#[derive(Default, Deserialize)]
+struct LakefsError {
+    error: String,
+}
+
+impl Debug for LakefsError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut de = f.debug_struct("LakefsError");
+        de.field("message", &self.error.replace('\n', " "));
+
+        de.finish()
+    }
+}
+
+pub async fn parse_error(resp: Response<Buffer>) -> Result<Error> {
+    let (parts, mut body) = resp.into_parts();
+    let bs = body.copy_to_bytes(body.remaining());
+
+    let (kind, retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        StatusCode::PRECONDITION_FAILED => (ErrorKind::ConditionNotMatch, false),
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let message = match serde_json::from_slice::<LakefsError>(&bs) {
+        Ok(hf_error) => format!("{:?}", hf_error.error),
+        Err(_) => String::from_utf8_lossy(&bs).into_owned(),
+    };
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    Ok(err)
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::raw::new_json_deserialize_error;
+    use crate::types::Result;
+
+    #[test]
+    fn test_parse_error() -> Result<()> {
+        let resp = r#"
+            {
+                "error": "Invalid username or password."
+            }
+            "#;
+        let decoded_response = serde_json::from_slice::<LakefsError>(resp.as_bytes())
+            .map_err(new_json_deserialize_error)?;
+
+        assert_eq!(decoded_response.error, "Invalid username or password.");
+
+        Ok(())
+    }
+}
diff --git a/core/src/services/lakefs/mod.rs b/core/src/services/lakefs/mod.rs
new file mode 100644
index 000000000000..3dbdf858a216
--- /dev/null
+++ b/core/src/services/lakefs/mod.rs
@@ -0,0 +1,23 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod backend;
+pub use backend::LakefsBuilder as Lakefs;
+pub use backend::LakefsConfig;
+
+mod core;
+mod error;
\ No newline at end of file
diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs
index 62ece326329d..3fb97aec1da3 100644
--- a/core/src/services/mod.rs
+++ b/core/src/services/mod.rs
@@ -213,3 +213,6 @@ pub use webhdfs::*;
 
 mod yandex_disk;
 pub use yandex_disk::*;
+
+mod lakefs;
+pub use lakefs::*;
\ No newline at end of file
diff --git a/core/src/types/operator/builder.rs b/core/src/types/operator/builder.rs
index 94e0b87bb09c..4691eda6b249 100644
--- a/core/src/types/operator/builder.rs
+++ b/core/src/types/operator/builder.rs
@@ -290,6 +290,8 @@ impl Operator {
             Scheme::Mongodb => Self::from_iter::<services::Mongodb>(iter)?.finish(),
             #[cfg(feature = "services-hdfs-native")]
             Scheme::HdfsNative => Self::from_iter::<services::HdfsNative>(iter)?.finish(),
+            #[cfg(feature = "services-lakefs")]
+            Scheme::Lakefs => Self::from_iter::<services::Lakefs>(iter)?.finish(),
             v => {
                 return Err(Error::new(
                     ErrorKind::Unsupported,
diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs
index a78fe5f6b220..4dd8acc69fb4 100644
--- a/core/src/types/scheme.rs
+++ b/core/src/types/scheme.rs
@@ -163,6 +163,8 @@ pub enum Scheme {
     HdfsNative,
     /// [surrealdb](crate::services::Surrealdb): Surrealdb Services
     Surrealdb,
+    /// [lakefs](crate::services::Lakefs): LakeFS Services
+    Lakefs,
     /// Custom that allow users to implement services outside of OpenDAL.
     ///
     /// # NOTE
@@ -311,6 +313,8 @@ impl Scheme {
             Scheme::HdfsNative,
             #[cfg(feature = "services-surrealdb")]
             Scheme::Surrealdb,
+            #[cfg(feature = "services-lakefs")]
+            Scheme::Lakefs,
         ])
     }
 }
@@ -401,6 +405,7 @@ impl FromStr for Scheme {
             "mongodb" => Ok(Scheme::Mongodb),
             "hdfs_native" => Ok(Scheme::HdfsNative),
             "surrealdb" => Ok(Scheme::Surrealdb),
+            "lakefs" => Ok(Scheme::Lakefs),
             _ => Ok(Scheme::Custom(Box::leak(s.into_boxed_str()))),
         }
     }
@@ -474,6 +479,7 @@ impl From<Scheme> for &'static str {
             Scheme::Pcloud => "pcloud",
             Scheme::HdfsNative => "hdfs_native",
             Scheme::Surrealdb => "surrealdb",
+            Scheme::Lakefs => "lakefs",
             Scheme::Custom(v) => v,
         }
     }

From c948ea4e1db47ca089dbca3ca34dfdec54e3def2 Mon Sep 17 00:00:00 2001
From: liugddx <liugddx@gmail.com>
Date: Sun, 1 Sep 2024 19:20:08 +0800
Subject: [PATCH 03/12] 1

---
 core/Cargo.lock                     |  1 +
 core/src/services/lakefs/backend.rs | 52 +++++++++++------------------
 core/src/services/lakefs/core.rs    | 47 +++++++++++++-------------
 core/src/services/lakefs/mod.rs     |  2 +-
 core/src/services/mod.rs            |  2 +-
 5 files changed, 46 insertions(+), 58 deletions(-)

diff --git a/core/Cargo.lock b/core/Cargo.lock
index d25c6e29f74a..a04f129ef105 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -4790,6 +4790,7 @@ dependencies = [
 name = "opendal-examples-basic"
 version = "0.49.1"
 dependencies = [
+ "anyhow",
  "futures",
  "opendal",
  "tokio",
diff --git a/core/src/services/lakefs/backend.rs b/core/src/services/lakefs/backend.rs
index c72cf9406121..a21b8a94f933 100644
--- a/core/src/services/lakefs/backend.rs
+++ b/core/src/services/lakefs/backend.rs
@@ -71,10 +71,10 @@ impl Debug for LakefsConfig {
         if let Some(endpoint) = &self.endpoint {
             ds.field("endpoint", &endpoint);
         }
-        if let Some(username) = &self.username {
+        if let Some(_username) = &self.username {
             ds.field("username", &"<redacted>");
         }
-        if let Some(password) = &self.password {
+        if let Some(_password) = &self.password {
             ds.field("password", &"<redacted>");
         }
         if let Some(root) = &self.root {
@@ -115,13 +115,13 @@ impl Debug for LakefsBuilder {
 }
 
 impl LakefsBuilder {
-   /// Set the endpoint of this backend.
-   ///
-   /// endpoint must be full uri.
-   ///
-   /// This is required.
-   /// - `http://127.0.0.1:8000` (lakefs daemon in local)
-   /// - `https://my-lakefs.example.com` (lakefs server)
+    /// Set the endpoint of this backend.
+    ///
+    /// endpoint must be full uri.
+    ///
+    /// This is required.
+    /// - `http://127.0.0.1:8000` (lakefs daemon in local)
+    /// - `https://my-lakefs.example.com` (lakefs server)
     pub fn endpoint(mut self, endpoint: &str) -> Self {
         if !endpoint.is_empty() {
             self.config.endpoint = Some(endpoint.to_string());
@@ -198,9 +198,11 @@ impl Builder for LakefsBuilder {
 
         let repository_id = match &self.config.repository_id {
             Some(repository_id) => Ok(repository_id.clone()),
-            None => Err(Error::new(ErrorKind::ConfigInvalid, "repository_id is empty")
-                .with_operation("Builder::build")
-                .with_context("service", Scheme::Lakefs)),
+            None => Err(
+                Error::new(ErrorKind::ConfigInvalid, "repository_id is empty")
+                    .with_operation("Builder::build")
+                    .with_context("service", Scheme::Lakefs),
+            ),
         }?;
         debug!("backend use repository_id: {}", &repository_id);
 
@@ -286,28 +288,15 @@ impl Access for LakefsBackend {
         match status {
             StatusCode::OK => {
                 let mut meta = parse_into_metadata(path, resp.headers())?;
-                let bs = resp.into_body();
+                let bs = resp.clone().into_body();
 
-                let decoded_response: Vec<LakefsStatus> =
+                let decoded_response: LakefsStatus =
                     serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
 
-                // NOTE: if the file is not found, the server will return 200 with an empty array
-                if let Some(status) = decoded_response.first() {
-                    if let Some(commit_info) = status.last_commit.as_ref() {
-                        meta.set_last_modified(parse_datetime_from_rfc3339(
-                            commit_info.date.as_str(),
-                        )?);
-                    }
-
-                    meta.set_content_length(status.size);
-
-                    match status.type_.as_str() {
-                        "directory" => meta.set_mode(EntryMode::DIR),
-                        "file" => meta.set_mode(EntryMode::FILE),
-                        _ => return Err(Error::new(ErrorKind::Unexpected, "unknown status type")),
-                    };
-                } else {
-                    return Err(Error::new(ErrorKind::NotFound, "path not found"));
+                meta.set_content_length(decoded_response.size_bytes);
+                meta.set_mode(EntryMode::FILE);
+                if let Some(v) = parse_content_disposition(resp.headers())? {
+                    meta.set_content_disposition(v);
                 }
 
                 Ok(RpStat::new(meta))
@@ -332,5 +321,4 @@ impl Access for LakefsBackend {
             }
         }
     }
-
 }
diff --git a/core/src/services/lakefs/core.rs b/core/src/services/lakefs/core.rs
index 256bb07ab2df..2292813266c1 100644
--- a/core/src/services/lakefs/core.rs
+++ b/core/src/services/lakefs/core.rs
@@ -17,7 +17,6 @@
 
 use std::fmt::Debug;
 
-use bytes::Bytes;
 use http::header;
 use http::Request;
 use http::Response;
@@ -55,17 +54,20 @@ impl LakefsCore {
             .trim_end_matches('/')
             .to_string();
 
-        let url = format!("{}/api/v1/repositories/{}/refs/{}/objects/stat?path={}"
-                          , self.endpoint, self.repository_id, self.branch, percent_encode_path(&p));
+        let url = format!(
+            "{}/api/v1/repositories/{}/refs/{}/objects/stat?path={}",
+            self.endpoint,
+            self.repository_id,
+            self.branch,
+            percent_encode_path(&p)
+        );
 
         let mut req = Request::get(&url);
 
-        let auth_header_content = format_authorization_by_basic(&self.username,&self.password)?;
+        let auth_header_content = format_authorization_by_basic(&self.username, &self.password)?;
         req = req.header(header::AUTHORIZATION, auth_header_content);
 
-        let req = req
-            .body(Buffer::new())
-            .map_err(new_request_build_error)?;
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
 
         self.client.send(req).await
     }
@@ -80,12 +82,17 @@ impl LakefsCore {
             .trim_end_matches('/')
             .to_string();
 
-        let url = format!("{}/api/v1/repositories/{}/refs/{}/objects?path={}"
-                          , self.endpoint, self.repository_id, self.branch, percent_encode_path(&p));
+        let url = format!(
+            "{}/api/v1/repositories/{}/refs/{}/objects?path={}",
+            self.endpoint,
+            self.repository_id,
+            self.branch,
+            percent_encode_path(&p)
+        );
 
         let mut req = Request::get(&url);
 
-        let auth_header_content = format_authorization_by_basic(&self.username,&self.password)?;
+        let auth_header_content = format_authorization_by_basic(&self.username, &self.password)?;
         req = req.header(header::AUTHORIZATION, auth_header_content);
 
         if !range.is_full() {
@@ -99,21 +106,18 @@ impl LakefsCore {
 }
 
 #[derive(Deserialize, Eq, PartialEq, Debug)]
-#[serde(rename_all = "camelCase")]
 #[allow(dead_code)]
 pub(super) struct LakefsStatus {
-    #[serde(rename = "type")]
-    pub type_: String,
-    pub oid: String,
-    pub size: u64,
-    pub lfs: Option<LakefsLfs>,
     pub path: String,
-    pub last_commit: Option<LakefsLastCommit>,
-    pub security: Option<LakefsSecurity>,
+    pub path_type: String,
+    pub physical_address: String,
+    pub checksum: String,
+    pub size_bytes: u64,
+    pub mtime: u32,
+    pub content_type: String,
 }
 
 #[derive(Deserialize, Eq, PartialEq, Debug)]
-#[serde(rename_all = "camelCase")]
 #[allow(dead_code)]
 pub(super) struct LakefsLfs {
     pub oid: String,
@@ -122,7 +126,6 @@ pub(super) struct LakefsLfs {
 }
 
 #[derive(Deserialize, Eq, PartialEq, Debug)]
-#[serde(rename_all = "camelCase")]
 #[allow(dead_code)]
 pub(super) struct LakefsLastCommit {
     pub id: String,
@@ -131,7 +134,6 @@ pub(super) struct LakefsLastCommit {
 }
 
 #[derive(Deserialize, Eq, PartialEq, Debug)]
-#[serde(rename_all = "camelCase")]
 #[allow(dead_code)]
 pub(super) struct LakefsSecurity {
     pub blob_id: String,
@@ -143,14 +145,12 @@ pub(super) struct LakefsSecurity {
 
 #[derive(Deserialize, Eq, PartialEq, Debug)]
 #[allow(dead_code)]
-#[serde(rename_all = "camelCase")]
 pub(super) struct LakefsAvScan {
     pub virus_found: bool,
     pub virus_names: Option<Vec<String>>,
 }
 
 #[derive(Deserialize, Eq, PartialEq, Debug)]
-#[serde(rename_all = "camelCase")]
 #[allow(dead_code)]
 pub(super) struct LakefsPickleImportScan {
     pub highest_safety_level: String,
@@ -164,4 +164,3 @@ pub(super) struct LakefsImport {
     pub name: String,
     pub safety: String,
 }
-
diff --git a/core/src/services/lakefs/mod.rs b/core/src/services/lakefs/mod.rs
index 3dbdf858a216..fdcde566f350 100644
--- a/core/src/services/lakefs/mod.rs
+++ b/core/src/services/lakefs/mod.rs
@@ -20,4 +20,4 @@ pub use backend::LakefsBuilder as Lakefs;
 pub use backend::LakefsConfig;
 
 mod core;
-mod error;
\ No newline at end of file
+mod error;
diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs
index 3fb97aec1da3..287934db2cac 100644
--- a/core/src/services/mod.rs
+++ b/core/src/services/mod.rs
@@ -215,4 +215,4 @@ mod yandex_disk;
 pub use yandex_disk::*;
 
 mod lakefs;
-pub use lakefs::*;
\ No newline at end of file
+pub use lakefs::*;

From ccc79ff55618b031dee99077a9988815bade78a9 Mon Sep 17 00:00:00 2001
From: liugddx <liugddx@gmail.com>
Date: Sun, 1 Sep 2024 20:11:48 +0800
Subject: [PATCH 04/12] 1

---
 core/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/Cargo.toml b/core/Cargo.toml
index aa7e475ecfeb..90a02fef8141 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -200,7 +200,7 @@ services-vercel-blob = []
 services-webdav = []
 services-webhdfs = []
 services-yandex-disk = []
-
+services-lakefs = []
 [lib]
 bench = false
 

From f10d6f3e8c89235933c2417230dd3ebae773f7dc Mon Sep 17 00:00:00 2001
From: liugddx <liugddx@gmail.com>
Date: Sun, 1 Sep 2024 20:42:28 +0800
Subject: [PATCH 05/12] 1

---
 core/src/services/lakefs/backend.rs | 3 +++
 core/src/services/lakefs/core.rs    | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/core/src/services/lakefs/backend.rs b/core/src/services/lakefs/backend.rs
index a21b8a94f933..d081c1ac6a71 100644
--- a/core/src/services/lakefs/backend.rs
+++ b/core/src/services/lakefs/backend.rs
@@ -20,6 +20,7 @@ use std::fmt::Formatter;
 use std::sync::Arc;
 
 use bytes::Buf;
+use chrono::{TimeZone, Utc};
 use http::Response;
 use http::StatusCode;
 use log::debug;
@@ -299,6 +300,8 @@ impl Access for LakefsBackend {
                     meta.set_content_disposition(v);
                 }
 
+                meta.set_last_modified(Utc.timestamp_opt(decoded_response.mtime, 0).unwrap());
+
                 Ok(RpStat::new(meta))
             }
             _ => Err(parse_error(resp).await?),
diff --git a/core/src/services/lakefs/core.rs b/core/src/services/lakefs/core.rs
index 2292813266c1..091f6584305a 100644
--- a/core/src/services/lakefs/core.rs
+++ b/core/src/services/lakefs/core.rs
@@ -113,7 +113,7 @@ pub(super) struct LakefsStatus {
     pub physical_address: String,
     pub checksum: String,
     pub size_bytes: u64,
-    pub mtime: u32,
+    pub mtime: i64,
     pub content_type: String,
 }
 

From 7ceec61685390755f29be5dc876b10652f0f3f9c Mon Sep 17 00:00:00 2001
From: liugddx <liugddx@gmail.com>
Date: Sun, 1 Sep 2024 20:54:41 +0800
Subject: [PATCH 06/12] 1

---
 core/Cargo.lock                     | 1 -
 core/src/services/lakefs/backend.rs | 3 ---
 2 files changed, 4 deletions(-)

diff --git a/core/Cargo.lock b/core/Cargo.lock
index a04f129ef105..d25c6e29f74a 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -4790,7 +4790,6 @@ dependencies = [
 name = "opendal-examples-basic"
 version = "0.49.1"
 dependencies = [
- "anyhow",
  "futures",
  "opendal",
  "tokio",
diff --git a/core/src/services/lakefs/backend.rs b/core/src/services/lakefs/backend.rs
index d081c1ac6a71..24b9c810eb30 100644
--- a/core/src/services/lakefs/backend.rs
+++ b/core/src/services/lakefs/backend.rs
@@ -268,9 +268,6 @@ impl Access for LakefsBackend {
 
                 read: true,
 
-                list: true,
-                list_with_recursive: true,
-
                 ..Default::default()
             });
         am.into()

From 53835f89367941f052b4204097dfe718a239b48a Mon Sep 17 00:00:00 2001
From: liugddx <liugddx@gmail.com>
Date: Sun, 1 Sep 2024 21:59:33 +0800
Subject: [PATCH 07/12] 1

---
 core/src/services/lakefs/backend.rs | 91 ++++++-----------------------
 core/src/services/lakefs/core.rs    | 60 ++-----------------
 core/src/services/lakefs/docs.md    |  4 +-
 core/src/services/lakefs/mod.rs     |  3 +
 4 files changed, 28 insertions(+), 130 deletions(-)

diff --git a/core/src/services/lakefs/backend.rs b/core/src/services/lakefs/backend.rs
index 24b9c810eb30..d111a918969f 100644
--- a/core/src/services/lakefs/backend.rs
+++ b/core/src/services/lakefs/backend.rs
@@ -31,67 +31,9 @@ use super::core::LakefsCore;
 use super::core::LakefsStatus;
 use super::error::parse_error;
 use crate::raw::*;
+use crate::services::LakefsConfig;
 use crate::*;
 
-/// Configuration for Lakefs service support.
-#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
-#[serde(default)]
-#[non_exhaustive]
-pub struct LakefsConfig {
-    /// Base url.
-    ///
-    /// This is required.
-    pub endpoint: Option<String>,
-    /// Username for HTTP basic authentication.
-    ///
-    /// This is required.
-    pub username: Option<String>,
-    /// Password for HTTP basic authentication.
-    ///
-    /// This is required.
-    pub password: Option<String>,
-    /// Root of this backend. Can be "/path/to/dir".
-    ///
-    /// Default is "/".
-    pub root: Option<String>,
-
-    /// The repository name
-    ///
-    /// This is required.
-    pub repository_id: Option<String>,
-    /// Name of the branch or a commit ID. Default is main.
-    ///
-    /// This is optional.
-    pub branch: Option<String>,
-}
-
-impl Debug for LakefsConfig {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        let mut ds = f.debug_struct("LakefsConfig");
-
-        if let Some(endpoint) = &self.endpoint {
-            ds.field("endpoint", &endpoint);
-        }
-        if let Some(_username) = &self.username {
-            ds.field("username", &"<redacted>");
-        }
-        if let Some(_password) = &self.password {
-            ds.field("password", &"<redacted>");
-        }
-        if let Some(root) = &self.root {
-            ds.field("root", &root);
-        }
-        if let Some(repository_id) = &self.repository_id {
-            ds.field("repository_id", &repository_id);
-        }
-        if let Some(branch) = &self.branch {
-            ds.field("branch", &branch);
-        }
-
-        ds.finish()
-    }
-}
-
 impl Configurator for LakefsConfig {
     type Builder = LakefsBuilder;
     fn into_builder(self) -> Self::Builder {
@@ -170,12 +112,12 @@ impl LakefsBuilder {
         self
     }
 
-    /// Set the repository_id of this backend.
+    /// Set the repository of this backend.
     ///
     /// This is required.
-    pub fn repository_id(mut self, repository_id: &str) -> Self {
-        if !repository_id.is_empty() {
-            self.config.repository_id = Some(repository_id.to_string());
+    pub fn repository(mut self, repository: &str) -> Self {
+        if !repository.is_empty() {
+            self.config.repository = Some(repository.to_string());
         }
         self
     }
@@ -197,15 +139,13 @@ impl Builder for LakefsBuilder {
         }?;
         debug!("backend use endpoint: {:?}", &endpoint);
 
-        let repository_id = match &self.config.repository_id {
-            Some(repository_id) => Ok(repository_id.clone()),
-            None => Err(
-                Error::new(ErrorKind::ConfigInvalid, "repository_id is empty")
-                    .with_operation("Builder::build")
-                    .with_context("service", Scheme::Lakefs),
-            ),
+        let repository = match &self.config.repository {
+            Some(repository) => Ok(repository.clone()),
+            None => Err(Error::new(ErrorKind::ConfigInvalid, "repository is empty")
+                .with_operation("Builder::build")
+                .with_context("service", Scheme::Lakefs)),
         }?;
-        debug!("backend use repository_id: {}", &repository_id);
+        debug!("backend use repository: {}", &repository);
 
         let branch = match &self.config.branch {
             Some(branch) => branch.clone(),
@@ -235,7 +175,7 @@ impl Builder for LakefsBuilder {
         Ok(LakefsBackend {
             core: Arc::new(LakefsCore {
                 endpoint,
-                repository_id,
+                repository,
                 branch,
                 root,
                 username,
@@ -279,7 +219,7 @@ impl Access for LakefsBackend {
             return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
         }
 
-        let resp = self.core.get_file_meta(path).await?;
+        let resp = self.core.get_object_metadata(path).await?;
 
         let status = resp.status();
 
@@ -306,7 +246,10 @@ impl Access for LakefsBackend {
     }
 
     async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
-        let resp = self.core.get_file(path, args.range(), &args).await?;
+        let resp = self
+            .core
+            .get_object_content(path, args.range(), &args)
+            .await?;
 
         let status = resp.status();
 
diff --git a/core/src/services/lakefs/core.rs b/core/src/services/lakefs/core.rs
index 091f6584305a..c275bbc17fad 100644
--- a/core/src/services/lakefs/core.rs
+++ b/core/src/services/lakefs/core.rs
@@ -27,7 +27,7 @@ use crate::*;
 
 pub struct LakefsCore {
     pub endpoint: String,
-    pub repository_id: String,
+    pub repository: String,
     pub branch: String,
     pub root: String,
     pub username: String,
@@ -42,14 +42,14 @@ impl Debug for LakefsCore {
             .field("username", &self.username)
             .field("password", &self.password)
             .field("root", &self.root)
-            .field("repository_id", &self.repository_id)
+            .field("repository", &self.repository)
             .field("branch", &self.branch)
             .finish_non_exhaustive()
     }
 }
 
 impl LakefsCore {
-    pub async fn get_file_meta(&self, path: &str) -> Result<Response<Buffer>> {
+    pub async fn get_object_metadata(&self, path: &str) -> Result<Response<Buffer>> {
         let p = build_abs_path(&self.root, path)
             .trim_end_matches('/')
             .to_string();
@@ -57,7 +57,7 @@ impl LakefsCore {
         let url = format!(
             "{}/api/v1/repositories/{}/refs/{}/objects/stat?path={}",
             self.endpoint,
-            self.repository_id,
+            self.repository,
             self.branch,
             percent_encode_path(&p)
         );
@@ -72,7 +72,7 @@ impl LakefsCore {
         self.client.send(req).await
     }
 
-    pub async fn get_file(
+    pub async fn get_object_content(
         &self,
         path: &str,
         range: BytesRange,
@@ -85,7 +85,7 @@ impl LakefsCore {
         let url = format!(
             "{}/api/v1/repositories/{}/refs/{}/objects?path={}",
             self.endpoint,
-            self.repository_id,
+            self.repository,
             self.branch,
             percent_encode_path(&p)
         );
@@ -116,51 +116,3 @@ pub(super) struct LakefsStatus {
     pub mtime: i64,
     pub content_type: String,
 }
-
-#[derive(Deserialize, Eq, PartialEq, Debug)]
-#[allow(dead_code)]
-pub(super) struct LakefsLfs {
-    pub oid: String,
-    pub size: u64,
-    pub pointer_size: u64,
-}
-
-#[derive(Deserialize, Eq, PartialEq, Debug)]
-#[allow(dead_code)]
-pub(super) struct LakefsLastCommit {
-    pub id: String,
-    pub title: String,
-    pub date: String,
-}
-
-#[derive(Deserialize, Eq, PartialEq, Debug)]
-#[allow(dead_code)]
-pub(super) struct LakefsSecurity {
-    pub blob_id: String,
-    pub name: String,
-    pub safe: bool,
-    pub av_scan: Option<LakefsAvScan>,
-    pub pickle_import_scan: Option<LakefsPickleImportScan>,
-}
-
-#[derive(Deserialize, Eq, PartialEq, Debug)]
-#[allow(dead_code)]
-pub(super) struct LakefsAvScan {
-    pub virus_found: bool,
-    pub virus_names: Option<Vec<String>>,
-}
-
-#[derive(Deserialize, Eq, PartialEq, Debug)]
-#[allow(dead_code)]
-pub(super) struct LakefsPickleImportScan {
-    pub highest_safety_level: String,
-    pub imports: Vec<LakefsImport>,
-}
-
-#[derive(Deserialize, Eq, PartialEq, Debug)]
-#[allow(dead_code)]
-pub(super) struct LakefsImport {
-    pub module: String,
-    pub name: String,
-    pub safety: String,
-}
diff --git a/core/src/services/lakefs/docs.md b/core/src/services/lakefs/docs.md
index 498cdf0a4c65..2f71184512d5 100644
--- a/core/src/services/lakefs/docs.md
+++ b/core/src/services/lakefs/docs.md
@@ -21,7 +21,7 @@ This service can be used to:
 ## Configurations
 
 - `endpoint`: The endpoint of the Lakefs repository.
-- `repository_id`: The id of the repository.
+- `repository`: The id of the repository.
 - `branch`: The branch of the repository.
 - `root`: Set the work directory for backend.
 - `username`: The username for accessing the repository.
@@ -45,7 +45,7 @@ async fn main() -> Result<()> {
         // set the type of Lakefs endpoint
         .endpoint("https://whole-llama-mh6mux.us-east-1.lakefscloud.io")
         // set the id of Lakefs repository
-        .repository_id("sample-repo")
+        .repository("sample-repo")
         // set the branch of Lakefs repository
         .branch("main")
         // set the username for accessing the repository
diff --git a/core/src/services/lakefs/mod.rs b/core/src/services/lakefs/mod.rs
index fdcde566f350..cae5612f6465 100644
--- a/core/src/services/lakefs/mod.rs
+++ b/core/src/services/lakefs/mod.rs
@@ -19,5 +19,8 @@ mod backend;
 pub use backend::LakefsBuilder as Lakefs;
 pub use backend::LakefsConfig;
 
+mod config;
 mod core;
 mod error;
+
+pub use config::LakefsConfig;

From 156208dd9dccfc67c112daf35990b975f6341709 Mon Sep 17 00:00:00 2001
From: liugddx <liugddx@gmail.com>
Date: Sun, 1 Sep 2024 22:05:33 +0800
Subject: [PATCH 08/12] 1

---
 core/src/services/lakefs/config.rs | 79 ++++++++++++++++++++++++++++++
 core/src/services/lakefs/mod.rs    |  2 -
 2 files changed, 79 insertions(+), 2 deletions(-)
 create mode 100644 core/src/services/lakefs/config.rs

diff --git a/core/src/services/lakefs/config.rs b/core/src/services/lakefs/config.rs
new file mode 100644
index 000000000000..aa2826e93d04
--- /dev/null
+++ b/core/src/services/lakefs/config.rs
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::{Debug, Formatter};
+
+use serde::{Deserialize, Serialize};
+
+/// Configuration for Lakefs service support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct LakefsConfig {
+    /// Base url.
+    ///
+    /// This is required.
+    pub endpoint: Option<String>,
+    /// Username for Lakefs basic authentication.
+    ///
+    /// This is required.
+    pub username: Option<String>,
+    /// Password for Lakefs basic authentication.
+    ///
+    /// This is required.
+    pub password: Option<String>,
+    /// Root of this backend. Can be "/path/to/dir".
+    ///
+    /// Default is "/".
+    pub root: Option<String>,
+
+    /// The repository name
+    ///
+    /// This is required.
+    pub repository: Option<String>,
+    /// Name of the branch or a commit ID. Default is main.
+    ///
+    /// This is optional.
+    pub branch: Option<String>,
+}
+
+impl Debug for LakefsConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("LakefsConfig");
+
+        if let Some(endpoint) = &self.endpoint {
+            ds.field("endpoint", &endpoint);
+        }
+        if let Some(_username) = &self.username {
+            ds.field("username", &"<redacted>");
+        }
+        if let Some(_password) = &self.password {
+            ds.field("password", &"<redacted>");
+        }
+        if let Some(root) = &self.root {
+            ds.field("root", &root);
+        }
+        if let Some(repository) = &self.repository {
+            ds.field("repository", &repository);
+        }
+        if let Some(branch) = &self.branch {
+            ds.field("branch", &branch);
+        }
+
+        ds.finish()
+    }
+}
diff --git a/core/src/services/lakefs/mod.rs b/core/src/services/lakefs/mod.rs
index cae5612f6465..19d438355bb0 100644
--- a/core/src/services/lakefs/mod.rs
+++ b/core/src/services/lakefs/mod.rs
@@ -17,10 +17,8 @@
 
 mod backend;
 pub use backend::LakefsBuilder as Lakefs;
-pub use backend::LakefsConfig;
 
 mod config;
 mod core;
 mod error;
-
 pub use config::LakefsConfig;

From cf6969569391552414070d76771047e4c4acab2a Mon Sep 17 00:00:00 2001
From: Guangdong Liu <804167098@qq.com>
Date: Sun, 1 Sep 2024 22:08:24 +0800
Subject: [PATCH 09/12] Update ci_check.yml

---
 .github/workflows/ci_check.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml
index 2f2362221273..03558adcf54e 100644
--- a/.github/workflows/ci_check.yml
+++ b/.github/workflows/ci_check.yml
@@ -41,7 +41,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Check typos
-        uses: crate-ci/typos@v1.24.3
+        uses: crate-ci/typos@v1.23.6
 
   licenses:
     runs-on: ubuntu-latest

From f09f101b11a8e59565131fa4e12ed000378fb650 Mon Sep 17 00:00:00 2001
From: liugddx <liugddx@gmail.com>
Date: Sun, 1 Sep 2024 22:19:39 +0800
Subject: [PATCH 10/12] 1

---
 core/src/services/lakefs/backend.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/core/src/services/lakefs/backend.rs b/core/src/services/lakefs/backend.rs
index d111a918969f..3197ccc08411 100644
--- a/core/src/services/lakefs/backend.rs
+++ b/core/src/services/lakefs/backend.rs
@@ -24,8 +24,6 @@ use chrono::{TimeZone, Utc};
 use http::Response;
 use http::StatusCode;
 use log::debug;
-use serde::Deserialize;
-use serde::Serialize;
 
 use super::core::LakefsCore;
 use super::core::LakefsStatus;

From 2462ad33026daf54fd62c11632ca169cdcc14b7b Mon Sep 17 00:00:00 2001
From: liugddx <liugddx@gmail.com>
Date: Mon, 2 Sep 2024 08:20:37 +0800
Subject: [PATCH 11/12] 1

---
 core/src/services/lakefs/mod.rs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/core/src/services/lakefs/mod.rs b/core/src/services/lakefs/mod.rs
index 19d438355bb0..5f5675039925 100644
--- a/core/src/services/lakefs/mod.rs
+++ b/core/src/services/lakefs/mod.rs
@@ -15,10 +15,15 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#[cfg(feature = "services-lakefs")]
+mod core;
+#[cfg(feature = "services-lakefs")]
+mod error;
+
+#[cfg(feature = "services-lakefs")]
 mod backend;
+#[cfg(feature = "services-lakefs")]
 pub use backend::LakefsBuilder as Lakefs;
 
 mod config;
-mod core;
-mod error;
 pub use config::LakefsConfig;

From bbcc06bde27dcd415087d4fe75e745b519f72269 Mon Sep 17 00:00:00 2001
From: liugddx <liugddx@gmail.com>
Date: Mon, 2 Sep 2024 08:22:19 +0800
Subject: [PATCH 12/12] 1

---
 core/src/services/mod.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs
index 287934db2cac..753838b31314 100644
--- a/core/src/services/mod.rs
+++ b/core/src/services/mod.rs
@@ -118,6 +118,9 @@ pub use ipmfs::*;
 mod koofr;
 pub use koofr::*;
 
+mod lakefs;
+pub use lakefs::*;
+
 mod libsql;
 pub use libsql::*;
 
@@ -213,6 +216,3 @@ pub use webhdfs::*;
 
 mod yandex_disk;
 pub use yandex_disk::*;
-
-mod lakefs;
-pub use lakefs::*;