Commit 730c938 1 parent 10fce65 commit 730c938 Copy full SHA for 730c938
File tree 2 files changed +31
-33
lines changed
2 files changed +31
-33
lines changed Original file line number Diff line number Diff line change @@ -21,38 +21,6 @@ use crate::{
21
21
} ;
22
22
23
23
/// Configuration for the `webhdfs` sink.
24
- ///
25
- /// The Hadoop Distributed File System (HDFS) is a distributed file system
26
- /// designed to run on commodity hardware. HDFS consists of a namenode and a
27
- /// datanode. We will send rpc to namenode to know which datanode to send
28
- /// and receive data to. Also, HDFS will rebalance data across the cluster
29
- /// to make sure each file has enough redundancy.
30
- ///
31
- /// ```txt
32
- /// ┌───────────────┐
33
- /// │ Data Node 2 │
34
- /// └───────────────┘
35
- /// ▲
36
- /// ┌───────────────┐ │ ┌───────────────┐
37
- /// │ Data Node 1 │◄──────────┼───────────►│ Data Node 3 │
38
- /// └───────────────┘ │ └───────────────┘
39
- /// ┌───────┴───────┐
40
- /// │ Name Node │
41
- /// └───────────────┘
42
- /// ▲
43
- /// │
44
- /// ┌──────┴─────┐
45
- /// │ Vector │
46
- /// └────────────┘
47
- /// ```
48
- ///
49
- /// WebHDFS will connect to the HTTP RESTful API of HDFS.
50
- ///
51
- /// For more information, please refer to:
52
- ///
53
- /// - [HDFS Users Guide](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsUserGuide.html)
54
- /// - [WebHDFS REST API](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html)
55
- /// - [opendal::services::webhdfs](https://docs.rs/opendal/latest/opendal/services/struct.Webhdfs.html)
56
24
#[ configurable_component( sink( "webhdfs" ) ) ]
57
25
#[ derive( Clone , Debug ) ]
58
26
#[ serde( deny_unknown_fields) ]
Original file line number Diff line number Diff line change 1
1
//! `webhdfs` sink.
2
2
//!
3
- //! This sink will send it's output to WEBHDFS.
3
+ //! The Hadoop Distributed File System (HDFS) is a distributed file system
4
+ //! designed to run on commodity hardware. HDFS consists of a namenode and a
5
+ //! datanode. We will send rpc to namenode to know which datanode to send
6
+ //! and receive data to. Also, HDFS will rebalance data across the cluster
7
+ //! to make sure each file has enough redundancy.
8
+ //!
9
+ //! ```txt
10
+ //! ┌───────────────┐
11
+ //! │ Data Node 2 │
12
+ //! └───────────────┘
13
+ //! ▲
14
+ //! ┌───────────────┐ │ ┌───────────────┐
15
+ //! │ Data Node 1 │◄──────────┼───────────►│ Data Node 3 │
16
+ //! └───────────────┘ │ └───────────────┘
17
+ //! ┌───────┴───────┐
18
+ //! │ Name Node │
19
+ //! └───────────────┘
20
+ //! ▲
21
+ //! │
22
+ //! ┌──────┴─────┐
23
+ //! │ Vector │
24
+ //! └────────────┘
25
+ //! ```
26
+ //!
27
+ //! WebHDFS will connect to the HTTP RESTful API of HDFS.
28
+ //!
29
+ //! For more information, please refer to:
30
+ //!
31
+ //! - [HDFS Users Guide](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsUserGuide.html)
32
+ //! - [WebHDFS REST API](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html)
33
+ //! - [opendal::services::webhdfs](https://docs.rs/opendal/latest/opendal/services/struct.Webhdfs.html)
4
34
//!
5
35
//! `webhdfs` is an OpenDal based services. This mod itself only provide
6
36
//! config to build an [`crate::sinks::opendal_common::OpenDalSink`]. All real implement are powered by
You can’t perform that action at this time.
0 commit comments