Skip to content

Commit bad40e9

Browse files
krapiekrapie
committed
[Feature] Add Shard Cluster Mode PoC for docker
1 parent 01889fc commit bad40e9

File tree

8 files changed

+1862
-0
lines changed

8 files changed

+1862
-0
lines changed

‎docker/yorkie-shard-cluster-docker/README.md

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Yorkie Shard Cluster Docker PoC
2+
3+
> Warning: You need to put API Key in Yorkie Client to see proper routing!!!
4+
5+
## Architecture
6+
7+
- Router
8+
- envoy: router/proxy which performs ring hash algorithm based routing based on x-api-key (project_id)
9+
- Server
10+
- yorkie1, yorkie2, yorkie3: normal yorkie server with 3 replica. They do **NOT** communicate with each other as previous broadcasting based cluster mode
11+
- Database
12+
- mongo: stand-alone mongoDB. mongoDB with shard will be implemented soon (yorkie server's mongo client needs some shard related changes to use mongoDB shard)
13+
14+
## Getting Started
15+
16+
### Step 1. Deploy
17+
18+
Deploy docker based servers using docker-compose:
19+
20+
```
21+
docker-compose up --build -d
22+
```
23+
24+
### Step 2. Get public API key
25+
26+
API Key is **essential** to perform api key based routing:
27+
28+
```
29+
git clone https://github.com/yorkie-team/dashboard.git
30+
31+
npm install
32+
33+
npm run start
34+
35+
get API key
36+
```
37+
38+
### Step 3. Use any Yorkie implemented examples
39+
40+
Test with any Yorkie implemented examples:
41+
42+
```
43+
git clone https://github.com/krapie/yorkie-tldraw
44+
45+
put API Key in `REACT_APP_YORKIE_API_KEY` in `.env.Production`
46+
47+
yarn
48+
49+
yarn start
50+
```
51+
52+
### Step 4. Check routing behavior
53+
54+
Check routing behavior by watching docker logs:
55+
56+
```
57+
docker compose logs
58+
```

‎docker/yorkie-shard-cluster-docker/docker-compose.yml

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
version: "3.3"
2+
3+
services:
4+
envoy:
5+
build:
6+
context: ./
7+
dockerfile: ./envoy.Dockerfile
8+
image: "grpcweb:envoy"
9+
container_name: "envoy"
10+
restart: always
11+
ports:
12+
- "8080:8080"
13+
- "9090:9090"
14+
- "9901:9901"
15+
command: ["/etc/envoy/envoy.yaml"]
16+
depends_on:
17+
- yorkie1
18+
- yorkie2
19+
- yorkie3
20+
extra_hosts:
21+
- "host.docker.internal:host-gateway"
22+
23+
yorkie1:
24+
image: "yorkieteam/yorkie:latest"
25+
container_name: "yorkie1"
26+
command:
27+
[
28+
"server",
29+
"--mongo-connection-uri",
30+
"mongodb://mongo:27017",
31+
"--enable-pprof",
32+
]
33+
restart: always
34+
ports:
35+
- "11101:11101"
36+
- "11102:11102"
37+
- "11103:11103"
38+
depends_on:
39+
- mongo
40+
41+
yorkie2:
42+
image: "yorkieteam/yorkie:latest"
43+
container_name: "yorkie2"
44+
command:
45+
[
46+
"server",
47+
"--mongo-connection-uri",
48+
"mongodb://mongo:27017",
49+
"--enable-pprof",
50+
]
51+
restart: always
52+
ports:
53+
- "11201:11101"
54+
- "11202:11102"
55+
- "11203:11103"
56+
depends_on:
57+
- mongo
58+
59+
yorkie3:
60+
image: "yorkieteam/yorkie:latest"
61+
container_name: "yorkie3"
62+
command:
63+
[
64+
"server",
65+
"--mongo-connection-uri",
66+
"mongodb://mongo:27017",
67+
"--enable-pprof",
68+
]
69+
restart: always
70+
ports:
71+
- "11301:11101"
72+
- "11302:11102"
73+
- "11303:11103"
74+
depends_on:
75+
- mongo
76+
77+
mongo:
78+
image: mongo:latest
79+
container_name: mongo
80+
restart: always
81+
ports:
82+
- "27017:27017"

‎docker/yorkie-shard-cluster-docker/envoy-ring-hash-router.yaml

+127
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
admin:
2+
access_log_path: /tmp/admin_access.log
3+
address:
4+
socket_address: { address: 0.0.0.0, port_value: 9901 }
5+
6+
static_resources:
7+
listeners:
8+
- name: yorkie_listener
9+
address:
10+
socket_address: { address: 0.0.0.0, port_value: 8080 }
11+
filter_chains:
12+
- filters:
13+
- name: envoy.filters.network.http_connection_manager
14+
typed_config:
15+
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
16+
stat_prefix: ingress_http
17+
route_config:
18+
name: yorkie_ring_hash_routes
19+
virtual_hosts:
20+
- name: yorkie_service
21+
domains: ["*"]
22+
routes:
23+
- match: { prefix: "/" }
24+
route:
25+
cluster: yorkie_service
26+
hash_policy:
27+
header:
28+
header_name: x-api-key
29+
# https://github.com/grpc/grpc-web/issues/361
30+
max_stream_duration:
31+
grpc_timeout_header_max: 0s
32+
cors:
33+
allow_origin_string_match:
34+
- prefix: "*"
35+
allow_methods: GET, PUT, DELETE, POST, OPTIONS
36+
allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,custom-header-1,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout,authorization,x-api-key
37+
max_age: "1728000"
38+
expose_headers: custom-header-1,grpc-status,grpc-message, grpc-status-details-bin
39+
http_filters:
40+
- name: envoy.filters.http.grpc_web
41+
- name: envoy.filters.http.cors
42+
- name: envoy.filters.http.router
43+
- name: admin_listener
44+
address:
45+
socket_address: { address: 0.0.0.0, port_value: 9090 }
46+
filter_chains:
47+
- filters:
48+
- name: envoy.filters.network.http_connection_manager
49+
typed_config:
50+
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
51+
stat_prefix: ingress_http
52+
route_config:
53+
name: yorkie_ring_hash_routes
54+
virtual_hosts:
55+
- name: yorkie_service
56+
domains: ["*"]
57+
routes:
58+
- match: { prefix: "/" }
59+
route:
60+
cluster: admin_service
61+
# https://github.com/grpc/grpc-web/issues/361
62+
max_stream_duration:
63+
grpc_timeout_header_max: 0s
64+
cors:
65+
allow_origin_string_match:
66+
- prefix: "*"
67+
allow_methods: GET, PUT, DELETE, POST, OPTIONS
68+
allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,custom-header-1,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout,authorization,x-api-key
69+
max_age: "1728000"
70+
expose_headers: custom-header-1,grpc-status,grpc-message, grpc-status-details-bin
71+
http_filters:
72+
- name: envoy.filters.http.grpc_web
73+
- name: envoy.filters.http.cors
74+
- name: envoy.filters.http.router
75+
clusters:
76+
- name: yorkie_service
77+
connect_timeout: 0.25s
78+
type: strict_dns
79+
http2_protocol_options: {}
80+
lb_policy: ring_hash
81+
ring_hash_lb_config:
82+
minimum_ring_size: 125
83+
load_assignment:
84+
cluster_name: yorkie_cluster
85+
endpoints:
86+
- lb_endpoints:
87+
- endpoint:
88+
address:
89+
socket_address:
90+
address: host.docker.internal
91+
port_value: 11101
92+
- endpoint:
93+
address:
94+
socket_address:
95+
address: host.docker.internal
96+
port_value: 11201
97+
- endpoint:
98+
address:
99+
socket_address:
100+
address: host.docker.internal
101+
port_value: 11301
102+
- name: admin_service
103+
connect_timeout: 0.25s
104+
type: strict_dns
105+
http2_protocol_options: {}
106+
lb_policy: ring_hash
107+
ring_hash_lb_config:
108+
minimum_ring_size: 125
109+
load_assignment:
110+
cluster_name: admin_cluster
111+
endpoints:
112+
- lb_endpoints:
113+
- endpoint:
114+
address:
115+
socket_address:
116+
address: host.docker.internal
117+
port_value: 11103
118+
- endpoint:
119+
address:
120+
socket_address:
121+
address: host.docker.internal
122+
port_value: 11203
123+
- endpoint:
124+
address:
125+
socket_address:
126+
address: host.docker.internal
127+
port_value: 11303

‎docker/yorkie-shard-cluster-docker/envoy.Dockerfile

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM envoyproxy/envoy:v1.19.0
2+
3+
COPY ./envoy-ring-hash-router.yaml /etc/envoy/envoy.yaml
4+
5+
ENTRYPOINT ["/usr/local/bin/envoy", "-c"]
6+
7+
CMD /etc/envoy/envoy.yaml

‎docker/yorkie-shard-cluster-docker/mongodb-shard-docker-compose.yml

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
version: "3.7"
2+
3+
services:
4+
config:
5+
image: mongo
6+
container_name: config
7+
command: mongod --configsvr --replSet configRS --bind_ip_all
8+
ports:
9+
- 27019:27019
10+
networks:
11+
- mongo
12+
volumes:
13+
- config:/data/db
14+
15+
shard1:
16+
image: mongo
17+
container_name: shard1
18+
command: mongod --shardsvr --replSet shard1RS --bind_ip_all
19+
ports:
20+
- 27017:27017
21+
networks:
22+
- mongo
23+
volumes:
24+
- shard1:/data/db
25+
26+
shard2:
27+
image: mongo
28+
container_name: shard2
29+
command: mongod --shardsvr --replSet shard2RS --bind_ip_all
30+
ports:
31+
- 27018:27017
32+
networks:
33+
- mongo
34+
volumes:
35+
- shard2:/data/db
36+
37+
shard3:
38+
image: mongo
39+
container_name: shard3
40+
command: mongod --shardsvr --replSet shard3RS --bind_ip_all
41+
ports:
42+
- 27016:27017
43+
networks:
44+
- mongo
45+
volumes:
46+
- shard3:/data/db
47+
48+
mongos1:
49+
image: mongo
50+
container_name: mongos1
51+
command: mongos --configdb configRS/config:27019 --bind_ip_all
52+
ports:
53+
- 27020:27017
54+
networks:
55+
- mongo
56+
57+
mongos2:
58+
image: mongo
59+
container_name: mongos2
60+
command: mongos --configdb configRS/config:27019 --bind_ip_all
61+
ports:
62+
- 27021:27017
63+
networks:
64+
- mongo
65+
66+
volumes:
67+
config:
68+
shard1:
69+
shard2:
70+
shard3:
71+
72+
networks:
73+
mongo:

0 commit comments

Comments
 (0)