Skip to content

Commit f83591b

Browse files
author
yangxingxiang
committed
3FS Docker Deployment
1 parent be57462 commit f83591b

14 files changed

+925
-0
lines changed

‎build.sh‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Default build config
5+
OS_TYPE="ubuntu2204"
6+
7+
# Parse arguments
8+
if [[ "$1" == "-h" || "$1" == "--help" ]]; then
9+
cat <<EOF
10+
HF3FS Build System
11+
12+
Usage: $0 [OPTION]
13+
14+
Options:
15+
docker-ubuntu2204 Build using Ubuntu 22.04 Docker container (default)
16+
docker-ubuntu2004 Build using Ubuntu 20.04 Docker container
17+
docker-centos9 Build using CentOS 9 Docker container
18+
-h, --help Show this help message
19+
20+
Environment:
21+
- Docker builds create isolated environments with version-specific toolchains
22+
- Build artifacts are stored in: build/
23+
24+
Examples:
25+
./build.sh # Default Docker build with Ubuntu 22.04
26+
./build.sh docker-ubuntu2004 # Docker build with Ubuntu 20.04
27+
28+
EOF
29+
exit 0
30+
31+
elif [[ "$1" == "docker-ubuntu2204" ]]; then
32+
OS_TYPE="ubuntu2204"
33+
elif [[ "$1" == "docker-ubuntu2004" ]]; then
34+
OS_TYPE="ubuntu2004"
35+
elif [[ "$1" == "docker-centos9" ]]; then
36+
OS_TYPE="centos9"
37+
elif [[ -n "$1" ]]; then
38+
echo "Error: Invalid option '$1'"
39+
echo "Try './build.sh --help' for usage information"
40+
exit 1
41+
fi
42+
43+
# Common build parameters
44+
CPU_CORES=$(nproc)
45+
CMAKE_FLAGS=(
46+
-DCMAKE_CXX_COMPILER=clang++-14
47+
-DCMAKE_C_COMPILER=clang-14
48+
-DCMAKE_BUILD_TYPE=RelWithDebInfo
49+
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON
50+
)
51+
52+
docker_build() {
53+
echo "Starting Docker build for ${OS_TYPE}..."
54+
DOCKER_IMAGE="${OS_TYPE}-3fs-builder"
55+
docker build -t "${DOCKER_IMAGE}" -f "dockerfile/dev.${OS_TYPE}.dockerfile" . && \
56+
docker run --rm \
57+
-v "${PWD}:/build/src" \
58+
--cpus="${CPU_CORES}" \
59+
-e BUILD_JOBS="${CPU_CORES}" \
60+
"${DOCKER_IMAGE}" /bin/bash -c "
61+
set -ex
62+
cd /build/src
63+
cmake -S . -B build_dir ${CMAKE_FLAGS[*]}
64+
cmake --build build_dir -j\${BUILD_JOBS}
65+
"
66+
}
67+
68+
69+
# Execute build
70+
docker_build || echo "Docker build failed"

‎deploy/deploy_3fs_using_docker.md‎

Lines changed: 269 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,269 @@
1+
# 3FS Docker Deployment Guide
2+
3+
## Table of Contents
4+
1. [Prerequisites](#prerequisites)
5+
2. [Build Docker Images](#build-docker-images)
6+
3. [Deployment Steps](#deployment-steps)
7+
- [ClickHouse Service](#clickhouse-service)
8+
- [FoundationDB Service](#foundationdb-service)
9+
- [Monitor Service](#monitor-service)
10+
- [Management Service](#management-service)
11+
- [Metadata Service](#metadata-service)
12+
- [Storage Services](#storage-services)
13+
4. [Post-Deployment Configuration](#post-deployment-configuration)
14+
5. [FUSE Client Setup](#fuse-client-setup)
15+
16+
## Prerequisites <a name="prerequisites"></a>
17+
18+
### Hardware Requirements
19+
| Node | OS | IP Address | Memory | Storage | Networking | RDMA port|
20+
|-----------|--------------|--------------|---------|------------------|------------|----------|
21+
| Meta | Ubuntu 20.04 | 192.168.1.1 | 128GB | - | RoCE | mlx5_2 |
22+
| Storage1 | Ubuntu 20.04 | 192.168.1.2 | 128GB | 2×7TB NVMe SSD | RoCE | mlx5_2 |
23+
| Storage2 | Ubuntu 20.04 | 192.168.1.3 | 128GB | 2×7TB NVMe SSD | RoCE | mlx5_2 |
24+
25+
> **RDMA Configuration**
26+
> 1. Assign IP addresses to RDMA NICs. Multiple RDMA NICs (InfiniBand or RoCE) are supported on each node.
27+
> 2. Check RDMA connectivity between nodes using `ib_write_bw`.
28+
29+
30+
## Build Docker Images <a name="build-docker-images"></a>
31+
32+
1. Clone repository and checkout source code:
33+
```bash
34+
git clone https://github.com/your-org/3fs.git
35+
cd 3fs
36+
git submodule update --init --recursive
37+
./patches/apply.sh
38+
```
39+
40+
2. Build base environment:
41+
```bash
42+
./build.sh docker-ubuntu2004
43+
```
44+
45+
3. Build component images:
46+
```bash
47+
docker build -t hf3fs-monitor -f dockerfile/monitor.ubuntu2004.Dockerfile .
48+
docker build -t hf3fs-mgmtd -f dockerfile/mgmtd.ubuntu2004.Dockerfile .
49+
docker build -t hf3fs-meta -f dockerfile/meta.ubuntu2004.Dockerfile .
50+
docker build -t hf3fs-storage -f dockerfile/storage.ubuntu2004.Dockerfile .
51+
docker build -t hf3fs-fuse -f dockerfile/fuse.ubuntu2004.Dockerfile .
52+
```
53+
54+
## Deployment Steps <a name="deployment-steps"></a>
55+
56+
### ClickHouse Service <a name="clickhouse-service"></a>
57+
```bash
58+
# Pull official image
59+
docker pull clickhouse/clickhouse-server:25.3.1.2703
60+
61+
# Start container
62+
docker run -d \
63+
-p19000:9000 \
64+
-e CLICKHOUSE_PASSWORD=3fs \
65+
--name clickhouse-server \
66+
--ulimit nofile=262144:262144 \
67+
clickhouse/clickhouse-server:25.3.1.2703
68+
69+
# Copy 3fs-monitor.sql to clickhouse-server docker container
70+
docker cp 3fs-monitor.sql clickhouse-server:/root
71+
72+
# Login clickhouse-server docker container and import the SQL file into ClickHouse:
73+
docker exec -it clickhouse-server /bin/bash
74+
clickhouse-client -n < /root/3fs-monitor.sql
75+
```
76+
77+
### FoundationDB Service <a name="foundationdb-service"></a>
78+
```bash
79+
# Pull official image
80+
docker pull foundationdb/foundationdb:7.3.63
81+
82+
# Start container
83+
docker run -d \
84+
--privileged \
85+
--network host \
86+
-e FDB_NETWORKING_MODE=host \
87+
--name foundationdb-server \
88+
foundationdb/foundationdb:7.3.63
89+
90+
# Configure FoundationDB
91+
docker exec -it foundationdb-server /bin/sh -c "echo 'export PUBLIC_IP=192.168.1.1' > /var/fdb/.fdbenv"
92+
docker restart foundationdb-server
93+
94+
# Initialize database
95+
docker exec foundationdb-server /usr/bin/fdbcli -C /var/fdb/fdb.cluster --exec 'configure new single ssd'
96+
97+
# Check status
98+
docker exec foundationdb-server /usr/bin/fdbcli -C /var/fdb/fdb.cluster --exec 'status'
99+
```
100+
101+
### Monitor Service <a name="monitor-service"></a>
102+
```bash
103+
docker run --name hf3fs-monitor \
104+
--privileged \
105+
--network host \
106+
-d --restart always \
107+
--env CLICKHOUSE_DB=3fs \
108+
--env CLICKHOUSE_HOST=192.168.1.1 \
109+
--env CLICKHOUSE_PASSWD=3fs \
110+
--env CLICKHOUSE_PORT=19000 \
111+
--env CLICKHOUSE_USER=default \
112+
--env DEVICE_FILTER=mlx5_2 \
113+
hf3fs-monitor:latest
114+
```
115+
116+
### Management Service <a name="management-service"></a>
117+
```bash
118+
docker run --name hf3fs-mgmtd \
119+
--privileged \
120+
--network host \
121+
-d --restart always \
122+
--env CLUSTER_ID=stage \
123+
--env FDB_CLUSTER=docker:[email protected]:4500 \
124+
--env MGMTD_SERVER_ADDRESSES=RDMA://192.168.1.1:8000 \
125+
--env MGMTD_NODE_ID=1 \
126+
--env DEVICE_FILTER=mlx5_2 \
127+
--env REMOTE_IP=192.168.1.1:10000 \
128+
hf3fs-mgmtd:latest
129+
```
130+
131+
### Metadata Service <a name="metadata-service"></a>
132+
```bash
133+
docker run --name hf3fs-meta \
134+
--privileged \
135+
-d --restart always \
136+
--network host \
137+
--env CLUSTER_ID=stage \
138+
--env FDB_CLUSTER=docker:[email protected]:4500 \
139+
--env MGMTD_SERVER_ADDRESSES=RDMA://192.168.1.1:8000 \
140+
--env META_NODE_ID=100 \
141+
--env DEVICE_FILTER=mlx5_2 \
142+
--env REMOTE_IP=192.168.1.1:10000 \
143+
hf3fs-meta:latest
144+
```
145+
146+
### Storage Services <a name="storage-services"></a>
147+
148+
#### Storage Node Preparation
149+
```bash
150+
# Format and mount SSDs
151+
mkdir -p /storage/data{0..1}
152+
for i in {0..1}; do
153+
mkfs.xfs -L data${i} -s size=4096 /dev/nvme${i}n1
154+
mount -o noatime,nodiratime -L data${i} /storage/data${i}
155+
mkdir -p /storage/data${i}/3fs
156+
done
157+
```
158+
159+
#### Storage1 Deployment
160+
```bash
161+
docker run --name hf3fs-storage \
162+
--privileged \
163+
-d --restart always \
164+
--network host \
165+
-v /storage:/storage \
166+
--env CLUSTER_ID=stage \
167+
--env FDB_CLUSTER=docker:[email protected]:4500 \
168+
--env MGMTD_SERVER_ADDRESSES=RDMA://192.168.1.1:8000 \
169+
--env STORAGE_NODE_ID=10001 \
170+
--env TARGET_PATHS='/storage/data0/3fs,/storage/data1/3fs' \
171+
--env DEVICE_FILTER=mlx5_2 \
172+
--env REMOTE_IP=192.168.1.1:10000 \
173+
hf3fs-storage:latest
174+
```
175+
176+
#### Storage2 Deployment
177+
```bash
178+
docker run --name hf3fs-storage \
179+
--privileged \
180+
-d --restart always \
181+
--network host \
182+
-v /storage:/storage \
183+
--env CLUSTER_ID=stage \
184+
--env FDB_CLUSTER=docker:[email protected]:4500 \
185+
--env MGMTD_SERVER_ADDRESSES=RDMA://192.168.1.1:8000 \
186+
--env STORAGE_NODE_ID=10002 \
187+
--env TARGET_PATHS='/storage/data0/3fs,/storage/data1/3fs' \
188+
--env DEVICE_FILTER=mlx5_2 \
189+
--env REMOTE_IP=192.168.1.1:10000 \
190+
hf3fs-storage:latest
191+
```
192+
193+
## Post-Deployment Configuration <a name="post-deployment-configuration"></a>
194+
195+
1. Generate cluster configuration:
196+
```bash
197+
pip install -r deploy/data_placement/requirements.txt
198+
python3 deploy/data_placement/src/model/data_placement.py \
199+
-ql -relax -type CR --num_nodes 2 --replication_factor 2 --min_targets_per_disk 6
200+
201+
python3 deploy/data_placement/src/setup/gen_chain_table.py \
202+
--chain_table_type CR --node_id_begin 10001 --node_id_end 10002 \
203+
--num_disks_per_node 2 --num_targets_per_disk 6 \
204+
--target_id_prefix 1 --chain_id_prefix 9 \
205+
--incidence_matrix_path output/DataPlacementModel-v_2-b_6-r_6-k_2-λ_6-lb_1-ub_0/incidence_matrix.pickle
206+
```
207+
208+
2. Transfer configuration files:
209+
```bash
210+
docker cp output/create_target_cmd.txt hf3fs-mgmtd:/opt/3fs/etc/
211+
docker cp output/generated_chains.csv hf3fs-mgmtd:/opt/3fs/etc/
212+
docker cp output/generated_chain_table.csv hf3fs-mgmtd:/opt/3fs/etc/
213+
```
214+
215+
3. Configure administrative access:
216+
217+
```bash
218+
docker exec -it hf3fs-mgmtd /bin/bash
219+
```
220+
The admin token is printed to the console, save it to /opt/3fs/etc/token.txt.
221+
```bash
222+
/opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "user-add --root --admin 0 root"
223+
```
224+
225+
4. Initialize storage targets:
226+
```bash
227+
/opt/3fs/bin/admin_cli --cfg /opt/3fs/etc/admin_cli.toml \
228+
--config.user_info.token $(<"/opt/3fs/etc/token.txt") \
229+
< /opt/3fs/etc/create_target_cmd.txt
230+
```
231+
232+
5. Upload chain configuration:
233+
```bash
234+
/opt/3fs/bin/admin_cli --cfg /opt/3fs/etc/admin_cli.toml \
235+
--config.user_info.token $(<"/opt/3fs/etc/token.txt") \
236+
"upload-chains /opt/3fs/etc/generated_chains.csv"
237+
238+
/opt/3fs/bin/admin_cli --cfg /opt/3fs/etc/admin_cli.toml \
239+
--config.user_info.token $(<"/opt/3fs/etc/token.txt") \
240+
"upload-chain-table --desc stage 1 /opt/3fs/etc/generated_chain_table.csv"
241+
```
242+
243+
6. List chains and chain tables to check if they have been correctly uploaded
244+
```bash
245+
/opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "list-chains"
246+
/opt/3fs/bin/admin_cli -cfg /opt/3fs/etc/admin_cli.toml "list-chain-tables"
247+
```
248+
249+
## FUSE Client Setup on Meta Node <a name="fuse-client-setup"></a>
250+
```bash
251+
docker run --name hf3fs-fuse \
252+
--privileged \
253+
-d --restart always \
254+
--network host \
255+
--mount type=bind,source=/3fs/stage,target=/3fs/stage,bind-propagation=shared \
256+
--env CLUSTER_ID=stage \
257+
--env FDB_CLUSTER=docker:[email protected]:4500 \
258+
--env MGMTD_SERVER_ADDRESSES=RDMA://192.168.1.1:8000 \
259+
--env REMOTE_IP=192.168.1.1:10000 \
260+
--env DEVICE_FILTER=mlx5_2 \
261+
--env TOKEN=${TOKEN} \
262+
hf3fs-fuse:latest
263+
264+
# Login hf3fs-fuse Docker container
265+
docker exec -it hf3fs-fuse /bin/bash
266+
267+
# Verify mount
268+
mount | grep '/3fs/stage'
269+
```

‎deploy/scripts/_3fs_common.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/bash
2+
CONFIG_DONE_FLAG="/opt/3fs/etc/.done"
3+
4+
function config_cluster_id() {
5+
# env: CLUSTER_ID
6+
sed -i "s/^cluster_id.*/cluster_id = \"${CLUSTER_ID:-default}\"/" /opt/3fs/etc/*
7+
}
8+
9+
10+
function config_admin_cli() {
11+
# env: FDB_CLUSTER, MGMTD_SERVER_ADDRESSES, DEVICE_FILTER, REMOTE_IP
12+
# admin_cli.toml
13+
echo ${FDB_CLUSTER} >/etc/foundationdb/fdb.cluster
14+
sed -i "s|^clusterFile.*|clusterFile = '/etc/foundationdb/fdb.cluster'|" /opt/3fs/etc/admin_cli.toml
15+
# device_filter
16+
if [[ -n "${DEVICE_FILTER}" ]]; then
17+
sed -i "s|device_filter = \[\]|device_filter = [\"${DEVICE_FILTER//,/\",\"}\"]|g" /opt/3fs/etc/admin_cli.toml
18+
fi
19+
sed -i "s|mgmtd_server_addresses = \[\]|mgmtd_server_addresses = [\"${MGMTD_SERVER_ADDRESSES//,/\",\"}\"]|g" /opt/3fs/etc/admin_cli.toml
20+
}

0 commit comments

Comments
 (0)