From 4a1da9be7fdff51ecc8b24e7c5331883d895937f Mon Sep 17 00:00:00 2001 From: Yujia Qiao Date: Tue, 28 Dec 2021 20:04:54 +0800 Subject: [PATCH] docs: update benchmark and add about memory usage --- README.md | 9 ++-- benches/scripts/http/latency.sh | 24 ++++++++++ benches/scripts/mem/mem.sh | 46 ++++++++++++++++++++ benches/scripts/mem/plot.plt | 18 ++++++++ docs/benchmark.md | 75 +++++++++++++++++++++++++------- docs/img/http_throughput.svg | 1 + docs/img/mem-graph.png | Bin 0 -> 4661 bytes docs/img/tcp_bitrate.svg | 2 +- docs/img/tcp_latency.svg | 1 - docs/img/udp_bitrate.svg | 1 + 10 files changed, 156 insertions(+), 21 deletions(-) create mode 100644 benches/scripts/http/latency.sh create mode 100644 benches/scripts/mem/mem.sh create mode 100644 benches/scripts/mem/plot.plt create mode 100644 docs/img/http_throughput.svg create mode 100644 docs/img/mem-graph.png delete mode 100644 docs/img/tcp_latency.svg create mode 100644 docs/img/udp_bitrate.svg diff --git a/README.md b/README.md index d6be54b..32a0daf 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # rathole ![rathole-logo](./docs/img/rathole-logo.png) -A fast, secure and stable reverse proxy for NAT traversal, written in Rust +A secure, stable and high-performance reverse proxy for NAT traversal, written in Rust rathole, like [frp](https://github.com/fatedier/frp) and [ngrok](https://github.com/inconshreveable/ngrok), can help to expose the service on the device behind the NAT to the Internet, via a server with a public IP. @@ -131,13 +131,14 @@ If `RUST_LOG` is not present, the default logging level is `info`. ## Benchmark -rathole has similiar latency to [frp](https://github.com/fatedier/frp), but can handle more connections. Also it can provide much better bandwidth than frp. +rathole has similiar latency to [frp](https://github.com/fatedier/frp), but can handle a more connections, provide larger bandwidth, with less memory usage. See also [Benchmark](./docs/benchmark.md). +![http_throughput](./docs/img/http_throughput.svg) ![tcp_bitrate](./docs/img/tcp_bitrate.svg) - -![tcp_latency](./docs/img/tcp_latency.svg) +![udp_bitrate](./docs/img/udp_bitrate.svg) +![mem](./docs/img/mem-graph.png) ## Development Status diff --git a/benches/scripts/http/latency.sh b/benches/scripts/http/latency.sh new file mode 100644 index 0000000..d7e43c3 --- /dev/null +++ b/benches/scripts/http/latency.sh @@ -0,0 +1,24 @@ +#!/bin/sh +RATE="1 1000 2000 3000 4000" +DURATION="60s" + +RATHOLE="http://127.0.0.1:5202" +FRP="http://127.0.0.1:5203" + +echo warming up frp +echo GET $FRP | vegeta attack -duration 10s > /dev/null +for rate in $RATE; do + name="frp-${rate}qps-$DURATION.bin" + echo $name + echo GET $FRP | vegeta attack -rate $rate -duration $DURATION > $name + vegeta report $name +done + +echo warming up rathole +echo GET $RATHOLE | vegeta attack -duration 10s > /dev/null +for rate in $RATE; do + name="rathole-${rate}qps-$DURATION.bin" + echo $name + echo GET $RATHOLE | vegeta attack -rate $rate -duration $DURATION > $name + vegeta report $name +done diff --git a/benches/scripts/mem/mem.sh b/benches/scripts/mem/mem.sh new file mode 100644 index 0000000..2c1218e --- /dev/null +++ b/benches/scripts/mem/mem.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +rm -v *-mem.log + +echo frp +while true; do + ps -C frpc -o rsz= >> frpc-mem.log +sleep 1 +done & + +while true; do + ps -C frps -o rsz= >> frps-mem.log +sleep 1 +done & + +echo GET http://127.0.0.1:5203 | vegeta attack -duration 30s -rate 1000 > /dev/null + +sleep 10 + +kill $(jobs -p) + + +echo rathole + +pid_s=$(ps aux | grep "rathole -s" | head -n 1 | awk '{print $2}') +while true; do + ps --pid $pid_s -o rsz= >> ratholec-mem.log +sleep 1 +done & + +pid_c=$(ps aux | grep "rathole -c" | head -n 1 | awk '{print $2}') +while true; do + ps --pid $pid_c -o rsz= >> ratholes-mem.log +sleep 1 +done & + +echo GET http://127.0.0.1:5202 | vegeta attack -duration 30s -rate 1000 > /dev/null + +sleep 10 + +kill $(jobs -p) + +gawk -i inplace '{print $1 "000"}' frpc-mem.log +gawk -i inplace '{print $1 "000"}' frps-mem.log +gawk -i inplace '{print $1 "000"}' ratholec-mem.log +gawk -i inplace '{print $1 "000"}' ratholes-mem.log diff --git a/benches/scripts/mem/plot.plt b/benches/scripts/mem/plot.plt new file mode 100644 index 0000000..ee4a548 --- /dev/null +++ b/benches/scripts/mem/plot.plt @@ -0,0 +1,18 @@ +set title "Memory Usage" font ",20" + +set term png small size 800,600 +set key box outside + +set output "mem-graph.png" + +set ylabel "RSZ" +set format y '%.0s%cB' + +set ytics nomirror + +set yrange [0:*] + +plot "frps-mem.log" using 1 with lines axes x1y1 title "frps RSZ", \ + "frpc-mem.log" using 1 with lines axes x1y1 title "frpc RSZ", \ + "ratholes-mem.log" using 1 with lines axes x1y1 title "ratholes RSZ", \ + "ratholec-mem.log" using 1 with lines axes x1y1 title "ratholec RSZ" diff --git a/docs/benchmark.md b/docs/benchmark.md index 71816a1..4a62089 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -1,17 +1,19 @@ # Benchmark -> Date: 2021/12/14 +> Date: 2021/12/28 +> +> Version: commit 1180c7e538564efd69742f22e77453a1b74a5ed2 > -> Arch Linux with 5.15.7-arch1-1 kernel +> Arch Linux with 5.15.11-arch2-1 kernel > -> Intel i7-6600U CPU @ 2.60GHz +> Intel Xeon CPU E5-2620 @ 2.00GHz *2 > -> 20GB RAM +> 16GB RAM - -## Bitrate +## Bandwidth ![tcp_bitrate](./img/tcp_bitrate.svg) +![udp_bitrate](./img/udp_bitrate.svg) rathole with the following configuration: ```toml @@ -19,14 +21,20 @@ rathole with the following configuration: remote_addr = "localhost:2333" default_token = "123" -[client.services.foo1] +[client.services.bench-tcp] +local_addr = "127.0.0.1:80" +[client.services.bench-udp] +type = "udp" local_addr = "127.0.0.1:80" [server] bind_addr = "0.0.0.0:2333" default_token = "123" -[server.services.foo1] +[server.services.bench-tcp] +bind_addr = "0.0.0.0:5202" +[server.services.bench-udp] +type = "udp" bind_addr = "0.0.0.0:5202" ``` @@ -41,16 +49,20 @@ token = 1233 # frpc.ini [common] server_addr = 127.0.0.1 -#server_addr = 47.100.208.60 server_port = 7000 authentication_method = token token = 1233 -[ssh] +[bench-tcp] type = tcp local_ip = 127.0.0.1 local_port = 80 remote_port = 5203 +[bench-udp] +type = udp +local_ip = 127.0.0.1 +local_port = 80 +remote_port = 5203 ``` ``` @@ -71,17 +83,50 @@ For frp benchmark: $ iperf3 -c 127.0.0.1 -p 5203 ``` -## Latency +## HTTP nginx/1.20.2 listens on port 80, with the default test page. frp and rathole configuration is same with the previous section. -Using [ali](https://github.com/nakabonne/ali) with different rate. +[vegeta](https://github.com/tsenart/vegeta) is used to generate HTTP load. + +### HTTP Throughput + +The following commands are used to benchmark rathole and frp. Note that if you want to do a benchmark yourself, `-max-workers` should be adjusted to get the accurate results for your machine. -e.g. for rathole 10 QPS benchmark: ``` -ali -r 10 http://127.0.0.1:5202 +echo 'GET http://127.0.0.1:5203' | vegeta attack -rate 0 -duration 30s -max-workers 48 +echo 'GET http://127.0.0.1:5202' | vegeta attack -rate 0 -duration 30s -max-workers 48 ``` -![tcp_latency](./img/tcp_latency.svg) +![http_throughput](./img/http_throughput.svg) + +### HTTP Latency + +`rathole` has very similar latency to `frp`, but can handle more connections + +Here's a table, latency is in ms + +|QPS|latency(rathole)|latency(frp)| +|--|--|---| +|1|2.113|2.55| +|1000|1.723|1.742| +|2000|1.845|1.749| +|3000|2.064|2.011| +|4000|2.569|7907| + +As you can see, for QPS from 1 to 3000, rathole and frp have nearly identical latency. +But with QPS of 4000, frp starts reporting lots of errors and the latency grows to even seconds. This kind of reflects the throughput in the previous section. + +Thus, in terms of latency, rathole and frp are nearly the same. But rathole can handle more connections. + +[Script to benchmark latency](../benches/scripts/http/latency.sh) + +## Memory Usage + +![mem](./img/mem-graph.png) + +rathole uses much less memory than frp. + +[Script to benchmark memory](../benches/scripts/mem/mem.sh) diff --git a/docs/img/http_throughput.svg b/docs/img/http_throughput.svg new file mode 100644 index 0000000..02f7c18 --- /dev/null +++ b/docs/img/http_throughput.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/img/mem-graph.png b/docs/img/mem-graph.png new file mode 100644 index 0000000000000000000000000000000000000000..c355803617ff2b9b7009c027f38039152f9f7ae3 GIT binary patch literal 4661 zcmeHLYgAKL7QQj;@3Z&W`)2rfxov(+|1AIjHhZ{#@)-cY(EzZ)|Icvf%e^0B4gdh`FFr>PWo+2)7b}CHa&knTt-HNw*f_};(I_KE)!FHa9^CokhcNkZQIB=sF2AaY;p)l zUgwVk%b*ehmPLbXHVCZ;WcM_c&Vh4fjXiB(TMxS?5yX{&=+@R&pfuL^zD!C329FK6 zK}+2j?H&*d0LYH@3zlz&Gys5i>O4N#|M|t@XX2Ct`hFV@he{_G?HjdgdbQba?=5@x z$CPH{V$9D^Iz}Z6!y7*3L>8EM#JM-&AQB=RT1jtXe3$OOLF4^h4+ufR4bQM*Jr zs%@`xWVQ^kTu>>RXF0~Y@i=v6Mf(=}RvT7}s|kY(k69^|36@MSQ_+SMvT?F`@&!%l z{N_51&f67B^qA@z%Ys{-Fr?@RM#&K~PxfG*t-$PuE$m5rO%QiH(3JSs0^ysq>K#|3 zN+kPO%=S{|Z8H^N$tg;K@k5><{6aQkY;IXDIHJe#>OGp(RlIm9Avaba>l4x4qCXHZ zoX}ZR|6}SmxcXS#w4^wPE0M%5YLf??zBw@%UvT;O;CvMG0&aMdLHwWriqcshc>#Br zQ4^ZpB8F+C0_LM?nLKE)roGV)EmWBE`npyE^sI+gOL?<{>oWrUT=(8B<-MtuxoXy2ch1 zj86Wt=%q~NG-Si7EnZQRJ?e@5t;`G? z$@RctmYsMXDD_I9Nqdf@vV0yq$(;D|Us6MuG~a~h9#=hsX;#Bn>%?NaU^0lY>lC6K zVP=Z&MRM*xc<;KGBvzBZFa!yNC3i_$keGWLO>nEr!RKw6Q(bouX40@EjejM}#a}gk zMI@V&1e`;dq-oXUg^2mk>0zwll;`U^B+JnMoF8sW8Yyo0CTxzM; z(Zf3txx%H{YaDvP1~G4V@+LQO6sLg{QM@BS{pQb@=!zu;+ z5tcz>Yv`mh0Mk_;Rc~ZKMeXBXiKs(5t-+iTbcFyFumOfMD z;yQU+9kK`Cn3(&py$R`FdS-&Ri2VVQliGBknr8Q*=I(d!FI`kJO8i%`;`A67o%^Wr8>Ryy5nyX zk5x>#z?h+Xq~orlw8r8HzYz|n&@-LoNkVeFg6T?B0x{?jr|?-dMk>Fm`4VzGHju7v z)!Lzs>`W@cQvZE7 zzt-CCxv42_)>LHmB1XXKP5zKrS!+#|#;Q~UAER;}M`Bmh%x`X34;g1`vURUavO-LU z*D*61n#Nk0Dc`{zUD1rH{pv)>Zdq5R$)^sc;o~3XwaUq-;A!{)$P6w1FRbg)&_F5s*(YptltRH*XS3*u>Ei>j%K z;l|T7lSEIb4;b*7xa0AMj~>r1r*9cqZ0)|4ogV|0&ljj{; z&1rGxrzeTq_u%7Kw{D{6(C?R4)K4i;M97|}3sK0-nj0yF&QDE9%Bz1pkt)c5L6^uDsMhu3+g9i`l0vvQ_HT=6l7k4y@!dkeO zo%xA@MBhdd z~~vS`@GV z)jxcl&}PAdcfba}mdgE@h`^=b`DS05^DHc2O()#ZI8c{Q61F`SqUl0bOBs?Qdb29d z{-6=%sjeqq@pUmvHN-VglM{zOKd8r5Yh zbmqCDoWG>!_s~9RxqLEVN2Ax3v&c(qI?-=cNleWDC5z`TkWxG|Ye$vK(BBIH@Hpi4 KN!5Xn)PDjCB$f{V literal 0 HcmV?d00001 diff --git a/docs/img/tcp_bitrate.svg b/docs/img/tcp_bitrate.svg index 6a3065a..e7f8374 100644 --- a/docs/img/tcp_bitrate.svg +++ b/docs/img/tcp_bitrate.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/docs/img/tcp_latency.svg b/docs/img/tcp_latency.svg deleted file mode 100644 index 172e802..0000000 --- a/docs/img/tcp_latency.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/img/udp_bitrate.svg b/docs/img/udp_bitrate.svg new file mode 100644 index 0000000..891f4a6 --- /dev/null +++ b/docs/img/udp_bitrate.svg @@ -0,0 +1 @@ + \ No newline at end of file