From 2c039f793fabd02f0df33f0cd890bd0223987ca3 Mon Sep 17 00:00:00 2001
Message-ID: <2c039f793fabd02f0df33f0cd890bd0223987ca3.1749546774.git.edwin.torok@cloud.com>
In-Reply-To: <cover.1749546774.git.edwin.torok@cloud.com>
References: <cover.1749546774.git.edwin.torok@cloud.com>
From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= <edwin.torok@cloud.com>
Date: Tue, 10 Jun 2025 10:06:45 +0100
Subject: [PATCH 3/3] Benchmark scripts
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

These are included so that others can attempt to reproduce my benchmark
results.
They are not necesarily meant to be included in the stunnel
distribution.

They currently work on Fedora 42, but with slight tweaks could be made
to work on other distributions.

Usage for initial setup on both client and server:
```
$ source tests/benchmark/benchmark.sh
$ install_packages
$ open_ports
$ set_cpu
```

To stop most services (e.g. on a remote machine, but keep SSH):
```
$ stop_services
```

To setup the server:
```
$ STUNNEL=$HOME/src/stunnel SERVER=1 tests/benchmark/launch.sh
```

To launch the client:
```
$ STUNNEL=$HOME/src/stunnel DEST=<IP-OF-SERVER> tests/benchmark/launch.sh
```

Results will be available in tmp/, rename it (e.g. to tmp-baseline),
make a change in stunnel, then run it again.
You can comment out most `measure` lines except for
`stunnel_client_nginx`, or `stunnel_client_stunnel`, or reduce TEST_SIZE
for quicker turnaround times.

ministat from https://github.com/leahneukirchen/ministat can be used to compare before/after.

Signed-off-by: Edwin Török <edwin.torok@cloud.com>
---
 tests/benchmark/benchmark.sh | 120 +++++++++++++++++++++++
 tests/benchmark/launch.sh    | 185 +++++++++++++++++++++++++++++++++++
 2 files changed, 305 insertions(+)
 create mode 100644 tests/benchmark/benchmark.sh
 create mode 100755 tests/benchmark/launch.sh

diff --git a/tests/benchmark/benchmark.sh b/tests/benchmark/benchmark.sh
new file mode 100644
index 0000000..49d5a8a
--- /dev/null
+++ b/tests/benchmark/benchmark.sh
@@ -0,0 +1,120 @@
+#!/bin/bash
+# Set a machine into benchmarking mode.
+#
+# For native Linux, and Intel-only for now (AMD has a different way of disabling turbo)
+# 
+# References: https://llvm.org/docs/Benchmarking.html
+
+# This is for Fedora 42, adapt for your own OS
+declare -a packages=(
+  "cpupower" "tuned" "ss" "turbostat" "lscpu" "lspci" "ip" "ethtool" "jq" "nginx" "hitch" "stunnel" "iperf3" "nuttcp" "httperf" "git" "hyperfine" "varnish" "socat" "curl"
+)
+
+# httperf --server $OTHER_IP --port 8080 --num-calls 3000 --num-conns=1 --uri /hello
+
+install_packages()
+{
+  sudo dnf install "${packages[@]}"
+}
+
+stop_services()
+{
+  cat >benchmark.target <<EOF
+[Unit]
+Description=Minimal Benchmarking mode with SSH/getty
+Requires=sysinit.target getty.target network.target sshd.service
+After=sysinit.target
+AllowIsolate=Yes
+EOF
+  sudo mv benchmark.target /run/systemd/system/
+  sudo systemctl daemon-reload
+  sudo systemctl isolate benchmark.target
+  sudo systemctl list-units --state=running --no-pager
+}
+
+start_services()
+{
+  sudo systemctl isolate multi-user.target
+}
+
+set_cpu()
+{
+  # Prevent frequency-scaling
+  sudo cpupower frequency-set -g performance
+
+  # Prevent entering deep C-states to reduce latency and avoid menu governor bugs (there was one recently)
+  # Although if we completely disable C1 then we are going to hit RAPL power limits immediately,
+  # which results in a 30% perf hit on single-stream iperf
+  # Disable just deeper C states, which avoids the big perf hit (although now we're exposed to menu governor bugs)
+  sudo cpupower idle-set -D 2
+
+  # Disable Turbo Boost for more stable measurements
+  # cpupower set --turbo-boost 0 doesn't work when the driver is intel_pstate instead of cpufreq
+  sudo sh -c 'echo 1 >/sys/devices/system/cpu/intel_pstate/no_turbo'
+
+  # AMD
+  sudo cpupower set -m passive
+  sudo cpupower set --turbo-boost 0
+
+  sudo cpupower -c all set --perf-bias 0
+
+  # Bring system into a consistent state
+  sudo tuned-adm profile network-throughput
+  
+  # we could use tuned-adm verify, but there is a bug where it tries to read non-existent hung_task_timeout_secs
+  # that only fails on network-latency, not network-throughput though
+  sudo tuned-adm verify
+}
+
+describe()
+{
+  # Check all the settings we've changed above
+
+  # List running processes
+  set -x
+  sudo systemctl list-units --state=running
+  LIBPROC_HIDE_KERNEL=1 ps -ejH
+
+  sudo cpupower -c all frequency-info
+  sudo cpupower -c all idle-info
+  sudo cpupower -c all info
+  sudo cat /sys/devices/system/cpu/intel_pstate/no_turbo
+  sudo tuned-adm profile_info
+  sudo ss -neopatium
+
+  lscpu
+  sudo turbostat true
+  sudo lspci -k
+  sudo ip a
+
+  for P in /sys/class/net/*; do
+    IFACE=$(basename "${P}")
+    sudo ethtool "${IFACE}"
+    sudo ethtool -k "${IFACE}"
+  done
+
+  sudo sos report --batch
+
+  set +x
+}
+
+open_ports()
+{
+  # iperf3, nuttcp
+  for P in 5201 5000 5101 9080 9443 9444 9445 9446 10443 10444 10445 10446 20443 20444 20445 20446; do
+    sudo firewall-cmd --add-port "${P}/tcp"
+  done
+}
+
+iperf3_server()
+{
+  iperf3 -s
+}
+
+iperf3_client()
+{
+  # TODO: autodetect IP
+  iperf3 --json --logfile iperf3.json -c  10.70.58.43
+  iperf3 --json --logfile iperf3_2.json -c  10.70.58.43 -P2 
+  iperf3 --json --logfile iperf3_4.json -c  10.70.58.43 -P4
+}
diff --git a/tests/benchmark/launch.sh b/tests/benchmark/launch.sh
new file mode 100755
index 0000000..423dadb
--- /dev/null
+++ b/tests/benchmark/launch.sh
@@ -0,0 +1,185 @@
+#!/bin/sh
+set -eu
+
+STUNNEL=${STUNNEL-stunnel}
+
+# Use a single, fixed cipher for consistent tests across different tools
+TLS_CIPHERS=ECDHE-RSA-AES256-GCM-SHA384
+TLS_PROTO=TLSv1.2
+
+# Use non-root ports
+NGINX_PORT=9080
+NGINX_TLS_PORT=9443
+STUNNEL_TLS_PORT=9444
+HITCH_TLS_PORT=9445
+SOCAT_TLS_PORT=9446
+
+REPEAT=${REPEAT-30}
+TEST_SIZE=${TEST_SIZE-10GiB}
+
+PEM=stunnel.pem
+TESTDIR="${PWD}/tmp"
+RAMDIR=/dev/shm/stunnel-test
+mkdir -p "${RAMDIR}" "${TESTDIR}"
+
+cd "${TESTDIR}"
+
+test -f "${PEM}" || openssl req -new -subj "/C=GB/CN=foo.example.com" -days 10 -x509 -noenc -batch -out "${PEM}" -keyout  "${PEM}" -noenc -batch
+
+cat >stunnel.conf <<EOF
+fips=no
+socket = r:TCP_NODELAY=1
+socket = a:TCP_NODELAY=1
+socket = l:TCP_NODELAY=1
+socket = r:SO_KEEPALIVE=1
+socket = a:SO_KEEPALIVE=1
+
+[test]
+accept = :::${STUNNEL_TLS_PORT}
+cert = ${PEM}
+connect = ${NGINX_PORT}
+ciphers = ${TLS_CIPHERS}
+options = CIPHER_SERVER_PREFERENCE
+sslVersion = ${TLS_PROTO}
+EOF
+
+truncate --size "${TEST_SIZE}" "${RAMDIR}/${TEST_SIZE}"
+
+cat >nginx.conf <<EOF
+error_log stderr error;
+pid ${TESTDIR}/nginx.pid;
+daemon on;
+events { }
+http {
+    client_body_temp_path "${TESTDIR}/nginx-client_body";
+    proxy_temp_path "${TESTDIR}/nginx-proxy";
+    fastcgi_temp_path "${TESTDIR}/nginx-fcgi";
+    uwsgi_temp_path ${TESTDIR}/nginx-uwsgi";
+    scgi_temp_path ${TESTDIR}/nginx-scgi";
+    access_log off;
+    tcp_nodelay on;
+    sendfile on;
+    server {
+        gzip off;
+        listen "${NGINX_PORT}" reuseport;
+        listen "${NGINX_TLS_PORT}" reuseport ssl;
+        ssl_certificate "${PEM}";
+        ssl_certificate_key "${PEM}";
+        ssl_protocols "${TLS_PROTO}";
+        ssl_ciphers "${TLS_CIPHERS}";
+        root "${RAMDIR}";
+        location = /hello {
+            default_type text/plain;
+            return 200 "Hello";
+        }
+    }
+}
+EOF
+
+cat >hitch.conf <<EOF
+backend="[127.0.0.1]:${NGINX_PORT}"
+ciphers="${TLS_CIPHERS}"
+daemon=on
+frontend={
+    host="*"
+    port="${HITCH_TLS_PORT}"
+    pem-file="${PEM}"
+    prefer-server-ciphers=on
+    tls-protos=${TLS_PROTO}
+}
+EOF
+
+killall -v -9 stunnel nginx hitch socat || :
+
+# Start servers
+nginx -c "${TESTDIR}/nginx.conf"
+socat -b65536 "OPENSSL-LISTEN:${SOCAT_TLS_PORT},fork,reuseaddr,cert=${PEM},verify=0,cipher=${TLS_CIPHERS},compress=none,reuseport" "TCP4:127.0.0.1:${NGINX_PORT}" &
+${STUNNEL} "${TESTDIR}/stunnel.conf"
+hitch --config "${TESTDIR}/hitch.conf"
+
+DEST=${DEST:-127.0.0.1}
+
+# Start clients (they could also use a pipe, for simplicity use a local port)
+for PORT in ${NGINX_TLS_PORT} ${STUNNEL_TLS_PORT} ${HITCH_TLS_PORT} ${SOCAT_TLS_PORT}; do
+    PORT2=$(( PORT + 10000 ))
+    cat >"stunnel-client-${PORT}.conf" <<EOF
+        client=yes
+        foreground=no
+        socket=r:TCP_NODELAY=1
+        socket=r:SO_KEEPALIVE=1
+        socket=a:SO_KEEPALIVE=1
+        fips=no
+        connect=${DEST}:${PORT}
+        sslVersion=${TLS_PROTO}
+        ciphers=${TLS_CIPHERS}
+        [client-proxy]
+        accept=127.0.0.1:${PORT2}
+EOF
+    "${STUNNEL}" "${TESTDIR}/stunnel-client-${PORT}.conf"
+    PORT3=$(( PORT + 20000 ))
+    socat -b65536 "TCP4-LISTEN:${PORT3},fork,bind=127.0.0.1,reuseport" "OPENSSL:${DEST}:${PORT},cipher=${TLS_CIPHERS},verify=0"&
+done
+
+[ "${SERVER-0}" -eq 1 ] && exit 0
+
+measure()
+{
+    echo
+    echo "${4}: curl --> ${1} ${2} ${3} ..." 
+    for _ in $(seq "${REPEAT}"); do
+        curl -k -o /dev/null "${2}/${TEST_SIZE}" --write-out "%{url},%{scheme},%{http_code},%{size_download},%{speed_download},%{time_pretransfer},%{time_total}\n" --silent >>"${4}.csv"
+    done
+
+    echo "${4}: curl --> ${1} ${2} ${3}" >&2 
+    # Calculate data transfer time
+    python3 >"${4}.dat" <<EOF
+import csv
+import sys
+from statistics import median
+
+def print_speed(msg:str, amount:float, time:float) -> None:
+    speed = amount * 8 / time / 1e9
+    print(f"{msg}: {speed} Gbit/s", file=sys.stderr)
+
+with open("${4}.csv", "r") as csvfile:
+    rows = list(row for row in csv.reader(csvfile))
+    amount=None
+    for row in rows:
+        if (row[2] != '200'):
+            print(row, file=sys.stderr,flush=True)
+            sys.exit(1)
+        if amount is None:
+            amount=row[3]
+        if (row[3] != amount):
+            print("Amount transferred is different", row, file=sys.stderr, flush=True)
+            sys.exit(1)
+    times = list(float(row[6]) - float(row[5]) for row in rows)
+    for time in times:
+        print(time)
+    print_speed("max", float(amount), min(times))
+    print_speed("median", float(amount), median(times))
+    print_speed("min", float(amount), max(times))
+sys.exit(0)
+EOF
+}
+
+measure "nginx" "http://${DEST}:${NGINX_PORT}" "" "plain"
+
+NGINX_URL="http://127.0.0.1:${NGINX_PORT}"
+
+measure "nginx-ssl" "https://${DEST}:${NGINX_TLS_PORT}" "" "nginx-ssl"
+
+measure "stunnel" "https://${DEST}:${STUNNEL_TLS_PORT}" "-> nginx ${NGINX_URL}" "stunnel_server"
+measure "hitch" "https://${DEST}:${HITCH_TLS_PORT}" "-> nginx ${NGINX_URL}" "hitch_server"
+measure "socat" "https://${DEST}:${SOCAT_TLS_PORT}" "-> nginx ${NGINX_URL}" "socat_server"
+
+measure "stunnel(client)" "http://127.0.0.1:1${STUNNEL_TLS_PORT}" "-> stunnel https://${DEST}:${STUNNEL_TLS_PORT} -> nginx ${NGINX_URL}" "stunnel_client_stunnel"
+measure "stunnel(client)" "http://127.0.0.1:1${NGINX_TLS_PORT}" "-> nginx https://${DEST}:${NGINX_TLS_PORT}" "stunnel_client_nginx"
+measure "stunnel(client)" "http://127.0.0.1:1${HITCH_TLS_PORT}" "-> hitch https://${DEST}:${HITCH_TLS_PORT} -> nginx ${NGINX_URL}" "stunnel_client_hitch"
+measure "stunnel(client)" "http://127.0.0.1:1${SOCAT_TLS_PORT}" "-> socat https://${DEST}:${SOCAT_TLS_PORT} -> nginx ${NGINX_URL}" "stunnel_client_socat"
+
+measure "socat(client)" "http://127.0.0.1:2${STUNNEL_TLS_PORT}"  "-> stunnel https://${DEST}:${STUNNEL_TLS_PORT} -> nginx ${NGINX_URL}" "socat_client_stunnel"
+
+echo "DONE"
+exit 0
+
-- 
2.43.5

