Expose P99 latency

Expose 99th percentile. It means that 99% of the requests should be faster than given latency. Longhorn 8254 Signed-off-by: Derek Su <[email protected]>
longhorn · Apr 8, 2024 · 305568e · 305568e
1 parent eaf910f
commit 305568e
Show file tree

Hide file tree

Showing 5 changed files with 108 additions and 63 deletions.
diff --git a/fio/cmp_parse.sh b/fio/cmp_parse.sh
@@ -17,6 +17,8 @@ if [ -z "${2}" ]; then
 fi
 METRICS="${2}"
 
+P99_LATENCY="${3:-false}"
+
 if [ -z "FIRST_VOL_NAME" ]; then
     echo Require the first volume name
     exit 1
@@ -49,6 +51,9 @@ parse_metrics() {
         for IO_TYPE in "${io_types_array[@]}"; do
             local output="${vol_name}-${IO_TYPE}-${METRIC}.json"
             local parse_func="parse_${IO_TYPE}_${METRIC}"
+            if [ "$P99_LATENCY" = "true" ]; then
+                parse_func="parse_${IO_TYPE}_${METRIC}_p99"
+            fi
 
             if declare -f "$parse_func" > /dev/null; then
                 $parse_func "$output"
@@ -80,8 +85,6 @@ calc_cmp_bandwidth
 calc_cmp_latency
 
 # Build the summary with header information
-RESULT=${FIRST_VOL_NAME}_vs_${SECOND_VOL_NAME}.summary
-
 QUICK_MODE_TEXT="Quick Mode: disabled"
 if [ -n "$QUICK_MODE" ]; then
     QUICK_MODE_TEXT="Quick Mode: enabled"
@@ -92,22 +95,31 @@ if [ -n "$SIZE" ]; then
     SIZE_TEXT="Size: $SIZE"
 fi
 
+# Determine the file name suffix and title based on P99_LATENCY
+if [ "$P99_LATENCY" = "false" ]; then
+    FILE_SUFFIX="summary"
+    TITLE="FIO Benchmark Comparison Summary"
+else
+    FILE_SUFFIX="p99_latency.summary"
+    TITLE="FIO Benchmark P99 Latency Comparison Summary"
+fi
+
+RESULT="${FIRST_VOL_NAME}_vs_${SECOND_VOL_NAME}_${FILE_SUFFIX}"
+
 SUMMARY="
-================================
-FIO Benchmark Comparsion Summary
+===============================
+$TITLE
 For: $FIRST_VOL_NAME vs $SECOND_VOL_NAME
 CPU Idleness Profiling: $CPU_IDLE_PROF
 $SIZE_TEXT
 $QUICK_MODE_TEXT
-================================
+===============================
 "
 
 printf -v header "$CMP_FMT" \
     "" $FIRST_VOL_NAME "vs" $SECOND_VOL_NAME ":" "Change"
 SUMMARY+=$header
 
-#!/bin/bash
-
 # Define a function to add metrics to the summary
 add_metrics_to_summary() {
     local metric_name="${1}"
@@ -139,7 +151,7 @@ add_metrics_to_summary() {
     local cmp_seqwrite="${25}"
     local cmp_cpu_idle_pct_seqwrite="${26}"
 
-    if [ "$CPU_IDLE_PROF" = "enabled" ]; then
+    if [ "$CPU_IDLE_PROF" = "enabled" ] && [ "$P99_LATENCY" = "false" ]; then
         printf -v cxt "${metric_name} in ${metric_unit} with CPU idleness in percent (Read/Write)\n${CMP_FMT}${CMP_FMT}${CMP_FMT}\n" \
             "Random:" \
             "$(commaize "${first_randread}") ($(commaize "${first_cpu_idle_pct_randread}")) / $(commaize "${first_randwrite}") ($(commaize "${first_cpu_idle_pct_randwrite}"))" \
@@ -174,33 +186,35 @@ if [ "x$CPU_IDLE_PROF" = "xenabled" ]; then
 fi
 
 # Example usage
-add_metrics_to_summary "IOPS" "ops" \
-    "$FIRST_RANDREAD_IOPS" "$FIRST_CPU_IDLE_PCT_RANDREAD_IOPS" \
-    "$FIRST_RANDWRITE_IOPS" "$FIRST_CPU_IDLE_PCT_RANDWRITE_IOPS" \
-    "$SECOND_RANDREAD_IOPS" "$SECOND_CPU_IDLE_PCT_RANDREAD_IOPS" \
-    "$SECOND_RANDWRITE_IOPS" "$SECOND_CPU_IDLE_PCT_RANDWRITE_IOPS" \
-    "$CMP_RANDREAD_IOPS" "$CMP_CPU_IDLE_PCT_RANDREAD_IOPS" \
-    "$CMP_RANDWRITE_IOPS" "$CMP_CPU_IDLE_PCT_RANDWRITE_IOPS" \
-    "$FIRST_SEQREAD_IOPS" "$FIRST_CPU_IDLE_PCT_SEQREAD_IOPS" \
-    "$FIRST_SEQWRITE_IOPS" "$FIRST_CPU_IDLE_PCT_SEQWRITE_IOPS" \
-    "$SECOND_SEQREAD_IOPS" "$SECOND_CPU_IDLE_PCT_SEQREAD_IOPS" \
-    "$SECOND_SEQWRITE_IOPS" "$SECOND_CPU_IDLE_PCT_SEQWRITE_IOPS" \
-    "$CMP_SEQREAD_IOPS" "$CMP_CPU_IDLE_PCT_SEQREAD_IOPS" \
-    "$CMP_SEQWRITE_IOPS" "$CMP_CPU_IDLE_PCT_SEQWRITE_IOPS"
-
-add_metrics_to_summary "Bandwidth" "KiB/sec" \
-    "$FIRST_RANDREAD_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
-    "$FIRST_RANDWRITE_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
-    "$SECOND_RANDREAD_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
-    "$SECOND_RANDWRITE_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
-    "$CMP_RANDREAD_BANDWIDTH" "$CMP_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
-    "$CMP_RANDWRITE_BANDWIDTH" "$CMP_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
-    "$FIRST_SEQREAD_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
-    "$FIRST_SEQWRITE_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" \
-    "$SECOND_SEQREAD_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
-    "$SECOND_SEQWRITE_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" \
-    "$CMP_SEQREAD_BANDWIDTH" "$CMP_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
-    "$CMP_SEQWRITE_BANDWIDTH" "$CMP_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH"
+if [ "$P99_LATENCY" = "false" ]; then
+    add_metrics_to_summary "IOPS" "ops" \
+        "$FIRST_RANDREAD_IOPS" "$FIRST_CPU_IDLE_PCT_RANDREAD_IOPS" \
+        "$FIRST_RANDWRITE_IOPS" "$FIRST_CPU_IDLE_PCT_RANDWRITE_IOPS" \
+        "$SECOND_RANDREAD_IOPS" "$SECOND_CPU_IDLE_PCT_RANDREAD_IOPS" \
+        "$SECOND_RANDWRITE_IOPS" "$SECOND_CPU_IDLE_PCT_RANDWRITE_IOPS" \
+        "$CMP_RANDREAD_IOPS" "$CMP_CPU_IDLE_PCT_RANDREAD_IOPS" \
+        "$CMP_RANDWRITE_IOPS" "$CMP_CPU_IDLE_PCT_RANDWRITE_IOPS" \
+        "$FIRST_SEQREAD_IOPS" "$FIRST_CPU_IDLE_PCT_SEQREAD_IOPS" \
+        "$FIRST_SEQWRITE_IOPS" "$FIRST_CPU_IDLE_PCT_SEQWRITE_IOPS" \
+        "$SECOND_SEQREAD_IOPS" "$SECOND_CPU_IDLE_PCT_SEQREAD_IOPS" \
+        "$SECOND_SEQWRITE_IOPS" "$SECOND_CPU_IDLE_PCT_SEQWRITE_IOPS" \
+        "$CMP_SEQREAD_IOPS" "$CMP_CPU_IDLE_PCT_SEQREAD_IOPS" \
+        "$CMP_SEQWRITE_IOPS" "$CMP_CPU_IDLE_PCT_SEQWRITE_IOPS"
+
+    add_metrics_to_summary "Bandwidth" "KiB/sec" \
+        "$FIRST_RANDREAD_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
+        "$FIRST_RANDWRITE_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
+        "$SECOND_RANDREAD_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
+        "$SECOND_RANDWRITE_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
+        "$CMP_RANDREAD_BANDWIDTH" "$CMP_CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
+        "$CMP_RANDWRITE_BANDWIDTH" "$CMP_CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
+        "$FIRST_SEQREAD_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
+        "$FIRST_SEQWRITE_BANDWIDTH" "$FIRST_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" \
+        "$SECOND_SEQREAD_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
+        "$SECOND_SEQWRITE_BANDWIDTH" "$SECOND_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH" \
+        "$CMP_SEQREAD_BANDWIDTH" "$CMP_CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
+        "$CMP_SEQWRITE_BANDWIDTH" "$CMP_CPU_IDLE_PCT_SEQWRITE_BANDWIDTH"
+fi
 
 add_metrics_to_summary "Latency" "ns" \
     "$FIRST_RANDREAD_LATENCY" "$FIRST_CPU_IDLE_PCT_RANDREAD_LATENCY" \
@@ -218,6 +232,4 @@ add_metrics_to_summary "Latency" "ns" \
 
 
 echo "$SUMMARY" > $RESULT
-cat $RESULT
-
-
+cat $RESULT
diff --git a/fio/cmp_run.sh b/fio/cmp_run.sh
@@ -25,10 +25,8 @@ if [ -z "SECOND_VOL_FILE" ]; then
     exit 1
 fi
 
-#disable parsing in run.sh
-export SKIP_PARSE=1
-
 $CURRENT_DIR/run.sh $FIRST_VOL_FILE $FIRST_VOL_NAME
 $CURRENT_DIR/run.sh $SECOND_VOL_FILE $SECOND_VOL_NAME
 
-$CURRENT_DIR/cmp_parse.sh "$IO_TYPES" "$METRICS"
+$CURRENT_DIR/cmp_parse.sh "$IO_TYPES" "$METRICS" "false"
+$CURRENT_DIR/cmp_parse.sh "$IO_TYPES" "latency" "true"
diff --git a/fio/func.sh b/fio/func.sh
@@ -75,8 +75,29 @@ parse_randwrite_latency() {
     CPU_IDLE_PCT_RANDWRITE_LATENCY=`cat $OUTPUT | jq '.cpu_idleness.system' | cut -f1 -d.`
 }
 
+# Latency 99th percentile
+parse_seqread_latency_p99() {
+    local OUTPUT=${1}
+    SEQREAD_LATENCY=`cat $OUTPUT | jq '.jobs[0].read.clat_ns.percentile["99.000000"]'| cut -f1 -d.`
+}
+
+parse_seqwrite_latency_p99() {
+    local OUTPUT=${1}
+    SEQWRITE_LATENCY=`cat $OUTPUT | jq '.jobs[0].write.clat_ns.percentile["99.000000"]'| cut -f1 -d.`
+}
+
+parse_randread_latency_p99() {
+    local OUTPUT=${1}
+    RANDREAD_LATENCY=`cat $OUTPUT | jq '.jobs[0].read.clat_ns.percentile["99.000000"]'| cut -f1 -d.`
+}
+
+parse_randwrite_latency_p99() {
+    local OUTPUT=${1}
+    RANDWRITE_LATENCY=`cat $OUTPUT | jq '.jobs[0].write.clat_ns.percentile["99.000000"]'| cut -f1 -d.`
+}
+
 
-FMT="%25s%25s\n"
+FMT="%30s%30s\n"
 CMP_FMT="%20s%30s%10s%30s%10s%25s\n"
 
 commaize() {

diff --git a/fio/parse.sh b/fio/parse.sh
@@ -17,7 +17,7 @@ append_metric() {
     local seqwrite="${9}"
     local cpu_idle_pct_seqwrite="${10}"
 
-    if [ "$CPU_IDLE_PROF" = "enabled" ]; then
+    if [ "$CPU_IDLE_PROF" = "enabled" ] && [ "$P99_LATENCY" = "false" ]; then
         # If CPU idle profiling is enabled, include it in the output
         printf -v cxt "%s in %s with CPU idleness in percent (Read/Write)\n$FMT$FMT\n" \
             "$metric_name" "$metric_unit" \
@@ -56,42 +56,57 @@ if [ -z "${3}" ]; then
 fi
 PREFIX="${3}"
 
+P99_LATENCY="${4:-true}"
 
 IFS=',' read -r -a io_types_array <<< "${IO_TYPES}"
 IFS=',' read -r -a metrics_array <<< "${METRICS}"
 
 for TYPE in "${io_types_array[@]}"; do
     for METRIC in "${metrics_array[@]}"; do
         OUTPUT="${PREFIX}-${TYPE}-${METRIC}.json"
-        parse_${TYPE}_${METRIC} "$OUTPUT"
+        if [ "$P99_LATENCY" = "true" ]; then
+            parse_${TYPE}_${METRIC}_p99 "$OUTPUT"
+        else
+            parse_${TYPE}_${METRIC} "$OUTPUT"
+        fi
     done
 done
 
 # Initialize the result file name
-RESULT=${PREFIX}.summary
-
 # Build the summary with header information
+if [ "$P99_LATENCY" = "false" ]; then
+    RESULT="${PREFIX}.summary"
+    TITLE="FIO Benchmark Summary"
+else
+    RESULT="${PREFIX}_p99_latency.summary"
+    TITLE="FIO Benchmark P99 Latency Summary"
+fi
+
+# Construct the SUMMARY with dynamic content
 SUMMARY="
-=========================
-FIO Benchmark Summary
+==================================
+$TITLE
 For: $PREFIX
 CPU Idleness Profiling: ${CPU_IDLE_PROF:-not provided}
 Size: ${SIZE:-10g}
 Quick Mode: ${QUICK_MODE:-disabled}
-=========================
+==================================
 "
 
 # Append performance metrics to the summary
-append_metric "IOPS" "ops" \
-    "$RANDREAD_IOPS" "$CPU_IDLE_PCT_RANDREAD_IOPS" \
-    "$RANDWRITE_IOPS" "$CPU_IDLE_PCT_RANDWRITE_IOPS" \
-    "$SEQREAD_IOPS" "$CPU_IDLE_PCT_SEQREAD_IOPS" \
-    "$SEQWRITE_IOPS" "$CPU_IDLE_PCT_SEQWRITE_IOPS"
-append_metric "Bandwidth" "KiB/sec" \
-    "$RANDREAD_BANDWIDTH" "$CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
-    "$RANDWRITE_BANDWIDTH" "$CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
-    "$SEQREAD_BANDWIDTH" "$CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
-    "$SEQWRITE_BANDWIDTH" "$CPU_IDLE_PCT_SEQWRITE_BANDWIDTH"
+if [ "$P99_LATENCY" = "false" ]; then
+    append_metric "IOPS" "ops" \
+        "$RANDREAD_IOPS" "$CPU_IDLE_PCT_RANDREAD_IOPS" \
+        "$RANDWRITE_IOPS" "$CPU_IDLE_PCT_RANDWRITE_IOPS" \
+        "$SEQREAD_IOPS" "$CPU_IDLE_PCT_SEQREAD_IOPS" \
+        "$SEQWRITE_IOPS" "$CPU_IDLE_PCT_SEQWRITE_IOPS"
+    append_metric "Bandwidth" "KiB/sec" \
+        "$RANDREAD_BANDWIDTH" "$CPU_IDLE_PCT_RANDREAD_BANDWIDTH" \
+        "$RANDWRITE_BANDWIDTH" "$CPU_IDLE_PCT_RANDWRITE_BANDWIDTH" \
+        "$SEQREAD_BANDWIDTH" "$CPU_IDLE_PCT_SEQREAD_BANDWIDTH" \
+        "$SEQWRITE_BANDWIDTH" "$CPU_IDLE_PCT_SEQWRITE_BANDWIDTH"
+fi
+
 append_metric "Latency" "ns" \
     "$RANDREAD_LATENCY" "$CPU_IDLE_PCT_RANDREAD_LATENCY" \
     "$RANDWRITE_LATENCY" "$CPU_IDLE_PCT_RANDWRITE_LATENCY" \

diff --git a/fio/run.sh b/fio/run.sh
@@ -42,7 +42,6 @@ for TYPE in "${io_types_array[@]}"; do
     done
 done
 
-if [ -z "$SKIP_PARSE" ]; then
-    "$CURRENT_DIR/parse.sh" "$IO_TYPES" "$METRICS" "$OUTPUT" 
-fi
+"$CURRENT_DIR/parse.sh" "$IO_TYPES" "$METRICS" "$OUTPUT" "false"
+"$CURRENT_DIR/parse.sh" "$IO_TYPES" "latency" "$OUTPUT" "true"