From f876653d48740091393209a549036c508e386b0d Mon Sep 17 00:00:00 2001 From: malucius-rh Date: Mon, 8 Sep 2025 14:23:33 -0400 Subject: [PATCH 1/8] Remove sections where to_pbench is checked and makes things happen --- fio/fio_run | 73 +++-------------------------------------------------- 1 file changed, 4 insertions(+), 69 deletions(-) diff --git a/fio/fio_run b/fio/fio_run index d7e47ae..29a722f 100755 --- a/fio/fio_run +++ b/fio/fio_run @@ -707,11 +707,7 @@ loop_block_sizes() test_index=$3 for io_size in $bs; do - if [ $to_pbench -eq 0 ]; then - ios=`echo "${io_size}*1024" | bc` - else - ios=$io_size - fi + ios=$io_size loop_io_tests $out_dir $disk_count $test_index $io_size done } @@ -739,11 +735,7 @@ straight_fio_execute() fio_execute() { - if [ $to_pbench -eq 1 ]; then - pbench_fio_execute $1 $2 - else - straight_fio_execute $1 $2 - fi + straight_fio_execute $1 $2 } # @@ -900,53 +892,7 @@ execute_test() pushd $working_dir > /dev/null popd > /dev/null - if [ $to_pbench -eq 1 ]; then - cd /var/lib/pbench-agent/ - mkdir ${curdir}/export_fio_data - cp ${curdir}/meta_data* ${curdir}/export_fio_data - ls -d pbench-user* > /dev/null - if [ $? -ne 0 ]; then - for i in `ls -d fio_*`; do - mkdir ${curdir}/export_fio_data/$i - cp ${curdir}/meta_data*yml $i - cp $i/result.csv ${curdir}/export_fio_data/$i - cp $i/result.txt ${curdir}/export_fio_data/$i - done - else - for i in `ls -d pbench-user*`; do - mkdir ${curdir}/export_fio_data/$i - cp ${curdir}/meta_data*yml $i - cp $i/result.csv ${curdir}/export_fio_data/$i - cp $i/result.txt ${curdir}/export_fio_data/$i - done - fi - if [ $use_pbench_version -eq 1 ]; then - cd ${curdir}/export_fio_data - cnt=`ls fio*/*csv | wc -l` - if [ $cnt -eq 0 ]; then - echo Failed pbench, no csv files present >> test_results_report - else - echo Ran pbench, at least some of the csv files are present. >> test_results_report - fi - cd - - fi - cd ${curdir} - if [ $etcd_opts -eq 1 ]; then - tar hcf /tmp/results_etcd_fio_${to_tuned_setting}.tar export_fio_data - else - tar hcf /tmp/results_pbench_fio_${to_tuned_setting}.tar export_fio_data - fi - cd - - value=`$TOOLS_BIN/set_pbench_variables --host_config $to_configuration --sys_type $to_sys_type --test fio --pbench_user $pbench_user --run_label $to_run_label` - results_prefix=`echo $value | cut -d: -f 2` - hostname=`echo $value | cut -d: -f 2` - - if [ $etcd_opts -eq 0 ]; then - $TOOLS_BIN/pbench_copy_data --hostname "${to_puser}_${to_run_label}" --user "${to_puser}" --prefix ${results_prefix}/fio_run_rt=${run_time} --copy $to_pbench_copy - else - $TOOLS_BIN/pbench_copy_data --hostname "${to_puser}_${to_run_label}" --user "${to_puser}" --prefix ${results_prefix}/etcd_run_rt=${run_time} --copy $to_pbench_copy - fi - else + cp /tmp/log ${curdir}/export_fio_data rtdir=`pwd` cd ${curdir}/export_fio_data @@ -955,7 +901,7 @@ execute_test() cd $rtdir mv /tmp/${test_name}.out ${curdir}/export_fio_data echo $run_results >> ${curdir}/export_fio_data/test_results_report - fi + ${curdir}/test_tools/save_results --curdir $curdir --home_root $to_home_root --copy_dir ${RESULTSDIR} --test_name $test_name --tuned_setting=$to_tuned_setting --version NONE --user $to_user } @@ -1147,18 +1093,7 @@ if [ $file_count -eq 0 ]; then fi fi -# -# One results dir for the entire run. Only create when $to_pbench is -# 0. The directory will get created when we call back in. -# -if [ $to_pbench -eq 0 ]; then - RESULTSDIR=${curdir}/export_fio_data_$(date "+%Y.%m.%d-%H.%M.%S") - rm export_fio_data - mkdir ${RESULTSDIR} - ln -s ${RESULTSDIR} ${curdir}/export_fio_data -else RESULTSDIR=${curdir}/export_fio_data -fi install_fio iodepth_list=`echo $iodepth_list | sed "s/,/ /g"` From 5c096b40679e7962f5414ee541469b4a68ef9912 Mon Sep 17 00:00:00 2001 From: malucius-rh Date: Mon, 8 Sep 2025 14:26:04 -0400 Subject: [PATCH 2/8] The call to pbench_fio_execute is gone, so the entire function can go away --- fio/fio_run | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/fio/fio_run b/fio/fio_run index 29a722f..7e148f0 100755 --- a/fio/fio_run +++ b/fio/fio_run @@ -585,33 +585,6 @@ install_fio() fi } -# -# Execute pbench-fio -# -pbench_fio_execute() -{ - disk_list=$1 - disk_count=$2 - if [ $use_pbench_version -eq 1 ]; then - local_config="bs_${block_size}_iod_${iodepth}_ndisks_${disk_count}_disksize_${ct_disk_size}_njobs_${njobs}" - - report_info "pbench-fio ${pbench_target} --block-sizes ${block_size} --iodepth ${iodepth} --numjobs ${njobs} --ioengine ${ioengine} -t ${test_type} --config ${local_config} ${pbench_options} --runtime ${run_time} --samples ${pbench_samples} --file-size $disk_size" - pbench-fio $pbench_target --block-sizes ${block_size} --iodepth $iodepth --numjobs $njobs --ioengine $ioengine --test-types $test_type --config $local_config ${pbench_options} --runtime ${run_time} --samples ${pbench_samples} --file-size $disk_size - if [ $? != 0 ]; then - exit_out "pbench-fio $pbench_target --block-sizes ${block_size} --iodepth $iodepth --numjobs $njobs --ioengine $ioengine --test-types $test_type --config $local_config ${pbench_options} --runtime ${run_time} --samples ${pbench_samples} --file-size $disk_size" 1 - fi - cp -R /var/pbench-agent/*fio* ${curdir}/export_fio_data - working_dir=${curdir}/export_fio_data - mkdir ${curdir}/export_fio_data - else - cd ${curdir} - $TOOLS_BIN/execute_via_pbench --cmd_executing "$0" $arguments --test $test_name --spacing 11 --pbench_stats $to_pstats - if [ $rtc != 0 ]; then - exit_out "$TOOLS_BIN/execute_via_pbench --cmd_executing "$0" $arguments --test $test_name --spacing 11 --pbench_stats $to_pstats failed." 1 - fi - fi -} - # # Create the file we need for runnig fio outside of pbench. # From 31b08584ee33296f68c696edf0c08aae927479ba Mon Sep 17 00:00:00 2001 From: malucius-rh Date: Mon, 8 Sep 2025 14:29:13 -0400 Subject: [PATCH 3/8] Remove setting and handling of pbench options --- fio/fio_run | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/fio/fio_run b/fio/fio_run index 7e148f0..bb236d0 100755 --- a/fio/fio_run +++ b/fio/fio_run @@ -120,7 +120,6 @@ io_engine_list="" njobs=0 iodepth=0 disk_size=0 -pbench_samples=5 os_vendor=`uname -a | cut -d'.' -f8` file_count=0 file_size=10 @@ -132,12 +131,10 @@ lvm_disk=0 mount_location=/fio etcd_opts=0 data_sync=0 -pbench_options="" mount_index=0 maximum_disks=0 jobs_list="" working_dir="" -use_pbench_version=0 results_version="1.0" RESULTSDIR="" target_count=0 @@ -894,8 +891,6 @@ ARGUMENT_LIST=( "jobs_max" "jobs_min" "max_disks" - "pbench_options" - "pbench_samples" "runtime" "test_type" ) @@ -907,7 +902,6 @@ NO_ARGUMENTS=( "lvm" "max_disks_only" "regression" - "use_pbench_version" ) # @@ -1006,14 +1000,6 @@ while [[ $# -gt 0 ]]; do maximum_disks=1 shift 1 ;; - --pbench_options) - fio_options=${2} - shift 2 - ;; - --pbench_samples) - pbench_samples=${2} - shift 2 - ;; --regression) regression=1 shift 1 @@ -1026,10 +1012,6 @@ while [[ $# -gt 0 ]]; do test_type=${2} shift 2 ;; - --use_pbench_version) - use_pbench_version=1 - shift 1 - ;; -h) usage $0 ;; @@ -1112,10 +1094,6 @@ if [ $etcd_opts -eq 1 ]; then if [ $file_count -eq 0 ]; then file_count=1 fi - # - # For pbench - # - pbench_options="--file-size ${file_size_opt} --sync ${data_sync}" fi if [ $file_count -ne 0 ]; then @@ -1124,13 +1102,11 @@ if [ $file_count -ne 0 ]; then # create_and_mount_fs pfile_list=`echo ${file_list} | sed "s/ /,/g"` - pbench_target="--targets ${pfile_list}" else # # Targets are the disks. # pdisk_list=`echo ${disks} | sed "s/ /,/g"` - pbench_target="--targets ${pdisk_list}" fi execute_test From b06c2c9b8bd5dde0845651bcc2336db6a34db69d Mon Sep 17 00:00:00 2001 From: malucius-rh Date: Mon, 8 Sep 2025 14:31:08 -0400 Subject: [PATCH 4/8] Remove mention of pbench in the usage notes --- fio/fio_run | 6 ------ 1 file changed, 6 deletions(-) diff --git a/fio/fio_run b/fio/fio_run index bb236d0..053d7ec 100755 --- a/fio/fio_run +++ b/fio/fio_run @@ -170,12 +170,9 @@ usage() echo " --jobs_min: minimum number of jobs to run" echo " --max_disks: maximum number of disks to run with" echo " --max_disks_only: Perform the run only with maximum disks" - echo " --pbench_samples: number of times pbench is to run each data point, default is 5" echo " --regression: regression run" echo " --runtime: run for the designated period, 60 seconds is the default" echo " --test_type: type of io doing." - echo " --use_pbench_version: Instead of running the wrappers version" - echo " of fio, use pbench-fio when pbench is requested" source test_tools/general_setup --usage } # @@ -231,9 +228,6 @@ fi # to_home_root: home directory # to_configuration: configuration information # to_times_to_run: number of times to run the test -# to_pbench_copy: Copy the data to the pbench repository, not move_it. -# to_pbench: Run the test via pbench -# to_puser: User running pbench # to_run_label: Label for the run # to_user: User on the test system running the test # to_sys_type: for results info, basically aws, azure or local From df0bd3e1bc6112c583c2c99b989e74b0b54da471 Mon Sep 17 00:00:00 2001 From: malucius-rh Date: Mon, 8 Sep 2025 14:32:47 -0400 Subject: [PATCH 5/8] There'no longer a pbench vs straight option, so make straight_fio_execute fio_execute and remove the original fio_execute --- fio/fio_run | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/fio/fio_run b/fio/fio_run index 053d7ec..4ba9107 100755 --- a/fio/fio_run +++ b/fio/fio_run @@ -679,7 +679,7 @@ loop_block_sizes() # # Run fio outisde of the pbench harness. # -straight_fio_execute() +fio_execute() { disk_count=$2 field_1=`date +%F | sed "s/-/./g"` @@ -697,11 +697,6 @@ straight_fio_execute() done } -fio_execute() -{ - straight_fio_execute $1 $2 -} - # # Perform the full run. # From fb7069e280dd53228dfa43cbb58900a522d12a4d Mon Sep 17 00:00:00 2001 From: malucius-rh Date: Mon, 8 Sep 2025 14:34:16 -0400 Subject: [PATCH 6/8] Remove mentions of pbench from the readme --- README.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/README.md b/README.md index 66aaf16..1a9880c 100644 --- a/README.md +++ b/README.md @@ -53,21 +53,13 @@ Usage: fio-wrapper/fio/fio_run --jobs_min: minimum number of jobs to run --max_disks: maximum number of disks to run with --max_disks_only: Perform the run only with maximum disks - --pbench_samples: number of times pbench is to run each data point, default is 5 --regression: regression run --runtime: run for the designated period, 60 seconds is the default --test_type: type of io doing. - --use_pbench_version: Instead of running the wrappers version - of fio, use pbench-fio when pbench is requested General options --home_parent : Our parent home directory. If not set, defaults to current working directory. --host_config : default is the current host name. --iterations : Number of times to run the test, defaults to 1. - --pbench: use pbench-user-benchmark and place information into pbench, defaults to do not use. - --pbench_user : user who started everything. Defaults to the current user. - --pbench_copy: Copy the pbench data, not move it. - --pbench_stats: What stats to gather. Defaults to all stats. - --run_label: the label to associate with the pbench run. No default setting. --run_user: user that is actually running the test on the test system. Defaults to user running wrapper. --sys_type: Type of system working with, aws, azure, hostname. Defaults to hostname. --sysname: name of the system running, used in determining config files. Defaults to hostname. @@ -76,4 +68,3 @@ General options --usage: this usage message. ``` -Note: The script does not install pbench for you. You need to do that manually. From feccb320ad75f55581686248c27a07a087cb7fa6 Mon Sep 17 00:00:00 2001 From: malucius-rh Date: Tue, 9 Sep 2025 12:31:22 -0400 Subject: [PATCH 7/8] Took out too much, put it back --- fio/fio_run | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fio/fio_run b/fio/fio_run index 4ba9107..afea93f 100755 --- a/fio/fio_run +++ b/fio/fio_run @@ -1037,7 +1037,11 @@ if [ $file_count -eq 0 ]; then fi fi - RESULTSDIR=${curdir}/export_fio_data +# Set up fresh results area +RESULTSDIR=${curdir}/export_fio_data_$(date "+%Y.%m.%d-%H.%M.%S") +rm export_fio_data +mkdir ${RESULTSDIR} +ln -s ${RESULTSDIR} ${curdir}/export_fio_data install_fio iodepth_list=`echo $iodepth_list | sed "s/,/ /g"` From ac7e702e2e06ff701d3f15ad518f7b3f893ef60e Mon Sep 17 00:00:00 2001 From: malucius-rh Date: Thu, 11 Sep 2025 08:19:51 -0400 Subject: [PATCH 8/8] remove run_label comment --- fio/fio_run | 1 - 1 file changed, 1 deletion(-) diff --git a/fio/fio_run b/fio/fio_run index afea93f..2209bec 100755 --- a/fio/fio_run +++ b/fio/fio_run @@ -228,7 +228,6 @@ fi # to_home_root: home directory # to_configuration: configuration information # to_times_to_run: number of times to run the test -# to_run_label: Label for the run # to_user: User on the test system running the test # to_sys_type: for results info, basically aws, azure or local # to_sysname: name of the system