Multiple stacks

.gitignore

@@ -1,3 +1,2 @@
 /nbproject/
-/output/
 vendor/

OTHER_RESULTS.md

@@ -0,0 +1,34 @@
+# PHP ORM Benchmark
+
+## Other Results
+
+### [motin](https://github.com/motin)
+
+(2015-12-09)
+
+Using [the supplied Docker Stack]((docker/README.md)):
+
+* Ubuntu 15.04 64bit (Docker)
+  * Nginx 1.7.12
+  * PHP-FPM 5.6.4
+    * Zend OPcache 7.0.4-dev
+    * PhalconPHP 2.0.9
+  * PHP-FPM 7.0.0
+    * Zend OPcache 7.0.6-dev
+    * PhalconPHP 2.0.9
+  * HHVM 3.10.1
+
+Running on a MacBook Pro (Retina, 15-inch, Mid 2014).
+
+By sharing underlying software stacks, the benchmark results vary only according to the host machine's hardware specs and differing code implementations.
+
+![Benchmark Results Graph](:large)
+
+## PHP-FPM 5.6.4 with opcode cache
+
+## HHVM CLI 3.10.1 (Corresponding roughly to an up-to-date PHP 5.6)
+
+## PHP CLI 7.0.0 with opcode cache
+
+
+

README.md

@@ -99,6 +99,8 @@ These are my benchmarks, not yours. **I encourage you to run on your (production
 
 Note(1): All the results are run on php with `phalcon.so` and `ice.so`. If you don't load phalcon.so or ice.so, the rps except for Phalcon or Ice probably increase a bit.
 
+If you are interested in other results, see [OTHER_RESULTS.md](OTHER_RESULTS.md).
+
 ## How to Benchmark
 
 Install source code as <http://localhost/php-framework-benchmark/>:
@@ -147,9 +149,28 @@ If you want to see current configuration, run `sudo sysctl -a`.
 ~~~
 <VirtualHost *:80>
   DocumentRoot /home/vagrant/public
+  SetEnv php_framework_benchmark_path /home/vagrant/public/php-framework-benchmark
 </VirtualHost>
 ~~~
 
+## Benchmarking using the supplied Docker Stack
+
+Use the supplied Docker Stack in order to automatically set up the following benchmarking environments:
+
+* Ubuntu 15.04 64bit (Docker)
+  * Nginx 1.7.12
+  * PHP-FPM 5.6.4
+    * Zend OPcache 7.0.4-dev
+    * PhalconPHP 2.0.9
+  * PHP-FPM 7.0.0
+    * Zend OPcache 7.0.6-dev
+    * PhalconPHP 2.0.9
+  * HHVM 3.10.1
+
+By sharing underlying software stacks, the benchmark results vary only according to the host machine's hardware specs and differing code implementations.
+
+See [docker/README.md](docker/README.md)
+
 ## References
 
 * [Aura](http://auraphp.com/) ([@auraphp](https://twitter.com/auraphp))

aura-2.0/web/index.php

@@ -16,4 +16,4 @@
 );
 $kernel();
 
-require $_SERVER['DOCUMENT_ROOT'].'/php-framework-benchmark/libs/output_data.php';
+require getenv('php_framework_benchmark_path').'/libs/output_data.php';

bear-0.10/bootstrap/contexts/prod.php

@@ -65,7 +65,7 @@
 //
 OK: {
     $app->response->setResource($app->page)->render()->send();
-    require $_SERVER['DOCUMENT_ROOT'].'/php-framework-benchmark/libs/output_data.php';
+    require getenv('php_framework_benchmark_path').'/libs/output_data.php';
     exit(0);
 }
 

bear-1.0/bootstrap/bootstrap.php

@@ -31,7 +31,7 @@
 
     // representation transfer
     $page()->transfer($app->responder, $_SERVER);
-    require $_SERVER['DOCUMENT_ROOT'].'/php-framework-benchmark/libs/output_data.php';
+    require getenv('php_framework_benchmark_path').'/libs/output_data.php';
     exit(0);
 } catch (\Exception $e) {
     $app->error->handle($e, $request)->transfer();

bear-1.0/var/log/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

bear-1.0/var/www/index.php

@@ -3,7 +3,7 @@
 // Adhoc fix for benchmarking
 // Remove sub directories from URI
 $_SERVER['REQUEST_URI'] = preg_replace(
-    '!/php-framework-benchmark/bear-1.0/var/www/index.php!',
+    ['!/php-framework-benchmark/bear-1.0/var/www/index.php!', '!/bear-1.0/var/www/index.php!'],
     '',
     $_SERVER['REQUEST_URI']
 );

benchmark.sh

@@ -1,6 +1,12 @@
 #!/bin/sh
 
-base="http://127.0.0.1/php-framework-benchmark"
+# exit on failure
+set -e
+
+if [ "$stack" = "" ]; then
+  stack="local"
+fi
+export stack;
 
 cd `dirname $0`
 
@@ -14,6 +20,6 @@ fi
 
 cd benchmarks
 
-sh hello_world.sh "$base"
+sh hello_world.sh "$stack"
 
 php ../bin/show_results_table.php

benchmarks/_functions.sh

@@ -2,38 +2,52 @@ benchmark ()
 {
     fw="$1"
     url="$2"
-    ab_log="output/$fw.ab.log"
-    output="output/$fw.output"
+    ab_log="$output_dir/$fw.ab.log"
+    output="$output_dir/$fw.output"
+    benchmark_data="$output_dir/$fw.benchmark_data"
 
     echo "ab -c 10 -t 3 $url"
     ab -c 10 -t 3 "$url" > "$ab_log"
-    curl "$url" > "$output"
+    curl -H 'X-Include-Benchmark-Output-Data: 1' --dump-header "$benchmark_data" "$url" > "$output"
 
     rps=`grep "Requests per second:" "$ab_log" | cut -f 7 -d " "`
-    memory=`tail -1 "$output" | cut -f 1 -d ':'`
-    time=`tail -1 "$output" | cut -f 2 -d ':'`
-    file=`tail -1 "$output" | cut -f 3 -d ':'`
+    memory=`grep "X-Benchmark-Output-Data:" "$benchmark_data" | cut -f 2 -d ':' | cut -f 2 -d ' '`
+    time=`grep "X-Benchmark-Output-Data:" "$benchmark_data" | cut -f 3 -d ':'`
+    file=`grep "X-Benchmark-Output-Data:" "$benchmark_data" | cut -f 4 -d ':'`
     echo "$fw: $rps: $memory: $time: $file" >> "$results_file"
 
     echo "$fw" >> "$check_file"
     grep "Document Length:" "$ab_log" >> "$check_file"
     grep "Failed requests:" "$ab_log" >> "$check_file"
     grep 'Hello World!' "$output" >> "$check_file"
+    grep ':' "$output" >> "$check_file"
     echo "---" >> "$check_file"
 
     # check errors
     touch "$error_file"
     error=''
-    x=`grep 'Failed requests:        0' "$ab_log"`
+    x=`grep 'Failed requests:        0' "$ab_log" || true`
     if [ "$x" = "" ]; then
         tmp=`grep "Failed requests:" "$ab_log"`
         error="$error$tmp"
     fi
-    x=`grep 'Hello World!' "$output"`
+    x=`grep 'Hello World!' "$output" || true`
     if [ "$x" = "" ]; then
         tmp=`cat "$output"`
         error="$error$tmp"
     fi
+    if [ "$memory" = "" ]; then
+        tmp=`cat "$benchmark_data"`
+        error="$error$tmp"
+    fi
+    if [ "$time" = "" ]; then
+        tmp=`cat "$benchmark_data"`
+        error="$error$tmp"
+    fi
+    if [ "$file" = "" ]; then
+        tmp=`cat "$benchmark_data"`
+        error="$error$tmp"
+    fi
     if [ "$error" != "" ]; then
         echo -e "$fw\n$error" >> "$error_file"
         echo "---" >> "$error_file"

benchmarks/hello_world.sh

@@ -1,22 +1,42 @@
 #!/bin/sh
 
+# exit on failure - use during development
+#set -e
+
 cd `dirname $0`
 . ./_functions.sh
 
-base="$1"
+stack="$1"
 bm_name=`basename $0 .sh`
 
-results_file="output/results.$bm_name.log"
-check_file="output/check.$bm_name.log"
-error_file="output/error.$bm_name.log"
-url_file="output/urls.log"
+if [ "$stack" = "local" ]; then
+  base="http://127.0.0.1/php-framework-benchmark"
+fi
+if [ "$stack" = "docker_nginx_php_5_6_4" ]; then
+  base="http://nginx_php_5_6_4"
+fi
+if [ "$stack" = "docker_nginx_hhvm_3_10_1" ]; then
+  base="http://nginx_hhvm_3_10_1"
+fi
+if [ "$stack" = "docker_nginx_php_7_0_0" ]; then
+  base="http://nginx_php_7_0_0"
+fi
+
+output_dir="output/$stack"
+
+results_file="$output_dir/results.$bm_name.log"
+check_file="$output_dir/check.$bm_name.log"
+error_file="$output_dir/error.$bm_name.log"
+url_file="$output_dir/urls.log"
 
 cd ..
 
-mv "$results_file" "$results_file.old"
-mv "$check_file" "$check_file.old"
-mv "$error_file" "$error_file.old"
-mv "$url_file" "$url_file.old"
+mkdir -p "$output_dir"
+
+mv "$results_file" "$results_file.old" || true
+mv "$check_file" "$check_file.old" || true
+mv "$error_file" "$error_file.old" || true
+mv "$url_file" "$url_file.old" || true
 
 for fw in `echo $targets`
 do
@@ -27,4 +47,4 @@ do
     fi
 done
 
-cat "$error_file"
+cat "$error_file" || true

bin/docker-urls.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+DIR=`dirname $0`;
+
+echo "PHP 5.6.4 Stack:";
+echo "http://"$(docker-machine ip default)":"$(docker-compose -f $DIR/../docker/docker-compose.yml port nginx_php_5_6_4 80 | sed 's/[0-9.]*://')
+echo "HHVM 3.10.1 Stack:";
+echo "http://"$(docker-machine ip default)":"$(docker-compose -f $DIR/../docker/docker-compose.yml port nginx_hhvm_3_10_1 80 | sed 's/[0-9.]*://')
+echo "PHP 7.0.0 Stack:";
+echo "http://"$(docker-machine ip default)":"$(docker-compose -f $DIR/../docker/docker-compose.yml port nginx_php_7_0_0 80 | sed 's/[0-9.]*://')
+
+exit 0

bin/show_comparison_table.php

@@ -4,16 +4,19 @@
 require __DIR__ . '/../libs/build_table.php';
 require __DIR__ . '/../libs/recalc_relative.php';
 
+$stack = getenv('stack') ? getenv('stack') : 'local';
+$output_dir = __DIR__ . '/../output/' . $stack;
+
 $list = [
     'cake-3.0',
     'symfony-2.6',
     'zf-2.4',
 ];
 
 system('git checkout master');
-$results_master = parse_results(__DIR__ . '/../output/results.hello_world.log');
+$results_master = parse_results($output_dir . '/results.hello_world.log');
 system('git checkout optimize');
-$results_optimize = parse_results(__DIR__ . '/../output/results.hello_world.log');
+$results_optimize = parse_results($output_dir . '/results.hello_world.log');
 //var_dump($results_master, $results_optimize);
 
 $is_fisrt = true;

bin/show_results_table.php

@@ -3,7 +3,10 @@
 require __DIR__ . '/../libs/parse_results.php';
 require __DIR__ . '/../libs/build_table.php';
 
-$results = parse_results(__DIR__ . '/../output/results.hello_world.log');
+$stack = getenv('stack') ? getenv('stack') : 'local';
+$output_dir = __DIR__ . '/../output/' . $stack;
+
+$results = parse_results($output_dir . '/results.hello_world.log');
 //var_dump($results);
 
 echo build_table($results);

cake-3.0/webroot/index.php

@@ -36,4 +36,4 @@
     new Response()
 );
 
-require $_SERVER['DOCUMENT_ROOT'].'/php-framework-benchmark/libs/output_data.php';
+require getenv('php_framework_benchmark_path').'/libs/output_data.php';

cake-3.1/webroot/index.php

@@ -36,4 +36,4 @@
     new Response()
 );
 
-require $_SERVER['DOCUMENT_ROOT'].'/php-framework-benchmark/libs/output_data.php';
+require getenv('php_framework_benchmark_path').'/libs/output_data.php';

ci-3.0/index.php

@@ -291,4 +291,4 @@
  */
 require_once BASEPATH.'core/CodeIgniter.php';
 
-require $_SERVER['DOCUMENT_ROOT'].'/php-framework-benchmark/libs/output_data.php';
+require getenv('php_framework_benchmark_path').'/libs/output_data.php';

cygnite-1.3/index.php

@@ -59,4 +59,4 @@
 */
 require_once CF_BOOTSTRAP.DS.'start'.EXT;
 
-require $_SERVER['DOCUMENT_ROOT'].'/php-framework-benchmark/libs/output_data.php';
+require getenv('php_framework_benchmark_path').'/libs/output_data.php';

docker/README.md

@@ -0,0 +1,64 @@
+## Benchmarking using the supplied Docker Stack
+
+Use the supplied Docker Stack in order to automatically set up the following benchmarking environments:
+
+* Ubuntu 15.04 64bit (Docker)
+  * Nginx 1.7.12
+  * PHP-FPM 5.6.4
+    * Zend OPcache 7.0.4-dev
+    * PhalconPHP 2.0.9
+  * PHP-FPM 7.0.0
+    * Zend OPcache 7.0.6-dev
+    * PhalconPHP 2.0.9
+  * HHVM 3.10.1
+
+By sharing underlying software stacks, the benchmark results vary only according to the host machine's hardware specs and differing code implementations.
+
+### Getting Started
+
+If running locally, install [Docker Toolbox](https://www.docker.com/docker-toolbox).
+
+Clone the source code:
+~~~
+$ git clone https://github.com/kenjis/php-framework-benchmark.git
+$ cd php-framework-benchmark
+~~~
+
+Cd into the docker directory of this repo and make sure that docker toolbox is available:
+~~~
+$ cd docker
+$ eval "$(docker-machine env default)"
+~~~
+
+Start the Nginx/PHP server stacks:
+~~~
+$ docker-compose up -d
+~~~
+
+Start the supplied docker shell from within this repository's `docker` folder:
+~~~
+$ docker-compose run shell
+~~~
+
+Run the set-up script:
+~~~
+# sh setup.sh
+~~~
+
+Run benchmarks against each stack:
+~~~
+$ stack=docker_nginx_php_5_6_4 sh benchmark.sh
+$ stack=docker_nginx_hhvm_3_10_1 sh benchmark.sh
+$ stack=docker_nginx_php_7_0_0 sh benchmark.sh
+~~~
+
+### Check the results
+
+To see the results graph, run the following script from outside the docker shell, from the repository root:
+
+~~~
+$ bin/docker-urls.sh
+~~~
+
+It echoes URLs, which you should open up in your browser.
+