From c8eb5a88f41745949fdebc80dbf89d4f1a0b0bf8 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Wed, 3 Jan 2018 19:53:39 -0500 Subject: [PATCH 01/30] Initial commit of the dockerized environment --- docker-compose.yml | 65 ++++++++++++++++++ docker/bot/Dockerfile | 4 ++ docker/db-data/.gitignore | 1 + docker/db-seed/Dockerfile | 5 ++ docker/db-seed/database.sql | 131 ++++++++++++++++++++++++++++++++++++ docker/db-seed/init.sh | 5 ++ docker/db/Dockerfile | 2 + docker/db/config/my.cnf | 1 + docker/web/config/default | 22 ++++++ docker/web/logs/.gitignore | 2 + 10 files changed, 238 insertions(+) create mode 100644 docker-compose.yml create mode 100644 docker/bot/Dockerfile create mode 100644 docker/db-data/.gitignore create mode 100644 docker/db-seed/Dockerfile create mode 100644 docker/db-seed/database.sql create mode 100644 docker/db-seed/init.sh create mode 100644 docker/db/Dockerfile create mode 100644 docker/db/config/my.cnf create mode 100644 docker/web/config/default create mode 100644 docker/web/logs/.gitignore diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c66fa25 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,65 @@ +version: '3' + +services: + db: + image: mysql:5.7 + ports: + - "3306:3306" + volumes: + - ./docker/db/config/my.cnf:/etc/mysql/conf.d/arbbot.cnf + - ./docker/db-data:/var/lib/mysql + networks: + backend: + aliases: + - database + restart: always + environment: + - MYSQL_ROOT_PASSWORD=changeme + - MYSQL_DATABASE=arbitrage + - MYSQL_USER=arbitrage + - MYSQL_PASSWORD=changeme + db_seed: + depends_on: + - db + build: docker/db-seed + networks: + - backend + bot: + depends_on: + - db_seed + build: docker/bot + volumes: + - .:/home/arbbot + restart: always + networks: + - backend + web: + depends_on: + - db_seed + - php + image: nginx:1.13 + ports: + - "80:80" + volumes: + - ./web:/usr/share/nginx/html + - ./docker/web/config/default:/etc/nginx/sites-available/default + - ./docker/web/config/default:/etc/nginx/sites-enabled/default + + - ./docker/web/logs/access.log:/var/log/nginx/access.log + - ./docker/web/logs/error.log:/var/log/nginx/error.log + networks: + - frontend + restart: always + php: + image: php:7.2-fpm + ports: + - "9000:9000" + restart: always + volumes: + - ./web:/usr/share/nginx/html + networks: + - frontend + - backend +networks: + frontend: + backend: diff --git a/docker/bot/Dockerfile b/docker/bot/Dockerfile new file mode 100644 index 0000000..d83f313 --- /dev/null +++ b/docker/bot/Dockerfile @@ -0,0 +1,4 @@ +FROM hhvm/hhvm:3.21-lts-latest +RUN mkdir /home/arbbot +COPY config.ini /home/arbbot +ENTRYPOINT [ "hhvm", "/home/arbbot/main.php" ] diff --git a/docker/db-data/.gitignore b/docker/db-data/.gitignore new file mode 100644 index 0000000..f59ec20 --- /dev/null +++ b/docker/db-data/.gitignore @@ -0,0 +1 @@ +* \ No newline at end of file diff --git a/docker/db-seed/Dockerfile b/docker/db-seed/Dockerfile new file mode 100644 index 0000000..b86a87d --- /dev/null +++ b/docker/db-seed/Dockerfile @@ -0,0 +1,5 @@ +FROM mysql:5.7 +COPY init.sh /init.sh +COPY database.sql /database.sql +RUN chmod +x /init.sh +ENTRYPOINT "/init.sh" diff --git a/docker/db-seed/database.sql b/docker/db-seed/database.sql new file mode 100644 index 0000000..8f57ed8 --- /dev/null +++ b/docker/db-seed/database.sql @@ -0,0 +1,131 @@ +SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"; +SET time_zone = "+00:00"; + +-- -------------------------------------------------------- + +-- +-- Table structure for table `alerts` +-- + +CREATE TABLE IF NOT EXISTS `alerts` ( + `ID` int(11) NOT NULL AUTO_INCREMENT, + `created` int(11) NOT NULL, + `type` ENUM('stuck-transfer', 'poloniex-withdrawal-limit') NOT NULL, + `message` text NOT NULL, + PRIMARY KEY (`ID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; + + +-- -------------------------------------------------------- + +-- +-- Table structure for table `log` +-- + +CREATE TABLE IF NOT EXISTS `log` ( + `ID` int(11) NOT NULL AUTO_INCREMENT, + `created` int(11) NOT NULL, + `message` text NOT NULL, + PRIMARY KEY (`ID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; + +-- -------------------------------------------------------- + +-- +-- Table structure for table `management` +-- + +CREATE TABLE IF NOT EXISTS `management` ( + `ID` int(11) NOT NULL AUTO_INCREMENT, + `created` int(11) NOT NULL, + `ID_exchange` int(11) NOT NULL, + `coin` char(5) NOT NULL, + `amount` varchar(18) NOT NULL, + `rate` varchar(18) NOT NULL, + PRIMARY KEY (`ID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; + +-- -------------------------------------------------------- + +-- +-- Table structure for table `snapshot` +-- + +CREATE TABLE IF NOT EXISTS `snapshot` ( + `ID` int(11) NOT NULL AUTO_INCREMENT, + `created` int(11) NOT NULL, + `coin` char(5) NOT NULL, + `balance` varchar(18) NOT NULL, + `desired_balance` varchar(18) DEFAULT NULL, + `uses` int(11) NOT NULL, + `trades` int(11) NOT NULL, + `rate` varchar(18) NOT NULL, + `ID_exchange` int(11) NOT NULL, + PRIMARY KEY (`ID`), + KEY `created` (`created`), + KEY `coin` (`coin`), + KEY `ID_exchange` (`ID_exchange`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; + +-- -------------------------------------------------------- + +-- +-- Table structure for table `stats` +-- + +CREATE TABLE IF NOT EXISTS `stats` ( + `keyy` varchar(255) NOT NULL, + `value` varchar(255) NOT NULL, + PRIMARY KEY (`keyy`), + KEY `key` (`keyy`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- -------------------------------------------------------- + +-- +-- Table structure for table `track` +-- + +CREATE TABLE IF NOT EXISTS `track` ( + `ID` int(11) NOT NULL AUTO_INCREMENT, + `created` int(11) NOT NULL, + `coin` char(5) NOT NULL, + `amount` varchar(18) NOT NULL, + `profit` varchar(18) NOT NULL, + `ID_exchange` int(11) NOT NULL, + PRIMARY KEY (`ID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; + +-- -------------------------------------------------------- + +-- +-- Table structure for table `trade` +-- + +CREATE TABLE IF NOT EXISTS `trade` ( + `ID` int(11) NOT NULL AUTO_INCREMENT, + `created` int(11) NOT NULL, + `coin` char(5) NOT NULL, + `currency` char(3) NOT NULL, + `amount` varchar(18) NOT NULL, + `ID_exchange_source` int(11) NOT NULL, + `ID_exchange_target` int(11) NOT NULL, + PRIMARY KEY (`ID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; + +-- -------------------------------------------------------- + +-- +-- Table structure for table `withdrawal` +-- + +CREATE TABLE IF NOT EXISTS `withdrawal` ( + `ID` int(11) NOT NULL AUTO_INCREMENT, + `created` int(11) NOT NULL, + `ID_exchange_source` int(11) NOT NULL, + `ID_exchange_target` int(11) NOT NULL, + `coin` char(5) NOT NULL, + `amount` varchar(18) NOT NULL, + `address` text NOT NULL, + PRIMARY KEY (`ID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; diff --git a/docker/db-seed/init.sh b/docker/db-seed/init.sh new file mode 100644 index 0000000..ce6eecc --- /dev/null +++ b/docker/db-seed/init.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# Wait for the server to come online... +sleep 10 +mysql -u root -p=changeme -h database arbitrage Date: Thu, 18 Jan 2018 19:45:50 -0500 Subject: [PATCH 02/30] Bring the dockerization closer to a working state --- .env.example | 15 ++ config.ini.example | 4 +- docker-compose.yml | 68 ++++----- docker/bot/Dockerfile | 31 ++++- docker/bot/ntp.sh | 12 ++ docker/bot/php.ini | 10 ++ docker/bot/wait-for-db.sh | 23 +++ docker/db-data/.gitignore | 1 - docker/db-seed/Dockerfile | 5 - docker/db-seed/database.sql | 131 ------------------ docker/db-seed/init.sh | 5 - docker/db/Dockerfile | 2 - docker/db/config/my.cnf | 15 +- docker/db/seed/.gitignore | 1 + .../db/seed/00-database.sql | 0 docker/php/Dockerfile | 41 ++++++ docker/php/php.ini | 10 ++ docker/web/Dockerfile | 27 ++++ docker/web/config/default | 6 +- docker/web/nginx.conf | 43 ++++++ web/WebDB.php | 6 +- web/config.inc.php.example | 2 +- 22 files changed, 272 insertions(+), 186 deletions(-) create mode 100644 .env.example create mode 100644 docker/bot/ntp.sh create mode 100644 docker/bot/php.ini create mode 100644 docker/bot/wait-for-db.sh delete mode 100644 docker/db-data/.gitignore delete mode 100644 docker/db-seed/Dockerfile delete mode 100644 docker/db-seed/database.sql delete mode 100644 docker/db-seed/init.sh delete mode 100644 docker/db/Dockerfile create mode 100644 docker/db/seed/.gitignore rename database.sql => docker/db/seed/00-database.sql (100%) create mode 100644 docker/php/Dockerfile create mode 100644 docker/php/php.ini create mode 100644 docker/web/Dockerfile create mode 100644 docker/web/nginx.conf diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..89762b7 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# Linux user account information for running the bot +UID=1000 +USER=bot + +# Username for HTTP access +HTTP_AUTH_USER=arbb0t + +# Password for HTTP access +HTTP_AUTH_PASSWORD=arbb0tpass + +# MySQL related config +MYSQL_ROOT_PASSWORD=root +MYSQL_DATABASE=arbitrage +MYSQL_USER=arbitrage +MYSQL_PASSWORD=MySqlPassword diff --git a/config.ini.example b/config.ini.example index 43bc0e7..b5eac09 100644 --- a/config.ini.example +++ b/config.ini.example @@ -38,8 +38,8 @@ user = arbitrage ; Your MySQL database password pass = YOUR_PASSWORD -; Your MySQL database host -host = localhost +; Your MySQL database host (DONT MODIFY IF USING DOCKER) +host = db ; Your MySQL database name name = arbitrage diff --git a/docker-compose.yml b/docker-compose.yml index c66fa25..f40d7e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,60 +3,66 @@ version: '3' services: db: image: mysql:5.7 - ports: - - "3306:3306" + env_file: + - .env volumes: - ./docker/db/config/my.cnf:/etc/mysql/conf.d/arbbot.cnf + - ./docker/db/seed:/docker-entrypoint-initdb.d - ./docker/db-data:/var/lib/mysql networks: backend: - aliases: - - database restart: always - environment: - - MYSQL_ROOT_PASSWORD=changeme - - MYSQL_DATABASE=arbitrage - - MYSQL_USER=arbitrage - - MYSQL_PASSWORD=changeme - db_seed: - depends_on: - - db - build: docker/db-seed - networks: - - backend bot: depends_on: - - db_seed - build: docker/bot + - db + env_file: + - .env + build: + context: ./docker/bot + args: + - uid=${UID:-1000} + - username=${USER:-bot} volumes: - - .:/home/arbbot - restart: always + - .:/home/${USER:-bot} networks: - backend + command: ["/bin/wait-for-db.sh", "profit_loss", "${MYSQL_DATABASE:-arbitrage}", "${MYSQL_ROOT_PASSWORD:-root}", "/usr/bin/hhvm", "main.php"] web: depends_on: - - db_seed + - db - php - image: nginx:1.13 + env_file: + - .env + build: + context: ./docker/web + args: + - uid=${UID:-1000} + - username=${USER:-bot} + - http_auth_user=${HTTP_AUTH_USER:-arbb0t} + - http_auth_password=${HTTP_AUTH_PASSWORD:-arbb0tpass} ports: - "80:80" volumes: - - ./web:/usr/share/nginx/html - - ./docker/web/config/default:/etc/nginx/sites-available/default - - ./docker/web/config/default:/etc/nginx/sites-enabled/default - - - ./docker/web/logs/access.log:/var/log/nginx/access.log - - ./docker/web/logs/error.log:/var/log/nginx/error.log + - ./web:/var/www/html networks: - frontend restart: always php: - image: php:7.2-fpm - ports: - - "9000:9000" + depends_on: + - db + build: + context: ./docker/php + args: + - uid=${UID:-1000} + - username=${USER:-root} + - db=${MYSQL_DATABASE:-arbitrage} + - db_username=${MYSQL_USER:-arbitrage} + - db_password=${MYSQL_PASSWORD:-MySqlPassword} + expose: + - 9000 restart: always volumes: - - ./web:/usr/share/nginx/html + - ./web:/var/www/html networks: - frontend - backend diff --git a/docker/bot/Dockerfile b/docker/bot/Dockerfile index d83f313..1b8d52b 100644 --- a/docker/bot/Dockerfile +++ b/docker/bot/Dockerfile @@ -1,4 +1,29 @@ FROM hhvm/hhvm:3.21-lts-latest -RUN mkdir /home/arbbot -COPY config.ini /home/arbbot -ENTRYPOINT [ "hhvm", "/home/arbbot/main.php" ] + +RUN apt-get update && apt-get install -y apt-utils + +RUN apt-get install -y \ + libicu-dev unzip ntpdate libmcrypt-dev zlib1g-dev g++ mysql-client ntp \ + # php-curl php-mbstring php-mcrypt php-intl php-mysqli php-zip \ + && rm -r /var/lib/apt/lists/* + +ADD wait-for-db.sh /bin/wait-for-db.sh +RUN chmod +x /bin/wait-for-db.sh +ADD ntp.sh /bin/ntp.sh +RUN /bin/ntp.sh + +COPY php.ini /usr/local/etc/php/php.ini + +ARG uid +ARG username +ARG password + +# Set UID +RUN (users | grep -q ${username}) || \ + (useradd -s /bin/bash -m -u ${uid} ${username} \ + && chown -R ${username}:${username} /home/${username} \ + && echo "Using user ${username} with UID ${uid}.") + +WORKDIR "/home/${username}" +#CMD ["/bin/ls", "/bin"] +#CMD ["/bin/wait-for-db.sh", "profit_loss", ${password}, "hhvm", "main.php"] diff --git a/docker/bot/ntp.sh b/docker/bot/ntp.sh new file mode 100644 index 0000000..b49c5b6 --- /dev/null +++ b/docker/bot/ntp.sh @@ -0,0 +1,12 @@ +#!/bin/bash +rm /etc/localtime +ln -s /usr/share/zoneinfo/UTC /etc/localtime +service ntp stop +ntpdate ntp.ubuntu.com +# || date -s "$(wget -qSO- --max-redirect=0 google.com 2>&1 | grep Date: | cut -d' ' -f5-8)Z" +service ntp start +update-rc.d ntp enable +sleep 5 +echo "Checking for Peers" +ntpq -c lpeer + diff --git a/docker/bot/php.ini b/docker/bot/php.ini new file mode 100644 index 0000000..061bb56 --- /dev/null +++ b/docker/bot/php.ini @@ -0,0 +1,10 @@ +log_errors = On +error_log = /dev/stderr +file_uploads = On +memory_limit = 512M +upload_max_filesize = 256M +post_max_size = 256M +max_execution_time = 60 +error_reporting = E_ALL +date.timezone = "UTC" +include_path = .:/usr/share/php:/usr/share/pear \ No newline at end of file diff --git a/docker/bot/wait-for-db.sh b/docker/bot/wait-for-db.sh new file mode 100644 index 0000000..be45d45 --- /dev/null +++ b/docker/bot/wait-for-db.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# wait-for-db.sh + +set -e +sleep 10 + +waitForTable() { + table=$1; + db=$2 + count=$(mysql -N -s -u root -p$3 -h db -e \ + "select count(*) from information_schema.tables where \ + table_schema='${db}' and table_name='${table}';" 2>/dev/null) + while [ "$count" -eq "0" ]; do + sleep 1 + done +} + +waitForTable $@ +sleep 1 + +shift 3; +echo $@ +exec $@ diff --git a/docker/db-data/.gitignore b/docker/db-data/.gitignore deleted file mode 100644 index f59ec20..0000000 --- a/docker/db-data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -* \ No newline at end of file diff --git a/docker/db-seed/Dockerfile b/docker/db-seed/Dockerfile deleted file mode 100644 index b86a87d..0000000 --- a/docker/db-seed/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM mysql:5.7 -COPY init.sh /init.sh -COPY database.sql /database.sql -RUN chmod +x /init.sh -ENTRYPOINT "/init.sh" diff --git a/docker/db-seed/database.sql b/docker/db-seed/database.sql deleted file mode 100644 index 8f57ed8..0000000 --- a/docker/db-seed/database.sql +++ /dev/null @@ -1,131 +0,0 @@ -SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"; -SET time_zone = "+00:00"; - --- -------------------------------------------------------- - --- --- Table structure for table `alerts` --- - -CREATE TABLE IF NOT EXISTS `alerts` ( - `ID` int(11) NOT NULL AUTO_INCREMENT, - `created` int(11) NOT NULL, - `type` ENUM('stuck-transfer', 'poloniex-withdrawal-limit') NOT NULL, - `message` text NOT NULL, - PRIMARY KEY (`ID`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; - - --- -------------------------------------------------------- - --- --- Table structure for table `log` --- - -CREATE TABLE IF NOT EXISTS `log` ( - `ID` int(11) NOT NULL AUTO_INCREMENT, - `created` int(11) NOT NULL, - `message` text NOT NULL, - PRIMARY KEY (`ID`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; - --- -------------------------------------------------------- - --- --- Table structure for table `management` --- - -CREATE TABLE IF NOT EXISTS `management` ( - `ID` int(11) NOT NULL AUTO_INCREMENT, - `created` int(11) NOT NULL, - `ID_exchange` int(11) NOT NULL, - `coin` char(5) NOT NULL, - `amount` varchar(18) NOT NULL, - `rate` varchar(18) NOT NULL, - PRIMARY KEY (`ID`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; - --- -------------------------------------------------------- - --- --- Table structure for table `snapshot` --- - -CREATE TABLE IF NOT EXISTS `snapshot` ( - `ID` int(11) NOT NULL AUTO_INCREMENT, - `created` int(11) NOT NULL, - `coin` char(5) NOT NULL, - `balance` varchar(18) NOT NULL, - `desired_balance` varchar(18) DEFAULT NULL, - `uses` int(11) NOT NULL, - `trades` int(11) NOT NULL, - `rate` varchar(18) NOT NULL, - `ID_exchange` int(11) NOT NULL, - PRIMARY KEY (`ID`), - KEY `created` (`created`), - KEY `coin` (`coin`), - KEY `ID_exchange` (`ID_exchange`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; - --- -------------------------------------------------------- - --- --- Table structure for table `stats` --- - -CREATE TABLE IF NOT EXISTS `stats` ( - `keyy` varchar(255) NOT NULL, - `value` varchar(255) NOT NULL, - PRIMARY KEY (`keyy`), - KEY `key` (`keyy`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - --- -------------------------------------------------------- - --- --- Table structure for table `track` --- - -CREATE TABLE IF NOT EXISTS `track` ( - `ID` int(11) NOT NULL AUTO_INCREMENT, - `created` int(11) NOT NULL, - `coin` char(5) NOT NULL, - `amount` varchar(18) NOT NULL, - `profit` varchar(18) NOT NULL, - `ID_exchange` int(11) NOT NULL, - PRIMARY KEY (`ID`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; - --- -------------------------------------------------------- - --- --- Table structure for table `trade` --- - -CREATE TABLE IF NOT EXISTS `trade` ( - `ID` int(11) NOT NULL AUTO_INCREMENT, - `created` int(11) NOT NULL, - `coin` char(5) NOT NULL, - `currency` char(3) NOT NULL, - `amount` varchar(18) NOT NULL, - `ID_exchange_source` int(11) NOT NULL, - `ID_exchange_target` int(11) NOT NULL, - PRIMARY KEY (`ID`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; - --- -------------------------------------------------------- - --- --- Table structure for table `withdrawal` --- - -CREATE TABLE IF NOT EXISTS `withdrawal` ( - `ID` int(11) NOT NULL AUTO_INCREMENT, - `created` int(11) NOT NULL, - `ID_exchange_source` int(11) NOT NULL, - `ID_exchange_target` int(11) NOT NULL, - `coin` char(5) NOT NULL, - `amount` varchar(18) NOT NULL, - `address` text NOT NULL, - PRIMARY KEY (`ID`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=1 ; diff --git a/docker/db-seed/init.sh b/docker/db-seed/init.sh deleted file mode 100644 index ce6eecc..0000000 --- a/docker/db-seed/init.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -# Wait for the server to come online... -sleep 10 -mysql -u root -p=changeme -h database arbitrage /etc/arbbot/config.inc.php +RUN echo '' >> /etc/arbbot/config.inc.php +RUN echo '$dbHost = "db"; // DO NOT MODIFY IF USING DOCKER' >> /etc/arbbot/config.inc.php +RUN echo -n '$dbName = "'>> /etc/arbbot/config.inc.php +RUN echo -n ${db} >> /etc/arbbot/config.inc.php +RUN echo '";' >> /etc/arbbot/config.inc.php +RUN echo -n '$dbUser = "' >> /etc/arbbot/config.inc.php +RUN echo -n ${db_username} >> /etc/arbbot/config.inc.php +RUN echo '";' >> /etc/arbbot/config.inc.php +RUN echo -n '$dbPass = "' >> /etc/arbbot/config.inc.php +RUN echo -n ${db_password} >> /etc/arbbot/config.inc.php +RUN echo '";' >> /etc/arbbot/config.inc.php + +# Set UID +RUN (users | grep -q ${username}) || \ + (useradd -s /bin/bash -m -u ${uid} ${username} \ + && chown -R ${username}:${username} /home/${username} \ + && echo "Using user ${username} with UID ${uid}.") + +CMD ["php-fpm"] diff --git a/docker/php/php.ini b/docker/php/php.ini new file mode 100644 index 0000000..061bb56 --- /dev/null +++ b/docker/php/php.ini @@ -0,0 +1,10 @@ +log_errors = On +error_log = /dev/stderr +file_uploads = On +memory_limit = 512M +upload_max_filesize = 256M +post_max_size = 256M +max_execution_time = 60 +error_reporting = E_ALL +date.timezone = "UTC" +include_path = .:/usr/share/php:/usr/share/pear \ No newline at end of file diff --git a/docker/web/Dockerfile b/docker/web/Dockerfile new file mode 100644 index 0000000..83807c4 --- /dev/null +++ b/docker/web/Dockerfile @@ -0,0 +1,27 @@ +FROM nginx:1.13 + +RUN apt-get update && apt-get install -y apt-utils + +RUN apt-get install -y openssl + +# Remove the default Nginx coniguration file +RUN rm -v /etc/nginx/nginx.conf && rm -v /etc/nginx/conf.d/* + +# Copy a configuration file from the current directory +ADD ./nginx.conf /etc/nginx/nginx.conf +ADD ./config/* /etc/nginx/conf.d/ + +ARG http_auth_user +ARG http_auth_password + +RUN echo -n ${http_auth_user}:$(openssl passwd -apr1 ${http_auth_password}) > /etc/nginx/arbitrage.passwd + +ARG uid +ARG username + +# Set UID +RUN (users | grep -q ${username}) || \ + (useradd -s /bin/bash -m -u ${uid} ${username} \ + && chown -R ${username}:${username} /home/${username} \ + && echo "Using user ${username} with UID ${uid}.") + diff --git a/docker/web/config/default b/docker/web/config/default index 3dba0da..160622e 100644 --- a/docker/web/config/default +++ b/docker/web/config/default @@ -1,14 +1,14 @@ server { listen 80; - root /usr/share/nginx/html; + root /var/www/html; index index.html; server_name localhost; location / { -# auth_basic "Restricted area"; -# auth_basic_user_file /var/www/conf/arbitrage.passwd; + auth_basic "Restricted area"; + auth_basic_user_file /var/www/conf/arbitrage.passwd; } diff --git a/docker/web/nginx.conf b/docker/web/nginx.conf new file mode 100644 index 0000000..93b24da --- /dev/null +++ b/docker/web/nginx.conf @@ -0,0 +1,43 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + + +http { + server_tokens off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + gzip on; + gzip_http_version 1.0; + gzip_proxied any; + gzip_min_length 500; + gzip_disable "MSIE [1-6]\."; + gzip_types text/plain text/xml text/css + text/comma-separated-values + text/javascript + application/x-javascript + application/atom+xml; + + + include /etc/nginx/conf.d/*.conf; +} diff --git a/web/WebDB.php b/web/WebDB.php index c5e49f3..0a2b9ed 100644 --- a/web/WebDB.php +++ b/web/WebDB.php @@ -1,7 +1,11 @@ Date: Thu, 18 Jan 2018 20:08:01 -0500 Subject: [PATCH 03/30] Create the database user --- docker-compose.yml | 9 +++++++-- docker/db/Dockerfile | 13 +++++++++++++ docker/db/seed/{00-database.sql => 01-database.sql} | 0 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 docker/db/Dockerfile rename docker/db/seed/{00-database.sql => 01-database.sql} (100%) diff --git a/docker-compose.yml b/docker-compose.yml index f40d7e9..7778265 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,12 +2,17 @@ version: '3' services: db: - image: mysql:5.7 + build: + context: ./docker/db + args: + - db=${MYSQL_DATABASE:-arbitrage} + - db_username=${MYSQL_USER:-arbitrage} + - db_password=${MYSQL_PASSWORD:-MySqlPassword} env_file: - .env volumes: - ./docker/db/config/my.cnf:/etc/mysql/conf.d/arbbot.cnf - - ./docker/db/seed:/docker-entrypoint-initdb.d + - ./docker/db/seed/01-database.sql:/docker-entrypoint-initdb.d/01-database.sql - ./docker/db-data:/var/lib/mysql networks: backend: diff --git a/docker/db/Dockerfile b/docker/db/Dockerfile new file mode 100644 index 0000000..87676ba --- /dev/null +++ b/docker/db/Dockerfile @@ -0,0 +1,13 @@ +FROM mysql:5.7 + +ARG db +ARG db_username +ARG db_password + +RUN echo -n 'GRANT ALL ON ' > /docker-entrypoint-initdb.d/00-privileges.sql +RUN echo -n ${db} >> /docker-entrypoint-initdb.d/00-privileges.sql +RUN echo -n '.* TO ' >> /docker-entrypoint-initdb.d/00-privileges.sql +RUN echo -n ${db_username} >> /docker-entrypoint-initdb.d/00-privileges.sql +RUN echo -n "@'%' IDENTIFIED BY '" >> /docker-entrypoint-initdb.d/00-privileges.sql +RUN echo -n ${db_password} >> /docker-entrypoint-initdb.d/00-privileges.sql +RUN echo "';" >> /docker-entrypoint-initdb.d/00-privileges.sql diff --git a/docker/db/seed/00-database.sql b/docker/db/seed/01-database.sql similarity index 100% rename from docker/db/seed/00-database.sql rename to docker/db/seed/01-database.sql From 572323a2058ee3a1ba9c29f4625960969ef3207b Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Thu, 18 Jan 2018 20:22:07 -0500 Subject: [PATCH 04/30] Edit README.md --- README.md | 125 ++++++------------------------------------------------ 1 file changed, 14 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 0db1473..d209f55 100644 --- a/README.md +++ b/README.md @@ -12,138 +12,41 @@ You can see the bot running on a cheap [linode](https://www.linode.com) here. C * Bleutrade (all BTC markets) * Poloniex (all BTC markets) -## Installation on Debian 9.0+ / Ubuntu 16.04+ +## Running the bot using Docker -Install required packages: +The only supported way to run this bot currently is to use Docker. Advanced users are welcome to study the docker files to see how the configuration works if they want to setup their own advanced setups, but all such configurations are unsupported and you're on your own if things go wrong! -``` -sudo apt-get install php-cli php-curl php-mysqlnd mysql-server nginx-full php-fpm unzip apache2-utils ntpdate -sudo ntpdate ntp.ubuntu.com -``` - -Clone the repository on the server. Note that installation from ZIP archives isn't supported any more. - -``` -cd /var/www -git clone --recursive https://github.com/cryptoeax/arbbot.git -``` - -cd into the directory: - -``` -cd arbbot -``` - -Prepare the MySQL database: - -``` - mysql -u root -p - mysql> CREATE DATABASE arbitrage; - mysql> GRANT ALL ON arbitrage.* TO arbitrage@localhost IDENTIFIED BY 'YOUR_PASSWORD'; - mysql> use arbitrage; - mysql> source database.sql; - mysql> quit - ``` +### Pre-requisites -Configure the database connection: +Install the latest versions of docker and docker-compose -``` -cp web/config.inc.php.example web/config.inc.php -nano web/config.inc.php -``` +Clone the repository somewhere using `git clone --recursive`. +### Preparation -Configure the bot: +Customize your environment settings: ``` -cp config.ini.example config.ini -nano config.ini +cp .env.example .env +vi .env # edit the file to customize the variables, NEVER use the default passwords +vi config.ini # edit the database settings to make them match .env +docker-compose build ``` -Edit all options to fit your needs and enter your API keys! You can change settings even while the bot is running. The changes will be automatically applied. - -## Configuring the Webinterface - -First, you need to configure a username and password: - -``` -htpasswd -b /var/www/conf/arbitrage.passwd username password -``` - -Then you need to edit the web server configuration. - -``` -rm /etc/nginx/sites-enabled/default -nano /etc/nginx/sites-enabled/default -``` - -The NGINX configuration file should look like this: - -``` -server { - - listen 80; - root /var/www/arbbot/web; - index index.html; - server_name localhost; - - location / { - - auth_basic "Restricted area"; - auth_basic_user_file /var/www/conf/arbitrage.passwd; - - } - - location ~ \.php$ { - - include snippets/fastcgi-php.conf; - fastcgi_pass unix:/var/run/php/php7.0-fpm.sock; - - } - -} -``` - -If your web server is exposed to the public Internet, the UI will by default refuse to load unless you enable HTTPS. You can get a free certificate using Let's Encrypt, see [this tutorial](https://www.nginx.com/blog/free-certificates-lets-encrypt-and-nginx/) on how to do that. Assuming you have a certificate, you would need to add a `ssl_certificate` and `ssl_certificate_key` directive to the configration file, if the Let's Encrypt client doesn't do that for you (which it should.) If you are going to access your bot on http://localhost then you don't need to perform this step. - -Now, restart the webserver: - -``` -/etc/init.d/nginx restart -``` - -You should now be able to access the webinterface with your browser. - -## Running the bot +### Running the bot Now you are ready to give the bot a test by running it: ``` -php main.php -``` - -(Note: It is recommended to run `main.php` with [`hhvm`](https://docs.hhvm.com/hhvm/installation/linux) instead of `php` in order to speed up the bot a bit.) - -You should see output like this: - +docker-compose up -d ``` -19:13:34: ARBITRATOR V2.0 launching -``` -To actually allow the bot to buy coins automatically, you need to reserve some autobuy funds. You can do that either by running the following commands against the database manually: -``` - mysql -u root -p - mysql> use arbitrage; - mysql> UPDATE stats SET value = "0.2" WHERE keyy = "autobuy_funds"; - mysql> quit -``` +To actually allow the bot to buy coins automatically, you need to reserve some autobuy funds. You can do that by turning on the admin UI by enabling the `general.admin-ui` setting, the web UI shows you the Admin interface which allows you to change the autobuy funds amount. It is not recommended to enable this if your web UI isn't secure using password authentication and HTTPS in case it's exposed to the Internet. This example assigns 0.2 BTC to the "autobuy_funds". The higher the amount, the more coins can be bought and the more arbitrage-opportunities can be taken. Be careful to keep at least 0.1 - 0.2 BTC at the exchange to give the bot enough room to trade. -Alternatively by enabling the `general.admin-ui` setting, the web UI shows you the Admin interface which allows you to change the autobuy funds amount. It is not recommended to enable this if your web UI isn't secure using password authentication and HTTPS in case it's exposed to the Internet. - ## How does the bot make profit? Arbitrage trading means that differences in exchange rates between two exchanges are used to gain a profit. From 889429efa1b5bf7b7cad2d8696502daa3c6987e9 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Thu, 18 Jan 2018 20:54:16 -0500 Subject: [PATCH 05/30] Add support for importing existing data into the bot --- README.md | 11 +++++++++++ docker-compose.yml | 1 + 2 files changed, 12 insertions(+) diff --git a/README.md b/README.md index d209f55..ae3b51f 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,17 @@ Customize your environment settings: cp .env.example .env vi .env # edit the file to customize the variables, NEVER use the default passwords vi config.ini # edit the database settings to make them match .env +``` + +In case you have been running the bot from the pre-dockerized versions, you probably want to import the data that the bot currently has saved in its database into the database that the new containerized bot launches. You can do so by planting a special `data.sql` file in the right place, like this: + +``` +mysqldump -h host -u username -p --no-create-info > docker/db/seed/data.sql +``` + +Once you have finished the above steps, you are ready to build your containers. + +``` docker-compose build ``` diff --git a/docker-compose.yml b/docker-compose.yml index 7778265..8a6401c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,7 @@ services: volumes: - ./docker/db/config/my.cnf:/etc/mysql/conf.d/arbbot.cnf - ./docker/db/seed/01-database.sql:/docker-entrypoint-initdb.d/01-database.sql + - ./docker/db/seed/data.sql:/docker-entrypoint-initdb.d/data.sql - ./docker/db-data:/var/lib/mysql networks: backend: From 2a5864feb34220f056743951cebf44c778f47483 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Thu, 18 Jan 2018 20:54:37 -0500 Subject: [PATCH 06/30] Add instructions about the bot's data --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index ae3b51f..e1931ce 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,12 @@ This example assigns 0.2 BTC to the "autobuy_funds". The higher the amount, the the more arbitrage-opportunities can be taken. Be careful to keep at least 0.1 - 0.2 BTC at the exchange to give the bot enough room to trade. +## Updating the bot +When you update the bot, you need to rebuild the docker containers in case they require rebuilding before rerunning the bot, you can do that with `docker-compose build`. + +## Backing up the bot's data +The bot's mysql server saves its data in `docker/db-data`. You are encouraged to back up the contents of this directory occasionally. If you delete the contents of this directory, the next time you run the bot the bot will reinitialize its database from scratch (and will import an initial seed data from `data.sql` if that file exists.) + ## How does the bot make profit? Arbitrage trading means that differences in exchange rates between two exchanges are used to gain a profit. From 3845474dc49f8a858bbbece20f3a61cf8d427a05 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Thu, 18 Jan 2018 21:01:15 -0500 Subject: [PATCH 07/30] Add a .gitignore to db-data --- docker/db-data/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 docker/db-data/.gitignore diff --git a/docker/db-data/.gitignore b/docker/db-data/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/docker/db-data/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file From eb97131782ef04499298446d2630e9a466db051c Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Thu, 18 Jan 2018 21:05:39 -0500 Subject: [PATCH 08/30] Add an empty base data.sql file --- docker/db/seed/data.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docker/db/seed/data.sql diff --git a/docker/db/seed/data.sql b/docker/db/seed/data.sql new file mode 100644 index 0000000..e69de29 From a4fbbf97c5f97af158d29d5719e0692a5ada4347 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Fri, 19 Jan 2018 00:11:05 -0500 Subject: [PATCH 09/30] Improve the bot definition --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8a6401c..69eb057 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,8 @@ services: - .:/home/${USER:-bot} networks: - backend - command: ["/bin/wait-for-db.sh", "profit_loss", "${MYSQL_DATABASE:-arbitrage}", "${MYSQL_ROOT_PASSWORD:-root}", "/usr/bin/hhvm", "main.php"] + restart: always + command: ["/bin/wait-for-db.sh", "current_simulated_profit_rate", "${MYSQL_DATABASE:-arbitrage}", "${MYSQL_ROOT_PASSWORD:-root}", "/usr/bin/hhvm", "main.php"] web: depends_on: - db From ea1ae71a1506c80bde0445321734cbf0681f97b3 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Fri, 19 Jan 2018 01:50:05 -0500 Subject: [PATCH 10/30] Stop echoing the command --- docker/bot/wait-for-db.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/bot/wait-for-db.sh b/docker/bot/wait-for-db.sh index be45d45..11417bf 100644 --- a/docker/bot/wait-for-db.sh +++ b/docker/bot/wait-for-db.sh @@ -19,5 +19,4 @@ waitForTable $@ sleep 1 shift 3; -echo $@ exec $@ From 7d3e4519320c120875eb8e16e88512b3c514c905 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Fri, 19 Jan 2018 01:56:15 -0500 Subject: [PATCH 11/30] Move all sql files to a subfolder --- bot/Database.php | 2 +- alerts.sql => db/alerts.sql | 0 balances.sql => db/balances.sql | 0 .../current_simulated_profits_rate.sql | 0 profit_loss.sql => db/profit_loss.sql | 0 profits.sql => db/profits.sql | 0 6 files changed, 1 insertion(+), 1 deletion(-) rename alerts.sql => db/alerts.sql (100%) rename balances.sql => db/balances.sql (100%) rename current_simulated_profits_rate.sql => db/current_simulated_profits_rate.sql (100%) rename profit_loss.sql => db/profit_loss.sql (100%) rename profits.sql => db/profits.sql (100%) diff --git a/bot/Database.php b/bot/Database.php index 8c538d5..bdea5e9 100644 --- a/bot/Database.php +++ b/bot/Database.php @@ -837,7 +837,7 @@ public static function createTableHelper( $name ) { $link = self::connect(); - $query = file_get_contents( __DIR__ . sprintf( '/../%s.sql', $name ) ); + $query = file_get_contents( __DIR__ . sprintf( '/../db/%s.sql', $name ) ); foreach ( explode( ';', $query ) as $q ) { $q = trim( $q ); diff --git a/alerts.sql b/db/alerts.sql similarity index 100% rename from alerts.sql rename to db/alerts.sql diff --git a/balances.sql b/db/balances.sql similarity index 100% rename from balances.sql rename to db/balances.sql diff --git a/current_simulated_profits_rate.sql b/db/current_simulated_profits_rate.sql similarity index 100% rename from current_simulated_profits_rate.sql rename to db/current_simulated_profits_rate.sql diff --git a/profit_loss.sql b/db/profit_loss.sql similarity index 100% rename from profit_loss.sql rename to db/profit_loss.sql diff --git a/profits.sql b/db/profits.sql similarity index 100% rename from profits.sql rename to db/profits.sql From 2d4cf18cc9c4ab23b8ab7104cc82f8b7d6cc2075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Drnov=C5=A1=C4=8Dek?= Date: Fri, 19 Jan 2018 14:47:41 +0100 Subject: [PATCH 12/30] Docker/nginx/php fixes --- .gitignore | 3 +++ docker-compose.yml | 4 ++-- docker/bot/ntp.sh | 0 docker/db-data/.gitignore | 2 -- docker/db/Dockerfile | 14 ++++++------- docker/php/Dockerfile | 36 ++++++++++++++++++-------------- docker/web/config/default | 22 -------------------- docker/web/config/default.conf | 38 ++++++++++++++++++++++++++++++++++ docker/web/nginx.conf | 6 +++--- 9 files changed, 73 insertions(+), 52 deletions(-) mode change 100644 => 100755 docker/bot/ntp.sh delete mode 100644 docker/db-data/.gitignore delete mode 100644 docker/web/config/default create mode 100644 docker/web/config/default.conf diff --git a/.gitignore b/.gitignore index f379d9d..b78f438 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ config.ini config.ini.[0-9]* poloniex-tradeHistory.csv web/config.inc.php +\.idea/ +\.env +/docker/db-data/ diff --git a/docker-compose.yml b/docker-compose.yml index 69eb057..175a5cf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,7 +50,7 @@ services: ports: - "80:80" volumes: - - ./web:/var/www/html + - ./:/var/www/html networks: - frontend restart: always @@ -69,7 +69,7 @@ services: - 9000 restart: always volumes: - - ./web:/var/www/html + - ./:/var/www/html networks: - frontend - backend diff --git a/docker/bot/ntp.sh b/docker/bot/ntp.sh old mode 100644 new mode 100755 diff --git a/docker/db-data/.gitignore b/docker/db-data/.gitignore deleted file mode 100644 index c96a04f..0000000 --- a/docker/db-data/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore \ No newline at end of file diff --git a/docker/db/Dockerfile b/docker/db/Dockerfile index 87676ba..34e4ab3 100644 --- a/docker/db/Dockerfile +++ b/docker/db/Dockerfile @@ -4,10 +4,10 @@ ARG db ARG db_username ARG db_password -RUN echo -n 'GRANT ALL ON ' > /docker-entrypoint-initdb.d/00-privileges.sql -RUN echo -n ${db} >> /docker-entrypoint-initdb.d/00-privileges.sql -RUN echo -n '.* TO ' >> /docker-entrypoint-initdb.d/00-privileges.sql -RUN echo -n ${db_username} >> /docker-entrypoint-initdb.d/00-privileges.sql -RUN echo -n "@'%' IDENTIFIED BY '" >> /docker-entrypoint-initdb.d/00-privileges.sql -RUN echo -n ${db_password} >> /docker-entrypoint-initdb.d/00-privileges.sql -RUN echo "';" >> /docker-entrypoint-initdb.d/00-privileges.sql +#RUN echo -n 'GRANT ALL ON ' > /docker-entrypoint-initdb.d/00-privileges.sql +#RUN echo -n ${db} >> /docker-entrypoint-initdb.d/00-privileges.sql +#RUN echo -n '.* TO ' >> /docker-entrypoint-initdb.d/00-privileges.sql +#RUN echo -n ${db_username} >> /docker-entrypoint-initdb.d/00-privileges.sql +#RUN echo -n "@'%' IDENTIFIED BY '" >> /docker-entrypoint-initdb.d/00-privileges.sql +#RUN echo -n ${db_password} >> /docker-entrypoint-initdb.d/00-privileges.sql +#RUN echo "';" >> /docker-entrypoint-initdb.d/00-privileges.sql diff --git a/docker/php/Dockerfile b/docker/php/Dockerfile index 49e5a3f..ca54052 100644 --- a/docker/php/Dockerfile +++ b/docker/php/Dockerfile @@ -2,11 +2,15 @@ FROM php:7.0-fpm RUN apt-get update && apt-get install -y apt-utils -RUN apt-get install -y \ + +RUN apt-get update && apt-get install -y \ libicu-dev unzip ntpdate libmcrypt-dev zlib1g-dev g++ ntp \ - # php-curl php-mbstring php-mcrypt php-intl php-mysqli php-zip \ && rm -r /var/lib/apt/lists/* + +RUN docker-php-ext-configure intl \ +&& docker-php-ext-install -j$(nproc) mbstring zip mcrypt bcmath intl mysqli + COPY php.ini /usr/local/etc/php/php.ini ARG uid @@ -16,21 +20,21 @@ ARG db_username ARG db_password # Autogenerate config.inc.php -RUN (test -f config.inc.php && rm -f config.inc.php) || \ +RUN (test -f /var/www/html/web/config.inc.php && rm -f /var/www/html/web/config.inc.php) || \ echo Ignorinng config.inc.php because it does not exist -RUN mkdir -p /etc/arbbot -RUN echo ' /etc/arbbot/config.inc.php -RUN echo '' >> /etc/arbbot/config.inc.php -RUN echo '$dbHost = "db"; // DO NOT MODIFY IF USING DOCKER' >> /etc/arbbot/config.inc.php -RUN echo -n '$dbName = "'>> /etc/arbbot/config.inc.php -RUN echo -n ${db} >> /etc/arbbot/config.inc.php -RUN echo '";' >> /etc/arbbot/config.inc.php -RUN echo -n '$dbUser = "' >> /etc/arbbot/config.inc.php -RUN echo -n ${db_username} >> /etc/arbbot/config.inc.php -RUN echo '";' >> /etc/arbbot/config.inc.php -RUN echo -n '$dbPass = "' >> /etc/arbbot/config.inc.php -RUN echo -n ${db_password} >> /etc/arbbot/config.inc.php -RUN echo '";' >> /etc/arbbot/config.inc.php +RUN mkdir -p /etc/arbbot \ + && echo ' /etc/arbbot/config.inc.php \ + && echo '' >> /etc/arbbot/config.inc.php \ + && echo '$dbHost = "db";' >> /etc/arbbot/config.inc.php \ + && echo -n '$dbName = "'>> /etc/arbbot/config.inc.php \ + && echo -n ${db} >> /etc/arbbot/config.inc.php \ + && echo '";' >> /etc/arbbot/config.inc.php \ + && echo -n '$dbUser = "' >> /etc/arbbot/config.inc.php \ + && echo -n ${db_username} >> /etc/arbbot/config.inc.php \ + && echo '";' >> /etc/arbbot/config.inc.php \ + && echo -n '$dbPass = "' >> /etc/arbbot/config.inc.php \ + && echo -n ${db_password} >> /etc/arbbot/config.inc.php \ + && echo '";' >> /etc/arbbot/config.inc.php # Set UID RUN (users | grep -q ${username}) || \ diff --git a/docker/web/config/default b/docker/web/config/default deleted file mode 100644 index 160622e..0000000 --- a/docker/web/config/default +++ /dev/null @@ -1,22 +0,0 @@ -server { - - listen 80; - root /var/www/html; - index index.html; - server_name localhost; - - location / { - - auth_basic "Restricted area"; - auth_basic_user_file /var/www/conf/arbitrage.passwd; - - } - - location ~ \.php$ { - - include snippets/fastcgi-php.conf; - fastcgi_pass unix:/var/run/php/php7.0-fpm.sock; - - } - -} diff --git a/docker/web/config/default.conf b/docker/web/config/default.conf new file mode 100644 index 0000000..9b76d6c --- /dev/null +++ b/docker/web/config/default.conf @@ -0,0 +1,38 @@ +server { + + listen 80; + root /var/www/html/web; + autoindex on; + index index.html; + charset utf-8; + + server_name localhost; + error_log /dev/stdout error; + access_log /dev/stdout; + + location / { + auth_basic "Restricted area"; + auth_basic_user_file /etc/nginx/arbitrage.passwd; + } + + location = /favicon.ico { access_log off; log_not_found off; } + location = /robots.txt { access_log off; log_not_found off; } + + location ~ \.php$ { + try_files $uri =404; + include fastcgi_params; + fastcgi_split_path_info ^(.+\.php)(/.+)$; + fastcgi_param SCRIPT_FILENAME $document_root/$fastcgi_script_name; + fastcgi_pass php:9000; + } + + location ~ /\. { + deny all; + } + + location ~* \.(js|css|jpg|jpeg|gif|png|svg|ico|pdf|html|htm|woff|woff2|eof)$ { + access_log off; + log_not_found off; + expires 30d; + } +} diff --git a/docker/web/nginx.conf b/docker/web/nginx.conf index 93b24da..d427861 100644 --- a/docker/web/nginx.conf +++ b/docker/web/nginx.conf @@ -1,7 +1,7 @@ user nginx; worker_processes 1; -error_log /var/log/nginx/error.log warn; +error_log /dev/stdout warn; pid /var/run/nginx.pid; events { @@ -19,8 +19,8 @@ http { '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; + access_log /dev/stdout; + error_log /dev/stdout; sendfile on; #tcp_nopush on; From 1b07b17030dbaa5bca1b14216727217759addaf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Drnov=C5=A1=C4=8Dek?= Date: Fri, 19 Jan 2018 15:04:11 +0100 Subject: [PATCH 13/30] Wrong autoindex setting --- docker/web/config/default.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/web/config/default.conf b/docker/web/config/default.conf index 9b76d6c..6d4a562 100644 --- a/docker/web/config/default.conf +++ b/docker/web/config/default.conf @@ -2,7 +2,7 @@ server { listen 80; root /var/www/html/web; - autoindex on; + autoindex off; index index.html; charset utf-8; From c29d13dc51a6a9fe383fa31ea931802107ded096 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Fri, 19 Jan 2018 19:39:17 -0500 Subject: [PATCH 14/30] Final touches --- docker-compose.yml | 7 +------ docker/bot/Dockerfile | 4 ---- docker/bot/wait-for-db.sh | 1 - docker/db/Dockerfile | 13 ------------- docker/php/Dockerfile | 3 --- docker/web/Dockerfile | 1 - docker/web/logs/.gitignore | 2 -- web/WebDB.php | 7 ++++--- 8 files changed, 5 insertions(+), 33 deletions(-) delete mode 100644 docker/db/Dockerfile delete mode 100644 docker/web/logs/.gitignore diff --git a/docker-compose.yml b/docker-compose.yml index 175a5cf..4485f4d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,12 +2,7 @@ version: '3' services: db: - build: - context: ./docker/db - args: - - db=${MYSQL_DATABASE:-arbitrage} - - db_username=${MYSQL_USER:-arbitrage} - - db_password=${MYSQL_PASSWORD:-MySqlPassword} + image: mysql:5.7 env_file: - .env volumes: diff --git a/docker/bot/Dockerfile b/docker/bot/Dockerfile index 1b8d52b..7a01f0f 100644 --- a/docker/bot/Dockerfile +++ b/docker/bot/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && apt-get install -y apt-utils RUN apt-get install -y \ libicu-dev unzip ntpdate libmcrypt-dev zlib1g-dev g++ mysql-client ntp \ - # php-curl php-mbstring php-mcrypt php-intl php-mysqli php-zip \ && rm -r /var/lib/apt/lists/* ADD wait-for-db.sh /bin/wait-for-db.sh @@ -18,12 +17,9 @@ ARG uid ARG username ARG password -# Set UID RUN (users | grep -q ${username}) || \ (useradd -s /bin/bash -m -u ${uid} ${username} \ && chown -R ${username}:${username} /home/${username} \ && echo "Using user ${username} with UID ${uid}.") WORKDIR "/home/${username}" -#CMD ["/bin/ls", "/bin"] -#CMD ["/bin/wait-for-db.sh", "profit_loss", ${password}, "hhvm", "main.php"] diff --git a/docker/bot/wait-for-db.sh b/docker/bot/wait-for-db.sh index 11417bf..869d308 100644 --- a/docker/bot/wait-for-db.sh +++ b/docker/bot/wait-for-db.sh @@ -1,5 +1,4 @@ #!/bin/bash -# wait-for-db.sh set -e sleep 10 diff --git a/docker/db/Dockerfile b/docker/db/Dockerfile deleted file mode 100644 index 34e4ab3..0000000 --- a/docker/db/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM mysql:5.7 - -ARG db -ARG db_username -ARG db_password - -#RUN echo -n 'GRANT ALL ON ' > /docker-entrypoint-initdb.d/00-privileges.sql -#RUN echo -n ${db} >> /docker-entrypoint-initdb.d/00-privileges.sql -#RUN echo -n '.* TO ' >> /docker-entrypoint-initdb.d/00-privileges.sql -#RUN echo -n ${db_username} >> /docker-entrypoint-initdb.d/00-privileges.sql -#RUN echo -n "@'%' IDENTIFIED BY '" >> /docker-entrypoint-initdb.d/00-privileges.sql -#RUN echo -n ${db_password} >> /docker-entrypoint-initdb.d/00-privileges.sql -#RUN echo "';" >> /docker-entrypoint-initdb.d/00-privileges.sql diff --git a/docker/php/Dockerfile b/docker/php/Dockerfile index ca54052..d79237d 100644 --- a/docker/php/Dockerfile +++ b/docker/php/Dockerfile @@ -20,8 +20,6 @@ ARG db_username ARG db_password # Autogenerate config.inc.php -RUN (test -f /var/www/html/web/config.inc.php && rm -f /var/www/html/web/config.inc.php) || \ - echo Ignorinng config.inc.php because it does not exist RUN mkdir -p /etc/arbbot \ && echo ' /etc/arbbot/config.inc.php \ && echo '' >> /etc/arbbot/config.inc.php \ @@ -36,7 +34,6 @@ RUN mkdir -p /etc/arbbot \ && echo -n ${db_password} >> /etc/arbbot/config.inc.php \ && echo '";' >> /etc/arbbot/config.inc.php -# Set UID RUN (users | grep -q ${username}) || \ (useradd -s /bin/bash -m -u ${uid} ${username} \ && chown -R ${username}:${username} /home/${username} \ diff --git a/docker/web/Dockerfile b/docker/web/Dockerfile index 83807c4..3793072 100644 --- a/docker/web/Dockerfile +++ b/docker/web/Dockerfile @@ -19,7 +19,6 @@ RUN echo -n ${http_auth_user}:$(openssl passwd -apr1 ${http_auth_password}) > /e ARG uid ARG username -# Set UID RUN (users | grep -q ${username}) || \ (useradd -s /bin/bash -m -u ${uid} ${username} \ && chown -R ${username}:${username} /home/${username} \ diff --git a/docker/web/logs/.gitignore b/docker/web/logs/.gitignore deleted file mode 100644 index 6ad95ef..0000000 --- a/docker/web/logs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -access.log -error.log \ No newline at end of file diff --git a/web/WebDB.php b/web/WebDB.php index 0a2b9ed..8fa5e14 100644 --- a/web/WebDB.php +++ b/web/WebDB.php @@ -1,10 +1,11 @@ Date: Fri, 19 Jan 2018 22:40:15 -0500 Subject: [PATCH 15/30] Allow customizing the port on which the HTTP server listens through the .env file --- .env.example | 3 ++- docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 89762b7..3d5f3f4 100644 --- a/.env.example +++ b/.env.example @@ -4,9 +4,10 @@ USER=bot # Username for HTTP access HTTP_AUTH_USER=arbb0t - # Password for HTTP access HTTP_AUTH_PASSWORD=arbb0tpass +# Port on which the HTTP server should listen on +HTTP_PORT=80 # MySQL related config MYSQL_ROOT_PASSWORD=root diff --git a/docker-compose.yml b/docker-compose.yml index 4485f4d..8863fb6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: - http_auth_user=${HTTP_AUTH_USER:-arbb0t} - http_auth_password=${HTTP_AUTH_PASSWORD:-arbb0tpass} ports: - - "80:80" + - "${HTTP_PORT:-80}:80" volumes: - ./:/var/www/html networks: From 69a303f6b18ac18e500d0769577b55e04fdf5c31 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Fri, 19 Jan 2018 23:52:54 -0500 Subject: [PATCH 16/30] Don't store the database configuration in config.ini This information is already stored in config.inc.php, so it's better to not duplicate it. --- bot/Config.php | 5 ----- bot/Database.php | 17 ++++++++++++----- config.ini.example | 15 --------------- docker-compose.yml | 3 +++ docker/bot/Dockerfile | 18 ++++++++++++++++++ 5 files changed, 33 insertions(+), 25 deletions(-) diff --git a/bot/Config.php b/bot/Config.php index e56a5f2..cbe143b 100644 --- a/bot/Config.php +++ b/bot/Config.php @@ -2,11 +2,6 @@ class Config { - // DATABASE - const DB_USER = 'db.user'; - const DB_PASS = 'db.pass'; - const DB_HOST = 'db.host'; - const DB_NAME = 'db.name'; // // MAIL const MAIL_RECIPIENT = 'mail.address'; diff --git a/bot/Database.php b/bot/Database.php index bdea5e9..a29146d 100644 --- a/bot/Database.php +++ b/bot/Database.php @@ -1,5 +1,13 @@ /etc/arbbot/config.inc.php \ + && echo '' >> /etc/arbbot/config.inc.php \ + && echo '$dbHost = "db";' >> /etc/arbbot/config.inc.php \ + && echo -n '$dbName = "'>> /etc/arbbot/config.inc.php \ + && echo -n ${db} >> /etc/arbbot/config.inc.php \ + && echo '";' >> /etc/arbbot/config.inc.php \ + && echo -n '$dbUser = "' >> /etc/arbbot/config.inc.php \ + && echo -n ${db_username} >> /etc/arbbot/config.inc.php \ + && echo '";' >> /etc/arbbot/config.inc.php \ + && echo -n '$dbPass = "' >> /etc/arbbot/config.inc.php \ + && echo -n ${db_password} >> /etc/arbbot/config.inc.php \ + && echo '";' >> /etc/arbbot/config.inc.php RUN (users | grep -q ${username}) || \ (useradd -s /bin/bash -m -u ${uid} ${username} \ From 6c1169304db3e717658cb240578fc233eeb72570 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Sat, 20 Jan 2018 01:00:51 -0500 Subject: [PATCH 17/30] Add a separate directory for storing the bot's config file --- README.md | 8 +++++++- bot/Config.php | 18 +++++++----------- docker-compose.yml | 2 ++ docker/bot/Dockerfile | 2 ++ docker/php/Dockerfile | 2 ++ 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index fcfec2c..b0fb5b6 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,11 @@ Customize your environment settings: ``` cp .env.example .env vi .env # edit the file to customize the variables, NEVER use the default passwords +mkdir -p docker/config +cp config.ini.example docker/config/config.ini +cd docker/config vi config.ini # edit the database settings to make them match .env +cd ../.. ``` In case you have been running the bot from the pre-dockerized versions, you probably want to import the data that the bot currently has saved in its database into the database that the new containerized bot launches. You can do so by planting a special `data.sql` file in the right place, like this: @@ -62,7 +66,9 @@ the bot enough room to trade. When you update the bot, you need to rebuild the docker containers in case they require rebuilding before rerunning the bot, you can do that with `docker-compose build`. ## Backing up the bot's data -The bot's mysql server saves its data in `docker/db-data`. You are encouraged to back up the contents of this directory occasionally. If you delete the contents of this directory, the next time you run the bot the bot will reinitialize its database from scratch (and will import an initial seed data from `data.sql` if that file exists.) +The bot's mysql server saves its configuration in `docker/config` and its data in `docker/db-data`. You are encouraged to back up the contents of these directories occasionally. If you delete the contents of these directories, the next time you run the bot the bot will reinitialize its database from scratch (and will import an initial seed data from `data.sql` if that file exists.) Please note that the bot currently requires a config file to start. + +When you use the Admin UI to modify the bot's settings, the bot saves backups of the config file before modifying `config.ini`. These backup copies are also stored in the `docker/config` directory. ## How does the bot make profit? diff --git a/bot/Config.php b/bot/Config.php index cbe143b..158f71c 100644 --- a/bot/Config.php +++ b/bot/Config.php @@ -2,6 +2,7 @@ class Config { + const CONFIG_DIR = "/var/arbbot/"; // // MAIL const MAIL_RECIPIENT = 'mail.address'; @@ -226,14 +227,9 @@ public static function get( $key, $default = null ) { public static function refresh() { - $config = @parse_ini_file( "config.ini", true ); + $config = @parse_ini_file( self::CONFIG_DIR . "/config.ini", true ); if ( !$config ) { - // The web UI accesses the Config object from ../bot, so config.ini will - // be placed in the parent directory. - $config = @parse_ini_file( "../config.ini", true ); - if ( !$config ) { - throw new Exception( "Configuration not found or invalid!" ); - } + throw new Exception( "Configuration not found or invalid!" ); } self::$config = $config; @@ -278,7 +274,7 @@ public static function getEditableKeys() { ); } - $file = file_get_contents( __DIR__ . '/../config.ini' ); + $file = file_get_contents( self::CONFIG_DIR . '/config.ini' ); $lines = array( ); if (strstr( $file, "\r\n" )) { $lines = explode( "\r\n", $file ); @@ -370,8 +366,8 @@ public static function setEditableKeys($input) { $inputs[ $item[ 'name' ] ] = $item[ 'value' ]; } - $file = file_get_contents( __DIR__ . '/../config.ini' ); - file_put_contents( __DIR__ . '/../config.ini.' . time(), $file ); + $file = file_get_contents( self::CONFIG_DIR . '/config.ini' ); + file_put_contents( self::CONFIG_DIR . '/config.ini.' . time(), $file ); $lines = array( ); $output = ''; @@ -429,7 +425,7 @@ public static function setEditableKeys($input) { } } - $bytes = file_put_contents( __DIR__ . '/../config.ini', $output ); + $bytes = file_put_contents( self::CONFIG_DIR . '/config.ini', $output ); return $bytes !== false; diff --git a/docker-compose.yml b/docker-compose.yml index 8e4fa64..d6dcbfb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,7 @@ services: - db_password=${MYSQL_PASSWORD:-MySqlPassword} volumes: - .:/home/${USER:-bot} + - ./docker/config:/var/arbbot networks: - backend restart: always @@ -68,6 +69,7 @@ services: restart: always volumes: - ./:/var/www/html + - ./docker/config:/var/arbbot networks: - frontend - backend diff --git a/docker/bot/Dockerfile b/docker/bot/Dockerfile index 1f90477..705a15a 100644 --- a/docker/bot/Dockerfile +++ b/docker/bot/Dockerfile @@ -40,4 +40,6 @@ RUN (users | grep -q ${username}) || \ && chown -R ${username}:${username} /home/${username} \ && echo "Using user ${username} with UID ${uid}.") +RUN mkdir -p /var/arbbot && chown -R ${username}:${username} /var/arbbot + WORKDIR "/home/${username}" diff --git a/docker/php/Dockerfile b/docker/php/Dockerfile index d79237d..a172ebe 100644 --- a/docker/php/Dockerfile +++ b/docker/php/Dockerfile @@ -39,4 +39,6 @@ RUN (users | grep -q ${username}) || \ && chown -R ${username}:${username} /home/${username} \ && echo "Using user ${username} with UID ${uid}.") +RUN mkdir -p /var/arbbot && chown -R ${username}:${username} /var/arbbot + CMD ["php-fpm"] From 4980865be9dbd9cdf4024aab1ea01ce20ef823f0 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Sat, 20 Jan 2018 01:15:48 -0500 Subject: [PATCH 18/30] Add a wrapper arbbot.sh script --- docker-compose.yml | 2 +- docker/bot/Dockerfile | 2 ++ docker/bot/arbbot.sh | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 docker/bot/arbbot.sh diff --git a/docker-compose.yml b/docker-compose.yml index d6dcbfb..becff19 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,7 @@ services: networks: - backend restart: always - command: ["/bin/wait-for-db.sh", "current_simulated_profit_rate", "${MYSQL_DATABASE:-arbitrage}", "${MYSQL_ROOT_PASSWORD:-root}", "/usr/bin/hhvm", "main.php"] + command: ["/bin/arbbot.sh", "${MYSQL_DATABASE:-arbitrage}", "${MYSQL_ROOT_PASSWORD:-root}"] web: depends_on: - db diff --git a/docker/bot/Dockerfile b/docker/bot/Dockerfile index 705a15a..cb14b5b 100644 --- a/docker/bot/Dockerfile +++ b/docker/bot/Dockerfile @@ -6,7 +6,9 @@ RUN apt-get install -y \ libicu-dev unzip ntpdate libmcrypt-dev zlib1g-dev g++ mysql-client ntp \ && rm -r /var/lib/apt/lists/* +ADD arbbot.sh /bin/arbbot.sh ADD wait-for-db.sh /bin/wait-for-db.sh +RUN chmod +x /bin/arbbot.sh RUN chmod +x /bin/wait-for-db.sh ADD ntp.sh /bin/ntp.sh RUN /bin/ntp.sh diff --git a/docker/bot/arbbot.sh b/docker/bot/arbbot.sh new file mode 100644 index 0000000..02deeb1 --- /dev/null +++ b/docker/bot/arbbot.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +/bin/wait-for-db.sh current_simulated_profit_rate "$1" "$2" /usr/bin/hhvm main.php \ No newline at end of file From d4d6dc1e57e58876692487bbbb4f16bfffc35a3e Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Sat, 20 Jan 2018 01:35:58 -0500 Subject: [PATCH 19/30] Remove config.inc.php --- .gitignore | 1 - bot/Database.php | 9 +++------ bot/utils.php | 9 +++++++++ docker-compose.yml | 8 ++------ docker/bot/Dockerfile | 18 ------------------ docker/php/Dockerfile | 18 ------------------ web/WebDB.php | 8 ++------ web/config.inc.php.example | 6 ------ 8 files changed, 16 insertions(+), 61 deletions(-) delete mode 100644 web/config.inc.php.example diff --git a/.gitignore b/.gitignore index b78f438..457751e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ bittrex-fullOrders.csv config.ini config.ini.[0-9]* poloniex-tradeHistory.csv -web/config.inc.php \.idea/ \.env /docker/db-data/ diff --git a/bot/Database.php b/bot/Database.php index a29146d..48f9753 100644 --- a/bot/Database.php +++ b/bot/Database.php @@ -1,11 +1,6 @@ /etc/arbbot/config.inc.php \ - && echo '' >> /etc/arbbot/config.inc.php \ - && echo '$dbHost = "db";' >> /etc/arbbot/config.inc.php \ - && echo -n '$dbName = "'>> /etc/arbbot/config.inc.php \ - && echo -n ${db} >> /etc/arbbot/config.inc.php \ - && echo '";' >> /etc/arbbot/config.inc.php \ - && echo -n '$dbUser = "' >> /etc/arbbot/config.inc.php \ - && echo -n ${db_username} >> /etc/arbbot/config.inc.php \ - && echo '";' >> /etc/arbbot/config.inc.php \ - && echo -n '$dbPass = "' >> /etc/arbbot/config.inc.php \ - && echo -n ${db_password} >> /etc/arbbot/config.inc.php \ - && echo '";' >> /etc/arbbot/config.inc.php RUN (users | grep -q ${username}) || \ (useradd -s /bin/bash -m -u ${uid} ${username} \ diff --git a/docker/php/Dockerfile b/docker/php/Dockerfile index a172ebe..f2658d1 100644 --- a/docker/php/Dockerfile +++ b/docker/php/Dockerfile @@ -15,24 +15,6 @@ COPY php.ini /usr/local/etc/php/php.ini ARG uid ARG username -ARG db -ARG db_username -ARG db_password - -# Autogenerate config.inc.php -RUN mkdir -p /etc/arbbot \ - && echo ' /etc/arbbot/config.inc.php \ - && echo '' >> /etc/arbbot/config.inc.php \ - && echo '$dbHost = "db";' >> /etc/arbbot/config.inc.php \ - && echo -n '$dbName = "'>> /etc/arbbot/config.inc.php \ - && echo -n ${db} >> /etc/arbbot/config.inc.php \ - && echo '";' >> /etc/arbbot/config.inc.php \ - && echo -n '$dbUser = "' >> /etc/arbbot/config.inc.php \ - && echo -n ${db_username} >> /etc/arbbot/config.inc.php \ - && echo '";' >> /etc/arbbot/config.inc.php \ - && echo -n '$dbPass = "' >> /etc/arbbot/config.inc.php \ - && echo -n ${db_password} >> /etc/arbbot/config.inc.php \ - && echo '";' >> /etc/arbbot/config.inc.php RUN (users | grep -q ${username}) || \ (useradd -s /bin/bash -m -u ${uid} ${username} \ diff --git a/web/WebDB.php b/web/WebDB.php index 8fa5e14..9d86793 100644 --- a/web/WebDB.php +++ b/web/WebDB.php @@ -1,12 +1,6 @@ Date: Sat, 20 Jan 2018 02:08:51 -0500 Subject: [PATCH 20/30] Remove the old P&L rewrite import code --- .gitignore | 2 - bot/Database.php | 12 --- bot/Exchange.php | 4 +- bot/TradeMatcher.php | 2 +- bot/xchange/BITTREX-csv-missing.txt | 13 ---- bot/xchange/Bittrex.php | 65 +--------------- bot/xchange/Bleutrade.php | 12 +-- bot/xchange/POLONIEX-csv-missing.txt | 13 ---- bot/xchange/Poloniex.php | 73 ++---------------- import-profit-loss-prompt.txt | 9 --- import-profit-loss.php | 106 --------------------------- main.php | 18 ----- 12 files changed, 10 insertions(+), 319 deletions(-) delete mode 100644 bot/xchange/BITTREX-csv-missing.txt delete mode 100644 bot/xchange/POLONIEX-csv-missing.txt delete mode 100644 import-profit-loss-prompt.txt delete mode 100644 import-profit-loss.php diff --git a/.gitignore b/.gitignore index 457751e..1ef1d32 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,5 @@ -bittrex-fullOrders.csv config.ini config.ini.[0-9]* -poloniex-tradeHistory.csv \.idea/ \.env /docker/db-data/ diff --git a/bot/Database.php b/bot/Database.php index 48f9753..2bd2cea 100644 --- a/bot/Database.php +++ b/bot/Database.php @@ -1089,18 +1089,6 @@ public static function recordProfit( $amount, $currency, $address, $created ) { } - public static function profitLossTableExists() { - - return self::tableExistsHelper( 'profit_loss' ); - - } - - public static function createProfitLossTable() { - - return self::createTableHelper( 'profit_loss' ); - - } - private static function ensureTradesUpdated( $link ) { if ( !is_null( self::$trades ) ) { diff --git a/bot/Exchange.php b/bot/Exchange.php index 9a0384e..3a3fd91 100644 --- a/bot/Exchange.php +++ b/bot/Exchange.php @@ -224,7 +224,7 @@ public abstract function cancelOrder( $orderID ); public abstract function getFilledOrderPrice( $type, $tradeable, $currency, $orderID ); - public abstract function queryTradeHistory( $options = array( ), $recentOnly = false ); + public abstract function queryTradeHistory( $options = array( ) ); public abstract function cancelAllOrders(); @@ -242,8 +242,6 @@ public abstract function getID(); public abstract function getName(); - public abstract function getTradeHistoryCSVName(); - public abstract function testAccess(); public abstract function getWalletsConsideringPendingDeposits(); diff --git a/bot/TradeMatcher.php b/bot/TradeMatcher.php index ac1bd94..29ae4cf 100644 --- a/bot/TradeMatcher.php +++ b/bot/TradeMatcher.php @@ -23,7 +23,7 @@ public function getExchangeNewTrades( $id ) { return array( ); } $ex = &$this->exchanges[ $id ]; - $hist = $ex->queryTradeHistory( array( ), true ); + $hist = $ex->queryTradeHistory( array( ) ); $tradeIDs = array( ); $map = array( ); foreach ( $hist as $market => &$data ) { diff --git a/bot/xchange/BITTREX-csv-missing.txt b/bot/xchange/BITTREX-csv-missing.txt deleted file mode 100644 index cc7bdc8..0000000 --- a/bot/xchange/BITTREX-csv-missing.txt +++ /dev/null @@ -1,13 +0,0 @@ -It seems that there are some new trades on Bittrex that the bot is not aware -of previously, so it needs to refresh its trades database from Bittrex, but it -can't find a `bittrex-fullOrders.csv' file in the installation directory. - -This file needs to be placed next to config.ini, and its name is case sensitive. -Please download this file from Bittrex and place it in the installation -directory and place Enter when ready. - -This file can be downloaded after logging in to your account from the following -address by pressing the "Load All" button and then pressing Submit. - - * https://bittrex.com/History - diff --git a/bot/xchange/Bittrex.php b/bot/xchange/Bittrex.php index 8866ebd..ea8adcf 100644 --- a/bot/xchange/Bittrex.php +++ b/bot/xchange/Bittrex.php @@ -54,7 +54,7 @@ public function getFilledOrderPrice( $type, $tradeable, $currency, $id ) { return null; } - public function queryTradeHistory( $options = array( ), $recentOnly = false ) { + public function queryTradeHistory( $options = array( ) ) { $results = array( ); $type_map = array( @@ -62,51 +62,8 @@ public function queryTradeHistory( $options = array( ), $recentOnly = false ) { 'LIMIT_SELL' => 'sell', ); - if (!$recentOnly && $this->fullOrderHistory !== null) { - $results = $this->fullOrderHistory; - } else if (!$recentOnly && !$this->fullOrderHistory && - file_exists( __DIR__ . '/../../bittrex-fullOrders.csv' )) { - $file = file_get_contents( __DIR__ . '/../../bittrex-fullOrders.csv' ); - $file = iconv( 'utf-16', 'utf-8', $file ); - $lines = explode( "\r\n", $file ); - $first = true; - foreach ($lines as $line) { - if ($first) { - // Ignore the first line. - $first = false; - continue; - } - $data = str_getcsv( $line ); - if (count( $data ) != 9) { - continue; - } - $market = $data[ 1 ]; - $arr = explode( '-', $market ); - $currency = $arr[ 0 ]; - $tradeable = $arr[ 1 ]; - $market = "${currency}_${tradeable}"; - $amount = $data[ 3 ]; - $feeFactor = ($data[ 2 ] == 'LIMIT_SELL') ? -1 : 1; - $results[ $market ][] = array( - 'rawID' => $data[ 0 ], - 'id' => $data[ 0 ], - 'currency' => $currency, - 'tradeable' => $tradeable, - 'type' => $type_map[ $data[ 2 ] ], - 'time' => strtotime( $data[ 7 ] ), - 'rate' => $data[ 6 ] / $amount, - 'amount' => $amount, - 'fee' => $feeFactor * $data[ 5 ], - 'total' => $data[ 6 ], - ); - } - $this->fullOrderHistory = $results; - } - $result = $this->queryAPI( 'account/getorderhistory' ); - $checkArray = !empty( $results ); - foreach ($result as $row) { $market = $row[ 'Exchange' ]; $arr = explode( '-', $market ); @@ -119,20 +76,6 @@ public function queryTradeHistory( $options = array( ), $recentOnly = false ) { $amount = $row[ 'Quantity' ] - $row[ 'QuantityRemaining' ]; $feeFactor = ($row[ 'OrderType' ] == 'LIMIT_SELL') ? -1 : 1; - if ($checkArray) { - $seen = false; - foreach ($results[ $market ] as $item) { - if ($item[ 'rawID' ] == $row[ 'OrderUuid' ]) { - // We have already recorder this ID. - $seen = true; - break; - } - } - if ($seen) { - continue; - } - } - $results[ $market ][] = array( 'rawID' => $row[ 'OrderUuid' ], 'id' => $row[ 'OrderUuid' ], @@ -188,12 +131,6 @@ public function getName() { } - public function getTradeHistoryCSVName() { - - return "bittrex-fullOrders.csv"; - - } - protected function getPublicURL() { return Bittrex::PUBLIC_URL; diff --git a/bot/xchange/Bleutrade.php b/bot/xchange/Bleutrade.php index f81794d..7835a6b 100644 --- a/bot/xchange/Bleutrade.php +++ b/bot/xchange/Bleutrade.php @@ -63,12 +63,9 @@ public function getFilledOrderPrice( $type, $tradeable, $currency, $id ) { } - public function queryTradeHistory( $options = array( ), $recentOnly = false ) { + public function queryTradeHistory( $options = array( ) ) { $results = array( ); - // Since this exchange was added after merging of the pl-rewrite branch, we don't - // need the full trade history for the initial import, so we can ignore $recentOnly! - $options = array( 'market' => 'ALL', 'orderstatus' => 'ALL', @@ -153,13 +150,6 @@ public function getName() { } - public function getTradeHistoryCSVName() { - - // See the comment in queryTradeHistory(). - return null; - - } - protected function getPublicURL() { return Bleutrade::PUBLIC_URL; diff --git a/bot/xchange/POLONIEX-csv-missing.txt b/bot/xchange/POLONIEX-csv-missing.txt deleted file mode 100644 index 985c71f..0000000 --- a/bot/xchange/POLONIEX-csv-missing.txt +++ /dev/null @@ -1,13 +0,0 @@ -It seems that there are some new trades on Poloniex that the bot is not aware -of previously, so it needs to refresh its trades database from Poloniex, but it -can't find a `poloniex-tradeHistory.csv' file in the installation directory. - -This file needs to be placed next to config.ini, and its name is case sensitive. -Please download this file from Poloniex and place it in the installation -directory and place Enter when ready. - -This file can be downloaded after logging in to your account from the following -address by clicking on the "Complete Trade History" link. - - * https://poloniex.com/tradeHistory - diff --git a/bot/xchange/Poloniex.php b/bot/xchange/Poloniex.php index bfa185b..c9ca558 100644 --- a/bot/xchange/Poloniex.php +++ b/bot/xchange/Poloniex.php @@ -140,55 +140,14 @@ public function getFilledOrderPrice( $type, $tradeable, $currency, $id ) { return null; } - public function queryTradeHistory( $options = array( ), $recentOnly = false ) { + public function queryTradeHistory( $options = array( ) ) { $results = array( ); - if (!$recentOnly && $this->fullOrderHistory !== null) { - $results = $this->fullOrderHistory; - } else if (!$recentOnly && !$this->fullOrderHistory && - file_exists( __DIR__ . '/../../poloniex-tradeHistory.csv' )) { - $file = file_get_contents( __DIR__ . '/../../poloniex-tradeHistory.csv' ); - $lines = explode( "\n", $file ); - $first = true; - foreach ($lines as $line) { - if ($first) { - // Ignore the first line. - $first = false; - continue; - } - $data = str_getcsv( $line ); - if (count( $data ) != 11) { - continue; - } - $market = $data[ 1 ]; - $arr = explode( '/', $market ); - $currency = $arr[ 1 ]; - $tradeable = $arr[ 0 ]; - $market = "${currency}_${tradeable}"; - $feeFactor = ($data[ 3 ] == 'Sell') ? -1 : 1; - $results[ $market ][] = array( - 'rawID' => $data[ 8 ], - 'id' => $currency . '_' . $tradeable . ':' . $data[ 8 ], - 'currency' => $currency, - 'tradeable' => $tradeable, - 'type' => strtolower( $data[ 3 ] ), - 'time' => strtotime( $data[ 0 ] ), - 'rate' => floatval( $data[ 4 ] ), - 'amount' => floatval( $data[ 5 ] ), - 'fee' => floatval( $data[ 6 ] ) * ($feeFactor * floatval( $data[ 7 ] )), - 'total' => floatval( $data[ 6 ] ), - ); - } - $this->fullOrderHistory = $results; - } - if (!in_array( 'currencyPair', $options )) { $options[ 'currencyPair' ] = 'all'; } $history = $this->queryAPI( 'returnTradeHistory', $options ); - $checkArray = !empty( $results ); - $idsSeen = array( ); foreach (array_keys($history) as $market) { @@ -196,31 +155,17 @@ public function queryTradeHistory( $options = array( ), $recentOnly = false ) { $currency = $arr[ 0 ]; $tradeable = $arr[ 1 ]; foreach ($history[ $market ] as $row) { - if ( isset( $idsSeen[ $row[ 'orderNumber' ] ] ) ) { - // Poloniex sometimes returns duplicate results! - continue; - } - $idsSeen[ $row[ 'orderNumber' ] ] = true; + if ( isset( $idsSeen[ $row[ 'orderNumber' ] ] ) ) { + // Poloniex sometimes returns duplicate results! + continue; + } + $idsSeen[ $row[ 'orderNumber' ] ] = true; if (!in_array( $market, array_keys( $results ) )) { $results[ $market ] = array(); } $feeFactor = ($row[ 'type' ] == 'sell') ? -1 : 1; - if ($checkArray) { - $seen = false; - foreach ($results[ $market ] as $item) { - if ($item[ 'rawID' ] == $row[ 'orderNumber' ]) { - // We have already recorder this ID. - $seen = true; - break; - } - } - if ($seen) { - continue; - } - } - $results[ $market ][] = array( 'rawID' => $row[ 'orderNumber' ], 'id' => $currency . '_' . $tradeable . ':' . $row[ 'orderNumber' ], @@ -490,12 +435,6 @@ public function getName() { } - public function getTradeHistoryCSVName() { - - return "poloniex-tradeHistory.csv"; - - } - // Internal functions for querying the exchange private function queryDepositsAndWithdrawals() { diff --git a/import-profit-loss-prompt.txt b/import-profit-loss-prompt.txt deleted file mode 100644 index 3b306fc..0000000 --- a/import-profit-loss-prompt.txt +++ /dev/null @@ -1,9 +0,0 @@ -The bot has detected that you have just upgraded to a new version which -supports storing the profit&loss information in the database which enables -faster retrieval times. - -We now need to do a one time conversion of your current data to store it -in the database, and from now on your new trade's data will be stored in -the database automatically. - -Press enter when you are ready to start. This operation may take a while. diff --git a/import-profit-loss.php b/import-profit-loss.php deleted file mode 100644 index d0de8b5..0000000 --- a/import-profit-loss.php +++ /dev/null @@ -1,106 +0,0 @@ -refreshWallets(); - $x2->refreshWallets(); - $x1->refreshExchangeData(); - $x2->refreshExchangeData(); - $exchanges = array( - '1' => $x2, - '3' => $x1, - ); - - $hist1 = $x1->queryTradeHistory(); - $hist2 = $x2->queryTradeHistory(array( - 'start' => strtotime( '1/1/1970 1:1:1' ), - 'end' => time(), - )); - $histories = array( - '1' => &$hist2, - '3' => &$hist1, - ); - - foreach ( $pl as $row ) { - $coin = $row[ 'coin' ]; - $currency = $row[ 'currency' ]; - $time = $row[ 'time' ]; - $sourceExchange = $row[ 'source_exchange' ]; - $targetExchange = $row[ 'target_exchange' ]; - $rawTradeIDsBuy = array( ); - $tradeIDsBuy = array( ); - $rawTradeIDsSell = array( ); - $tradeIDsSell = array( ); - $rateTimesAmountBuy = 0; - $rateTimesAmountSell = 0; - $tradeableBought = 0; - $tradeableSold = 0; - $currencyBought = 0; - $currencySold = 0; - $currencyRevenue = 0; - $currencyProfitLoss = 0; - $tradeableTransferFee = 0; - $currencyTransferFee = 0; - $buyFee = 0; - $sellFee = 0; - - $ex = $exchanges[ $row[ 'source_exchange' ] ]; - $rawTradeIDsBuy = ''; - $tradeIDsBuy = ''; - $rateTimesAmountBuy = $row[ 'currency_bought' ]; - $tradeableBought = $ex->deductFeeFromAmountSell( $row[ 'tradeable_bought' ], $coin, $currency ); - $currencyBought = $ex->deductFeeFromAmountSell( $row[ 'currency_bought' ], $coin, $currency ); - $boughtAmount = $ex->deductFeeFromAmountSell( $row[ 'amount_bought_tradeable' ], $coin, $currency ); - $buyFee = $currencyBought - $ex->addFeeToPrice( $currencyBought, $coin, $currency ); - $rateBuy = ($tradeableBought > 0.0e-9) ? ($rateTimesAmountBuy / $tradeableBought) : 0; - - $ex = $exchanges[ $row[ 'target_exchange' ] ]; - $rawTradeIDsSell = ''; - $tradeIDsSell = ''; - $rateTimesAmountSell = $row[ 'currency_sold' ]; - $tradeableSold = $row[ 'tradeable_sold' ]; - $currencySold = $row[ 'currency_sold' ]; - $soldAmount = $row[ 'amount_sold_tradeable' ]; - $sellFee = $row[ 'currency_sold' ] - $ex->addFeeToPrice( $row[ 'currency_sold' ], $coin, $currency ); - $rateSell = ($tradeableSold > 0.0e-9) ? ($rateTimesAmountSell / $tradeableSold) : 0; - - $tradeableTransferFee = $row[ 'tx_fee_tradeable' ]; - $currencyTransferFee = $tradeableTransferFee * $rateSell; - $currencyRevenue = $row[ 'currency_revenue' ]; - $currencyProfitLoss = $currencyRevenue - $currencyTransferFee; - - Database::saveProfitLoss( $coin, $currency, $time, $sourceExchange, $targetExchange, - $rawTradeIDsBuy, $tradeIDsBuy, $rawTradeIDsSell, $tradeIDsSell, - $rateBuy, $rateSell, $tradeableBought, $tradeableSold, $currencyBought, - $currencySold, $currencyRevenue, $currencyProfitLoss, $tradeableTransferFee, - $currencyTransferFee, $buyFee, $sellFee ); - - } -} - diff --git a/main.php b/main.php index 4f9dab2..247a1f6 100644 --- a/main.php +++ b/main.php @@ -66,11 +66,6 @@ Database::importProfits(); } -if ( !Database::profitLossTableExists() ) { - require_once __DIR__ . '/import-profit-loss.php'; - importProfitLoss(); -} - // Configure exchanges... $exchanges = [ ]; $msg = ''; @@ -119,19 +114,6 @@ logg( "Noticed new trades on " . $exchange->getName() . " that we haven't seen before, importing them now..." ); - $name = $exchange->getTradeHistoryCSVName(); - if ( $name ) { - $csvPath = __DIR__ . '/' . $name; - if ( ! is_readable( $csvPath ) ) { - $prompt = file_get_contents( __DIR__ . '/bot/xchange/' . $exchange->getName() . '-csv-missing.txt' ); - logg( $prompt ); - readline(); - if ( !is_readable( $csvPath ) ) { - die( "Still can't find the file, refusing to continue\n" ); - } - } - } - // Now read the full history $hist = $exchange->queryTradeHistory(); From d945ef85fe6649dc7a95bb8ea986396c5b0e0656 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Sat, 20 Jan 2018 20:51:11 -0500 Subject: [PATCH 21/30] Revert parts of the previous commit since Poloniex and Bittrex still need global knowledge about the trades --- .gitignore | 2 + bot/Exchange.php | 4 +- bot/TradeMatcher.php | 2 +- bot/xchange/BITTREX-csv-missing.txt | 13 +++++ bot/xchange/Bittrex.php | 65 ++++++++++++++++++++++++- bot/xchange/Bleutrade.php | 12 ++++- bot/xchange/POLONIEX-csv-missing.txt | 13 +++++ bot/xchange/Poloniex.php | 73 +++++++++++++++++++++++++--- main.php | 13 +++++ 9 files changed, 187 insertions(+), 10 deletions(-) create mode 100644 bot/xchange/BITTREX-csv-missing.txt create mode 100644 bot/xchange/POLONIEX-csv-missing.txt diff --git a/.gitignore b/.gitignore index 1ef1d32..457751e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ +bittrex-fullOrders.csv config.ini config.ini.[0-9]* +poloniex-tradeHistory.csv \.idea/ \.env /docker/db-data/ diff --git a/bot/Exchange.php b/bot/Exchange.php index 3a3fd91..9a0384e 100644 --- a/bot/Exchange.php +++ b/bot/Exchange.php @@ -224,7 +224,7 @@ public abstract function cancelOrder( $orderID ); public abstract function getFilledOrderPrice( $type, $tradeable, $currency, $orderID ); - public abstract function queryTradeHistory( $options = array( ) ); + public abstract function queryTradeHistory( $options = array( ), $recentOnly = false ); public abstract function cancelAllOrders(); @@ -242,6 +242,8 @@ public abstract function getID(); public abstract function getName(); + public abstract function getTradeHistoryCSVName(); + public abstract function testAccess(); public abstract function getWalletsConsideringPendingDeposits(); diff --git a/bot/TradeMatcher.php b/bot/TradeMatcher.php index 29ae4cf..ac1bd94 100644 --- a/bot/TradeMatcher.php +++ b/bot/TradeMatcher.php @@ -23,7 +23,7 @@ public function getExchangeNewTrades( $id ) { return array( ); } $ex = &$this->exchanges[ $id ]; - $hist = $ex->queryTradeHistory( array( ) ); + $hist = $ex->queryTradeHistory( array( ), true ); $tradeIDs = array( ); $map = array( ); foreach ( $hist as $market => &$data ) { diff --git a/bot/xchange/BITTREX-csv-missing.txt b/bot/xchange/BITTREX-csv-missing.txt new file mode 100644 index 0000000..cc7bdc8 --- /dev/null +++ b/bot/xchange/BITTREX-csv-missing.txt @@ -0,0 +1,13 @@ +It seems that there are some new trades on Bittrex that the bot is not aware +of previously, so it needs to refresh its trades database from Bittrex, but it +can't find a `bittrex-fullOrders.csv' file in the installation directory. + +This file needs to be placed next to config.ini, and its name is case sensitive. +Please download this file from Bittrex and place it in the installation +directory and place Enter when ready. + +This file can be downloaded after logging in to your account from the following +address by pressing the "Load All" button and then pressing Submit. + + * https://bittrex.com/History + diff --git a/bot/xchange/Bittrex.php b/bot/xchange/Bittrex.php index ea8adcf..8866ebd 100644 --- a/bot/xchange/Bittrex.php +++ b/bot/xchange/Bittrex.php @@ -54,7 +54,7 @@ public function getFilledOrderPrice( $type, $tradeable, $currency, $id ) { return null; } - public function queryTradeHistory( $options = array( ) ) { + public function queryTradeHistory( $options = array( ), $recentOnly = false ) { $results = array( ); $type_map = array( @@ -62,8 +62,51 @@ public function queryTradeHistory( $options = array( ) ) { 'LIMIT_SELL' => 'sell', ); + if (!$recentOnly && $this->fullOrderHistory !== null) { + $results = $this->fullOrderHistory; + } else if (!$recentOnly && !$this->fullOrderHistory && + file_exists( __DIR__ . '/../../bittrex-fullOrders.csv' )) { + $file = file_get_contents( __DIR__ . '/../../bittrex-fullOrders.csv' ); + $file = iconv( 'utf-16', 'utf-8', $file ); + $lines = explode( "\r\n", $file ); + $first = true; + foreach ($lines as $line) { + if ($first) { + // Ignore the first line. + $first = false; + continue; + } + $data = str_getcsv( $line ); + if (count( $data ) != 9) { + continue; + } + $market = $data[ 1 ]; + $arr = explode( '-', $market ); + $currency = $arr[ 0 ]; + $tradeable = $arr[ 1 ]; + $market = "${currency}_${tradeable}"; + $amount = $data[ 3 ]; + $feeFactor = ($data[ 2 ] == 'LIMIT_SELL') ? -1 : 1; + $results[ $market ][] = array( + 'rawID' => $data[ 0 ], + 'id' => $data[ 0 ], + 'currency' => $currency, + 'tradeable' => $tradeable, + 'type' => $type_map[ $data[ 2 ] ], + 'time' => strtotime( $data[ 7 ] ), + 'rate' => $data[ 6 ] / $amount, + 'amount' => $amount, + 'fee' => $feeFactor * $data[ 5 ], + 'total' => $data[ 6 ], + ); + } + $this->fullOrderHistory = $results; + } + $result = $this->queryAPI( 'account/getorderhistory' ); + $checkArray = !empty( $results ); + foreach ($result as $row) { $market = $row[ 'Exchange' ]; $arr = explode( '-', $market ); @@ -76,6 +119,20 @@ public function queryTradeHistory( $options = array( ) ) { $amount = $row[ 'Quantity' ] - $row[ 'QuantityRemaining' ]; $feeFactor = ($row[ 'OrderType' ] == 'LIMIT_SELL') ? -1 : 1; + if ($checkArray) { + $seen = false; + foreach ($results[ $market ] as $item) { + if ($item[ 'rawID' ] == $row[ 'OrderUuid' ]) { + // We have already recorder this ID. + $seen = true; + break; + } + } + if ($seen) { + continue; + } + } + $results[ $market ][] = array( 'rawID' => $row[ 'OrderUuid' ], 'id' => $row[ 'OrderUuid' ], @@ -131,6 +188,12 @@ public function getName() { } + public function getTradeHistoryCSVName() { + + return "bittrex-fullOrders.csv"; + + } + protected function getPublicURL() { return Bittrex::PUBLIC_URL; diff --git a/bot/xchange/Bleutrade.php b/bot/xchange/Bleutrade.php index 7835a6b..f81794d 100644 --- a/bot/xchange/Bleutrade.php +++ b/bot/xchange/Bleutrade.php @@ -63,9 +63,12 @@ public function getFilledOrderPrice( $type, $tradeable, $currency, $id ) { } - public function queryTradeHistory( $options = array( ) ) { + public function queryTradeHistory( $options = array( ), $recentOnly = false ) { $results = array( ); + // Since this exchange was added after merging of the pl-rewrite branch, we don't + // need the full trade history for the initial import, so we can ignore $recentOnly! + $options = array( 'market' => 'ALL', 'orderstatus' => 'ALL', @@ -150,6 +153,13 @@ public function getName() { } + public function getTradeHistoryCSVName() { + + // See the comment in queryTradeHistory(). + return null; + + } + protected function getPublicURL() { return Bleutrade::PUBLIC_URL; diff --git a/bot/xchange/POLONIEX-csv-missing.txt b/bot/xchange/POLONIEX-csv-missing.txt new file mode 100644 index 0000000..985c71f --- /dev/null +++ b/bot/xchange/POLONIEX-csv-missing.txt @@ -0,0 +1,13 @@ +It seems that there are some new trades on Poloniex that the bot is not aware +of previously, so it needs to refresh its trades database from Poloniex, but it +can't find a `poloniex-tradeHistory.csv' file in the installation directory. + +This file needs to be placed next to config.ini, and its name is case sensitive. +Please download this file from Poloniex and place it in the installation +directory and place Enter when ready. + +This file can be downloaded after logging in to your account from the following +address by clicking on the "Complete Trade History" link. + + * https://poloniex.com/tradeHistory + diff --git a/bot/xchange/Poloniex.php b/bot/xchange/Poloniex.php index c9ca558..bfa185b 100644 --- a/bot/xchange/Poloniex.php +++ b/bot/xchange/Poloniex.php @@ -140,14 +140,55 @@ public function getFilledOrderPrice( $type, $tradeable, $currency, $id ) { return null; } - public function queryTradeHistory( $options = array( ) ) { + public function queryTradeHistory( $options = array( ), $recentOnly = false ) { $results = array( ); + if (!$recentOnly && $this->fullOrderHistory !== null) { + $results = $this->fullOrderHistory; + } else if (!$recentOnly && !$this->fullOrderHistory && + file_exists( __DIR__ . '/../../poloniex-tradeHistory.csv' )) { + $file = file_get_contents( __DIR__ . '/../../poloniex-tradeHistory.csv' ); + $lines = explode( "\n", $file ); + $first = true; + foreach ($lines as $line) { + if ($first) { + // Ignore the first line. + $first = false; + continue; + } + $data = str_getcsv( $line ); + if (count( $data ) != 11) { + continue; + } + $market = $data[ 1 ]; + $arr = explode( '/', $market ); + $currency = $arr[ 1 ]; + $tradeable = $arr[ 0 ]; + $market = "${currency}_${tradeable}"; + $feeFactor = ($data[ 3 ] == 'Sell') ? -1 : 1; + $results[ $market ][] = array( + 'rawID' => $data[ 8 ], + 'id' => $currency . '_' . $tradeable . ':' . $data[ 8 ], + 'currency' => $currency, + 'tradeable' => $tradeable, + 'type' => strtolower( $data[ 3 ] ), + 'time' => strtotime( $data[ 0 ] ), + 'rate' => floatval( $data[ 4 ] ), + 'amount' => floatval( $data[ 5 ] ), + 'fee' => floatval( $data[ 6 ] ) * ($feeFactor * floatval( $data[ 7 ] )), + 'total' => floatval( $data[ 6 ] ), + ); + } + $this->fullOrderHistory = $results; + } + if (!in_array( 'currencyPair', $options )) { $options[ 'currencyPair' ] = 'all'; } $history = $this->queryAPI( 'returnTradeHistory', $options ); + $checkArray = !empty( $results ); + $idsSeen = array( ); foreach (array_keys($history) as $market) { @@ -155,17 +196,31 @@ public function queryTradeHistory( $options = array( ) ) { $currency = $arr[ 0 ]; $tradeable = $arr[ 1 ]; foreach ($history[ $market ] as $row) { - if ( isset( $idsSeen[ $row[ 'orderNumber' ] ] ) ) { - // Poloniex sometimes returns duplicate results! - continue; - } - $idsSeen[ $row[ 'orderNumber' ] ] = true; + if ( isset( $idsSeen[ $row[ 'orderNumber' ] ] ) ) { + // Poloniex sometimes returns duplicate results! + continue; + } + $idsSeen[ $row[ 'orderNumber' ] ] = true; if (!in_array( $market, array_keys( $results ) )) { $results[ $market ] = array(); } $feeFactor = ($row[ 'type' ] == 'sell') ? -1 : 1; + if ($checkArray) { + $seen = false; + foreach ($results[ $market ] as $item) { + if ($item[ 'rawID' ] == $row[ 'orderNumber' ]) { + // We have already recorder this ID. + $seen = true; + break; + } + } + if ($seen) { + continue; + } + } + $results[ $market ][] = array( 'rawID' => $row[ 'orderNumber' ], 'id' => $currency . '_' . $tradeable . ':' . $row[ 'orderNumber' ], @@ -435,6 +490,12 @@ public function getName() { } + public function getTradeHistoryCSVName() { + + return "poloniex-tradeHistory.csv"; + + } + // Internal functions for querying the exchange private function queryDepositsAndWithdrawals() { diff --git a/main.php b/main.php index 247a1f6..b47a389 100644 --- a/main.php +++ b/main.php @@ -114,6 +114,19 @@ logg( "Noticed new trades on " . $exchange->getName() . " that we haven't seen before, importing them now..." ); + $name = $exchange->getTradeHistoryCSVName(); + if ( $name ) { + $csvPath = __DIR__ . '/' . $name; + if ( ! is_readable( $csvPath ) ) { + $prompt = file_get_contents( __DIR__ . '/bot/xchange/' . $exchange->getName() . '-csv-missing.txt' ); + logg( $prompt ); + readline(); + if ( !is_readable( $csvPath ) ) { + die( "Still can't find the file, refusing to continue\n" ); + } + } + } + // Now read the full history $hist = $exchange->queryTradeHistory(); From 05ddadf29c952c4dcf20fd6865b6be6f111adde7 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Sat, 20 Jan 2018 21:00:18 -0500 Subject: [PATCH 22/30] Add a helper for getting the config directory --- bot/Config.php | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/bot/Config.php b/bot/Config.php index 158f71c..7dd63e8 100644 --- a/bot/Config.php +++ b/bot/Config.php @@ -153,6 +153,12 @@ private function __construct() { } + public function getConfigDir() { + + return self::CONFIG_DIR; + + } + public static function isCurrency( $coin ) { return $coin == 'BTC'; @@ -227,7 +233,7 @@ public static function get( $key, $default = null ) { public static function refresh() { - $config = @parse_ini_file( self::CONFIG_DIR . "/config.ini", true ); + $config = @parse_ini_file( self::getConfigDir() . "/config.ini", true ); if ( !$config ) { throw new Exception( "Configuration not found or invalid!" ); } @@ -274,7 +280,7 @@ public static function getEditableKeys() { ); } - $file = file_get_contents( self::CONFIG_DIR . '/config.ini' ); + $file = file_get_contents( self::getConfigDir() . '/config.ini' ); $lines = array( ); if (strstr( $file, "\r\n" )) { $lines = explode( "\r\n", $file ); @@ -366,8 +372,8 @@ public static function setEditableKeys($input) { $inputs[ $item[ 'name' ] ] = $item[ 'value' ]; } - $file = file_get_contents( self::CONFIG_DIR . '/config.ini' ); - file_put_contents( self::CONFIG_DIR . '/config.ini.' . time(), $file ); + $file = file_get_contents( self::getConfigDir() . '/config.ini' ); + file_put_contents( self::getConfigDir() . '/config.ini.' . time(), $file ); $lines = array( ); $output = ''; @@ -425,7 +431,7 @@ public static function setEditableKeys($input) { } } - $bytes = file_put_contents( self::CONFIG_DIR . '/config.ini', $output ); + $bytes = file_put_contents( self::getConfigDir() . '/config.ini', $output ); return $bytes !== false; From ccd2b3a8874f8a8a199926a141f55be5d24d218e Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Sat, 20 Jan 2018 21:00:59 -0500 Subject: [PATCH 23/30] Move the exchange trade history files to the config directory --- bot/xchange/BITTREX-csv-missing.txt | 4 ++-- bot/xchange/POLONIEX-csv-missing.txt | 4 ++-- main.php | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/xchange/BITTREX-csv-missing.txt b/bot/xchange/BITTREX-csv-missing.txt index cc7bdc8..dc07c55 100644 --- a/bot/xchange/BITTREX-csv-missing.txt +++ b/bot/xchange/BITTREX-csv-missing.txt @@ -1,9 +1,9 @@ It seems that there are some new trades on Bittrex that the bot is not aware of previously, so it needs to refresh its trades database from Bittrex, but it -can't find a `bittrex-fullOrders.csv' file in the installation directory. +can't find a `bittrex-fullOrders.csv' file in the docker/config directory. This file needs to be placed next to config.ini, and its name is case sensitive. -Please download this file from Bittrex and place it in the installation +Please download this file from Bittrex and place it in the docker/config directory and place Enter when ready. This file can be downloaded after logging in to your account from the following diff --git a/bot/xchange/POLONIEX-csv-missing.txt b/bot/xchange/POLONIEX-csv-missing.txt index 985c71f..66c548b 100644 --- a/bot/xchange/POLONIEX-csv-missing.txt +++ b/bot/xchange/POLONIEX-csv-missing.txt @@ -1,9 +1,9 @@ It seems that there are some new trades on Poloniex that the bot is not aware of previously, so it needs to refresh its trades database from Poloniex, but it -can't find a `poloniex-tradeHistory.csv' file in the installation directory. +can't find a `poloniex-tradeHistory.csv' file in the docker/config directory. This file needs to be placed next to config.ini, and its name is case sensitive. -Please download this file from Poloniex and place it in the installation +Please download this file from Poloniex and place it in the docker/config directory and place Enter when ready. This file can be downloaded after logging in to your account from the following diff --git a/main.php b/main.php index b47a389..15af9ed 100644 --- a/main.php +++ b/main.php @@ -116,7 +116,7 @@ $name = $exchange->getTradeHistoryCSVName(); if ( $name ) { - $csvPath = __DIR__ . '/' . $name; + $csvPath = Config::getConfigDir() . '/' . $name; if ( ! is_readable( $csvPath ) ) { $prompt = file_get_contents( __DIR__ . '/bot/xchange/' . $exchange->getName() . '-csv-missing.txt' ); logg( $prompt ); From a051dcdd910bab2d1482494e9295df6cce7be7aa Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Sat, 20 Jan 2018 21:06:54 -0500 Subject: [PATCH 24/30] Make getConfigDir() static --- bot/Config.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/Config.php b/bot/Config.php index 7dd63e8..aef6bec 100644 --- a/bot/Config.php +++ b/bot/Config.php @@ -153,7 +153,7 @@ private function __construct() { } - public function getConfigDir() { + public static function getConfigDir() { return self::CONFIG_DIR; From 8db067b8c8b7113cd6782639ba9a2d480e8304ae Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Sun, 21 Jan 2018 01:34:34 -0500 Subject: [PATCH 25/30] Add the docker-compose helpers for TLS deployment --- .env.ssl.example | 5 + docker-compose-ssl.yml | 39 +++++ docker/web-config-gen/nginx.tmpl | 237 +++++++++++++++++++++++++++++++ 3 files changed, 281 insertions(+) create mode 100644 .env.ssl.example create mode 100644 docker-compose-ssl.yml create mode 100644 docker/web-config-gen/nginx.tmpl diff --git a/.env.ssl.example b/.env.ssl.example new file mode 100644 index 0000000..c82811a --- /dev/null +++ b/.env.ssl.example @@ -0,0 +1,5 @@ +# TLS Deployment using Let's Encrypt (requires a domain name pointing to your server/cluster's IP address) +VIRTUAL_HOST=arbbot.example.com # Your Domain Name +LETSENCRYPT_HOST=arbbot.example.com # Your Domain Name +LETSENCRYPT_EMAIL=arbbot@example.com # Email Address Let's Encrypt Uses For Sending Expiration Notices + # (highly suggested to fill in a valid email address.) \ No newline at end of file diff --git a/docker-compose-ssl.yml b/docker-compose-ssl.yml new file mode 100644 index 0000000..3a1a0f6 --- /dev/null +++ b/docker-compose-ssl.yml @@ -0,0 +1,39 @@ +version: '2' + +# Based on https://github.com/gilyes/docker-nginx-letsencrypt-sample/blob/master/docker-compose.yml + +services: + nginx: + restart: always + image: nginx + ports: + - "80:80" + - "443:443" + volumes: + - "/etc/nginx/conf.d" + - "/etc/nginx/vhost.d" + - "/usr/share/nginx/html" + - "./docker/certs:/etc/nginx/certs:ro" + nginx-gen: + env_file: + - .env.ssl + restart: always + image: jwilder/docker-gen + volumes: + - "/var/run/docker.sock:/tmp/docker.sock:ro" + - "./docker/nginx-gen:/etc/docker-gen/templates/nginx.tmpl:ro" + volumes_from: + - nginx + entrypoint: /usr/local/bin/docker-gen -notify-sighup nginx -watch -wait 5s:30s /etc/docker-gen/templates/nginx.tmpl /etc/nginx/conf.d/default.conf + letsencrypt-nginx-proxy-companion: + env_file: + - .env.ssl + restart: always + image: jrcs/letsencrypt-nginx-proxy-companion + volumes_from: + - nginx + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:ro" + - "./docker/certs:/etc/nginx/certs:rw" + environment: + - NGINX_DOCKER_GEN_CONTAINER=nginx-gen \ No newline at end of file diff --git a/docker/web-config-gen/nginx.tmpl b/docker/web-config-gen/nginx.tmpl new file mode 100644 index 0000000..9eb9520 --- /dev/null +++ b/docker/web-config-gen/nginx.tmpl @@ -0,0 +1,237 @@ +{{ $CurrentContainer := where $ "ID" .Docker.CurrentContainerID | first }} + +{{ define "upstream" }} + {{ if .Address }} + {{/* If we got the containers from swarm and this container's port is published to host, use host IP:PORT */}} + {{ if and .Container.Node.ID .Address.HostPort }} + # {{ .Container.Node.Name }}/{{ .Container.Name }} + server {{ .Container.Node.Address.IP }}:{{ .Address.HostPort }}; + {{/* If there is no swarm node or the port is not published on host, use container's IP:PORT */}} + {{ else if .Network }} + # {{ .Container.Name }} + server {{ .Network.IP }}:{{ .Address.Port }}; + {{ end }} + {{ else if .Network }} + # {{ .Container.Name }} + server {{ .Network.IP }} down; + {{ end }} +{{ end }} + +# If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the +# scheme used to connect to this server +map $http_x_forwarded_proto $proxy_x_forwarded_proto { + default $http_x_forwarded_proto; + '' $scheme; +} + +# If we receive Upgrade, set Connection to "upgrade"; otherwise, delete any +# Connection header that may have been passed to this server +map $http_upgrade $proxy_connection { + default upgrade; + '' close; +} + +gzip_types text/plain text/css application/javascript application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + +log_format vhost '$host $remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent"'; + +access_log off; + +{{ if (exists "/etc/nginx/proxy.conf") }} +include /etc/nginx/proxy.conf; +{{ else }} +# HTTP 1.1 support +proxy_http_version 1.1; +proxy_buffering off; +proxy_set_header Host $http_host; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $proxy_connection; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; + +# Mitigate httpoxy attack (see README for details) +proxy_set_header Proxy ""; +{{ end }} + +server { + server_name _; # This is just an invalid value which will never trigger on a real hostname. + listen 80; + access_log /var/log/nginx/access.log vhost; + return 503; +} + +{{ if (and (exists "/etc/nginx/certs/default.crt") (exists "/etc/nginx/certs/default.key")) }} +server { + server_name _; # This is just an invalid value which will never trigger on a real hostname. + listen 443 ssl http2; + access_log /var/log/nginx/access.log vhost; + return 503; + + ssl_session_tickets off; + ssl_certificate /etc/nginx/certs/default.crt; + ssl_certificate_key /etc/nginx/certs/default.key; +} +{{ end }} + +{{ range $host, $containers := groupByMulti $ "Env.VIRTUAL_HOST" "," }} + +upstream {{ $host }} { +{{ range $container := $containers }} + {{ $addrLen := len $container.Addresses }} + + {{ range $knownNetwork := $CurrentContainer.Networks }} + {{ range $containerNetwork := $container.Networks }} + {{ if eq $knownNetwork.Name $containerNetwork.Name }} + ## Can be connect with "{{ $containerNetwork.Name }}" network + + {{/* If only 1 port exposed, use that */}} + {{ if eq $addrLen 1 }} + {{ $address := index $container.Addresses 0 }} + {{ template "upstream" (dict "Container" $container "Address" $address "Network" $containerNetwork) }} + {{/* If more than one port exposed, use the one matching VIRTUAL_PORT env var, falling back to standard web port 80 */}} + {{ else }} + {{ $port := coalesce $container.Env.VIRTUAL_PORT "80" }} + {{ $address := where $container.Addresses "Port" $port | first }} + {{ template "upstream" (dict "Container" $container "Address" $address "Network" $containerNetwork) }} + {{ end }} + {{ end }} + {{ end }} + {{ end }} +{{ end }} +} + +{{ $default_host := or ($.Env.DEFAULT_HOST) "" }} +{{ $default_server := index (dict $host "" $default_host "default_server") $host }} + +{{/* Get the VIRTUAL_PROTO defined by containers w/ the same vhost, falling back to "http" */}} +{{ $proto := or (first (groupByKeys $containers "Env.VIRTUAL_PROTO")) "http" }} + +{{/* Get the HTTPS_METHOD defined by containers w/ the same vhost, falling back to "redirect" */}} +{{ $https_method := or (first (groupByKeys $containers "Env.HTTPS_METHOD")) "redirect" }} + +{{/* Get the first cert name defined by containers w/ the same vhost */}} +{{ $certName := (first (groupByKeys $containers "Env.CERT_NAME")) }} + +{{/* Get the best matching cert by name for the vhost. */}} +{{ $vhostCert := (closest (dir "/etc/nginx/certs") (printf "%s.crt" $host))}} + +{{/* vhostCert is actually a filename so remove any suffixes since they are added later */}} +{{ $vhostCert := trimSuffix ".crt" $vhostCert }} +{{ $vhostCert := trimSuffix ".key" $vhostCert }} + +{{/* Use the cert specified on the container or fallback to the best vhost match */}} +{{ $cert := (coalesce $certName $vhostCert) }} + +{{ $is_https := (and (ne $cert "") (exists (printf "/etc/nginx/certs/%s.crt" $cert)) (exists (printf "/etc/nginx/certs/%s.key" $cert))) }} + +{{ if $is_https }} + +{{ if eq $https_method "redirect" }} +server { + server_name {{ $host }}; + listen 80 {{ $default_server }}; + access_log /var/log/nginx/access.log vhost; + return 301 https://$host$request_uri; +} +{{ end }} + +server { + server_name {{ $host }}; + listen 443 ssl http2 {{ $default_server }}; + access_log /var/log/nginx/access.log vhost; + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS'; + + ssl_prefer_server_ciphers on; + ssl_session_timeout 5m; + ssl_session_cache shared:SSL:50m; + ssl_session_tickets off; + + ssl_certificate /etc/nginx/certs/{{ (printf "%s.crt" $cert) }}; + ssl_certificate_key /etc/nginx/certs/{{ (printf "%s.key" $cert) }}; + + {{ if (exists (printf "/etc/nginx/certs/%s.dhparam.pem" $cert)) }} + ssl_dhparam {{ printf "/etc/nginx/certs/%s.dhparam.pem" $cert }}; + {{ end }} + + {{ if (ne $https_method "noredirect") }} + add_header Strict-Transport-Security "max-age=31536000"; + {{ end }} + + {{ if (exists (printf "/etc/nginx/vhost.d/%s" $host)) }} + include {{ printf "/etc/nginx/vhost.d/%s" $host }}; + {{ else if (exists "/etc/nginx/vhost.d/default") }} + include /etc/nginx/vhost.d/default; + {{ end }} + + location / { + {{ if eq $proto "uwsgi" }} + include uwsgi_params; + uwsgi_pass {{ trim $proto }}://{{ trim $host }}; + {{ else }} + proxy_pass {{ trim $proto }}://{{ trim $host }}; + {{ end }} + {{ if (exists (printf "/etc/nginx/htpasswd/%s" $host)) }} + auth_basic "Restricted {{ $host }}"; + auth_basic_user_file {{ (printf "/etc/nginx/htpasswd/%s" $host) }}; + {{ end }} + {{ if (exists (printf "/etc/nginx/vhost.d/%s_location" $host)) }} + include {{ printf "/etc/nginx/vhost.d/%s_location" $host}}; + {{ else if (exists "/etc/nginx/vhost.d/default_location") }} + include /etc/nginx/vhost.d/default_location; + {{ end }} + } +} + +{{ end }} + +{{ if or (not $is_https) (eq $https_method "noredirect") }} + +server { + server_name {{ $host }}; + listen 80 {{ $default_server }}; + access_log /var/log/nginx/access.log vhost; + + {{ if (exists (printf "/etc/nginx/vhost.d/%s" $host)) }} + include {{ printf "/etc/nginx/vhost.d/%s" $host }}; + {{ else if (exists "/etc/nginx/vhost.d/default") }} + include /etc/nginx/vhost.d/default; + {{ end }} + + location / { + {{ if eq $proto "uwsgi" }} + include uwsgi_params; + uwsgi_pass {{ trim $proto }}://{{ trim $host }}; + {{ else }} + proxy_pass {{ trim $proto }}://{{ trim $host }}; + {{ end }} + {{ if (exists (printf "/etc/nginx/htpasswd/%s" $host)) }} + auth_basic "Restricted {{ $host }}"; + auth_basic_user_file {{ (printf "/etc/nginx/htpasswd/%s" $host) }}; + {{ end }} + {{ if (exists (printf "/etc/nginx/vhost.d/%s_location" $host)) }} + include {{ printf "/etc/nginx/vhost.d/%s_location" $host}}; + {{ else if (exists "/etc/nginx/vhost.d/default_location") }} + include /etc/nginx/vhost.d/default_location; + {{ end }} + } +} + +{{ if (and (not $is_https) (exists "/etc/nginx/certs/default.crt") (exists "/etc/nginx/certs/default.key")) }} +server { + server_name {{ $host }}; + listen 443 ssl http2 {{ $default_server }}; + access_log /var/log/nginx/access.log vhost; + return 500; + + ssl_certificate /etc/nginx/certs/default.crt; + ssl_certificate_key /etc/nginx/certs/default.key; +} +{{ end }} + +{{ end }} +{{ end }} From 8b3abff80a01f306e351c8fba1bf7d174869f29f Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Tue, 23 Jan 2018 02:32:10 -0500 Subject: [PATCH 26/30] Rename web-config-gen to nginx-gen --- docker/{web-config-gen => nginx-gen}/nginx.tmpl | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker/{web-config-gen => nginx-gen}/nginx.tmpl (100%) diff --git a/docker/web-config-gen/nginx.tmpl b/docker/nginx-gen/nginx.tmpl similarity index 100% rename from docker/web-config-gen/nginx.tmpl rename to docker/nginx-gen/nginx.tmpl From 6dee3578e10089a371417e0955e6a6a795bfbbe1 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Tue, 23 Jan 2018 02:33:09 -0500 Subject: [PATCH 27/30] Upgrade docker-compose-ssl.yml to version 3 --- docker-compose-ssl.yml | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/docker-compose-ssl.yml b/docker-compose-ssl.yml index 3a1a0f6..8acec47 100644 --- a/docker-compose-ssl.yml +++ b/docker-compose-ssl.yml @@ -1,4 +1,4 @@ -version: '2' +version: '3' # Based on https://github.com/gilyes/docker-nginx-letsencrypt-sample/blob/master/docker-compose.yml @@ -10,9 +10,9 @@ services: - "80:80" - "443:443" volumes: - - "/etc/nginx/conf.d" - - "/etc/nginx/vhost.d" - - "/usr/share/nginx/html" + - "confs:/etc/nginx/conf.d:ro" + - "hosts:/etc/nginx/vhost.d:ro" + - "htmls:/usr/share/nginx/html:ro" - "./docker/certs:/etc/nginx/certs:ro" nginx-gen: env_file: @@ -20,20 +20,28 @@ services: restart: always image: jwilder/docker-gen volumes: + - "confs:/etc/nginx/conf.d:rw" + - "hosts:/etc/nginx/vhost.d:rw" + - "htmls:/usr/share/nginx/html:ro" - "/var/run/docker.sock:/tmp/docker.sock:ro" + - "./docker/certs:/etc/nginx/certs:ro" - "./docker/nginx-gen:/etc/docker-gen/templates/nginx.tmpl:ro" - volumes_from: - - nginx entrypoint: /usr/local/bin/docker-gen -notify-sighup nginx -watch -wait 5s:30s /etc/docker-gen/templates/nginx.tmpl /etc/nginx/conf.d/default.conf letsencrypt-nginx-proxy-companion: env_file: - .env.ssl restart: always image: jrcs/letsencrypt-nginx-proxy-companion - volumes_from: - - nginx volumes: + - "confs:/etc/nginx/conf.d:ro" + - "hosts:/etc/nginx/vhost.d:ro" + - "htmls:/usr/share/nginx/html:ro" - "/var/run/docker.sock:/var/run/docker.sock:ro" - "./docker/certs:/etc/nginx/certs:rw" environment: - - NGINX_DOCKER_GEN_CONTAINER=nginx-gen \ No newline at end of file + - NGINX_DOCKER_GEN_CONTAINER=nginx-gen + +volumes: + confs: + hosts: + htmls: \ No newline at end of file From 74c23150ff8773131bc404c3c1181a9b5f991feb Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Tue, 23 Jan 2018 02:47:19 -0500 Subject: [PATCH 28/30] Rename the USER variable to USER_NAME --- .env.example | 2 +- docker-compose.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index 3d5f3f4..bce8fea 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,6 @@ # Linux user account information for running the bot UID=1000 -USER=bot +USER_NAME=bot # Username for HTTP access HTTP_AUTH_USER=arbb0t diff --git a/docker-compose.yml b/docker-compose.yml index 55ce9ac..a2c1e92 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,9 +22,9 @@ services: context: ./docker/bot args: - uid=${UID:-1000} - - username=${USER:-bot} + - username=${USER_NAME:-bot} volumes: - - .:/home/${USER:-bot} + - .:/home/${USER_NAME:-bot} - ./docker/config:/var/arbbot networks: - backend @@ -40,7 +40,7 @@ services: context: ./docker/web args: - uid=${UID:-1000} - - username=${USER:-bot} + - username=${USER_NAME:-bot} - http_auth_user=${HTTP_AUTH_USER:-arbb0t} - http_auth_password=${HTTP_AUTH_PASSWORD:-arbb0tpass} ports: @@ -59,7 +59,7 @@ services: context: ./docker/php args: - uid=${UID:-1000} - - username=${USER:-root} + - username=${USER_NAME:-bot} expose: - 9000 restart: always From b0f3493fd7c4d49a1dc2dd261e8f5464a2fdaa93 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Sat, 27 Jan 2018 23:44:44 -0500 Subject: [PATCH 29/30] Various fixes --- .env.example | 8 ++++++++ .env.ssl.example | 5 ----- docker-compose-ssl.yml | 34 +++++++++++++++++++++++++--------- docker-compose.yml | 8 ++++---- 4 files changed, 37 insertions(+), 18 deletions(-) delete mode 100644 .env.ssl.example diff --git a/.env.example b/.env.example index bce8fea..e91d1ce 100644 --- a/.env.example +++ b/.env.example @@ -14,3 +14,11 @@ MYSQL_ROOT_PASSWORD=root MYSQL_DATABASE=arbitrage MYSQL_USER=arbitrage MYSQL_PASSWORD=MySqlPassword + +# TLS Deployment using Let's Encrypt (requires a domain name pointing to your server/cluster's IP address) +VIRTUAL_HOST=arbbot.example.com # Your Domain Name +LETSENCRYPT_HOST=arbbot.example.com # Your Domain Name +LETSENCRYPT_EMAIL=arbbot@example.com # Email Address Let's Encrypt Uses For Sending Expiration Notices + # (highly suggested to fill in a valid email address.) +HTTPS_PORT=443 # Port on which the HTTPS server should listen on +IP=0.0.0.0 # External IP to which the server should bind to diff --git a/.env.ssl.example b/.env.ssl.example deleted file mode 100644 index c82811a..0000000 --- a/.env.ssl.example +++ /dev/null @@ -1,5 +0,0 @@ -# TLS Deployment using Let's Encrypt (requires a domain name pointing to your server/cluster's IP address) -VIRTUAL_HOST=arbbot.example.com # Your Domain Name -LETSENCRYPT_HOST=arbbot.example.com # Your Domain Name -LETSENCRYPT_EMAIL=arbbot@example.com # Email Address Let's Encrypt Uses For Sending Expiration Notices - # (highly suggested to fill in a valid email address.) \ No newline at end of file diff --git a/docker-compose-ssl.yml b/docker-compose-ssl.yml index 8acec47..37f70fe 100644 --- a/docker-compose-ssl.yml +++ b/docker-compose-ssl.yml @@ -3,21 +3,34 @@ version: '3' # Based on https://github.com/gilyes/docker-nginx-letsencrypt-sample/blob/master/docker-compose.yml services: + web: + expose: + - 80 + environment: + - VIRTUAL_HOST=${VIRTUAL_HOST:-arbbot.example.com} nginx: + container_name: nginx + depends_on: + - nginx-gen + - letsencrypt-nginx-proxy-companion restart: always image: nginx + env_file: + - .env ports: - - "80:80" - - "443:443" + - "${IP:-0.0.0.0}:${HTTP_PORT:-80}:80" + - "${IP:-0.0.0.0}:${HTTPS_PORT:-443}:443" volumes: - "confs:/etc/nginx/conf.d:ro" - "hosts:/etc/nginx/vhost.d:ro" - "htmls:/usr/share/nginx/html:ro" - "./docker/certs:/etc/nginx/certs:ro" nginx-gen: + container_name: nginx-gen + depends_on: + - letsencrypt-nginx-proxy-companion env_file: - - .env.ssl - restart: always + - .env image: jwilder/docker-gen volumes: - "confs:/etc/nginx/conf.d:rw" @@ -25,17 +38,20 @@ services: - "htmls:/usr/share/nginx/html:ro" - "/var/run/docker.sock:/tmp/docker.sock:ro" - "./docker/certs:/etc/nginx/certs:ro" - - "./docker/nginx-gen:/etc/docker-gen/templates/nginx.tmpl:ro" + - "./docker/nginx-gen/nginx.tmpl:/etc/docker-gen/templates/nginx.tmpl:ro" + networks: + - webproxy entrypoint: /usr/local/bin/docker-gen -notify-sighup nginx -watch -wait 5s:30s /etc/docker-gen/templates/nginx.tmpl /etc/nginx/conf.d/default.conf letsencrypt-nginx-proxy-companion: + container_name: letsencrypt-nginx-proxy-companion env_file: - - .env.ssl + - .env restart: always image: jrcs/letsencrypt-nginx-proxy-companion volumes: - "confs:/etc/nginx/conf.d:ro" - - "hosts:/etc/nginx/vhost.d:ro" - - "htmls:/usr/share/nginx/html:ro" + - "hosts:/etc/nginx/vhost.d:rw" + - "htmls:/usr/share/nginx/html:rw" - "/var/run/docker.sock:/var/run/docker.sock:ro" - "./docker/certs:/etc/nginx/certs:rw" environment: @@ -44,4 +60,4 @@ services: volumes: confs: hosts: - htmls: \ No newline at end of file + htmls: diff --git a/docker-compose.yml b/docker-compose.yml index a2c1e92..b1facc2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - ./docker/db/seed/data.sql:/docker-entrypoint-initdb.d/data.sql - ./docker/db-data:/var/lib/mysql networks: - backend: + - backend restart: always bot: depends_on: @@ -41,14 +41,13 @@ services: args: - uid=${UID:-1000} - username=${USER_NAME:-bot} - - http_auth_user=${HTTP_AUTH_USER:-arbb0t} + - http_auth_user=${HTTP_AUTH_USER_NAME:-arbb0t} - http_auth_password=${HTTP_AUTH_PASSWORD:-arbb0tpass} - ports: - - "${HTTP_PORT:-80}:80" volumes: - ./:/var/www/html networks: - frontend + - webproxy restart: always php: depends_on: @@ -72,3 +71,4 @@ services: networks: frontend: backend: + webproxy: From 04e8e9c8b59fab0ecca79ba04d2f4dd79c4ddfb4 Mon Sep 17 00:00:00 2001 From: cryptoeax Date: Sun, 28 Jan 2018 22:25:05 +0000 Subject: [PATCH 30/30] More various fixes --- docker-compose-ssl.yml | 12 ++++++------ docker-compose.yml | 7 +++++-- docker/bot/wait-for-db.sh | 2 +- docker/nginx-gen/Dockerfile | 5 +++++ 4 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 docker/nginx-gen/Dockerfile diff --git a/docker-compose-ssl.yml b/docker-compose-ssl.yml index 37f70fe..438bf55 100644 --- a/docker-compose-ssl.yml +++ b/docker-compose-ssl.yml @@ -31,14 +31,14 @@ services: - letsencrypt-nginx-proxy-companion env_file: - .env - image: jwilder/docker-gen + build: + context: ./docker/nginx-gen volumes: - "confs:/etc/nginx/conf.d:rw" - "hosts:/etc/nginx/vhost.d:rw" - - "htmls:/usr/share/nginx/html:ro" - - "/var/run/docker.sock:/tmp/docker.sock:ro" + - "htmls:/usr/share/nginx/html:rw" + - "/var/run/docker.sock:/tmp/docker.sock:rw" - "./docker/certs:/etc/nginx/certs:ro" - - "./docker/nginx-gen/nginx.tmpl:/etc/docker-gen/templates/nginx.tmpl:ro" networks: - webproxy entrypoint: /usr/local/bin/docker-gen -notify-sighup nginx -watch -wait 5s:30s /etc/docker-gen/templates/nginx.tmpl /etc/nginx/conf.d/default.conf @@ -49,10 +49,10 @@ services: restart: always image: jrcs/letsencrypt-nginx-proxy-companion volumes: - - "confs:/etc/nginx/conf.d:ro" + - "confs:/etc/nginx/conf.d:rw" - "hosts:/etc/nginx/vhost.d:rw" - "htmls:/usr/share/nginx/html:rw" - - "/var/run/docker.sock:/var/run/docker.sock:ro" + - "/var/run/docker.sock:/var/run/docker.sock:rw" - "./docker/certs:/etc/nginx/certs:rw" environment: - NGINX_DOCKER_GEN_CONTAINER=nginx-gen diff --git a/docker-compose.yml b/docker-compose.yml index b1facc2..972a8e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,12 +3,12 @@ version: '3' services: db: image: mysql:5.7 + container_name: db env_file: - .env volumes: - ./docker/db/config/my.cnf:/etc/mysql/conf.d/arbbot.cnf - - ./docker/db/seed/01-database.sql:/docker-entrypoint-initdb.d/01-database.sql - - ./docker/db/seed/data.sql:/docker-entrypoint-initdb.d/data.sql + - ./docker/db/seed:/docker-entrypoint-initdb.d - ./docker/db-data:/var/lib/mysql networks: - backend @@ -16,6 +16,7 @@ services: bot: depends_on: - db + container_name: bot env_file: - .env build: @@ -34,6 +35,7 @@ services: depends_on: - db - php + container_name: web env_file: - .env build: @@ -52,6 +54,7 @@ services: php: depends_on: - db + container_name: php env_file: - .env build: diff --git a/docker/bot/wait-for-db.sh b/docker/bot/wait-for-db.sh index 869d308..c9feb81 100644 --- a/docker/bot/wait-for-db.sh +++ b/docker/bot/wait-for-db.sh @@ -6,7 +6,7 @@ sleep 10 waitForTable() { table=$1; db=$2 - count=$(mysql -N -s -u root -p$3 -h db -e \ + count=$(mysql -N -s -u root '-p$3' -h db -e \ "select count(*) from information_schema.tables where \ table_schema='${db}' and table_name='${table}';" 2>/dev/null) while [ "$count" -eq "0" ]; do diff --git a/docker/nginx-gen/Dockerfile b/docker/nginx-gen/Dockerfile new file mode 100644 index 0000000..f5ef602 --- /dev/null +++ b/docker/nginx-gen/Dockerfile @@ -0,0 +1,5 @@ +FROM jwilder/docker-gen + +# Create nginx.tmpl as a file to ensure it won't get mapped as a directory +RUN mkdir -p /etc/docker-gen/templates +ADD nginx.tmpl /etc/docker-gen/templates/nginx.tmpl