From 5b6b9e32f472c2a58d00ff95f96ec228685ba5be Mon Sep 17 00:00:00 2001 From: Michael Truell Date: Sat, 3 Dec 2016 00:11:05 -0500 Subject: [PATCH 1/6] Arbitrary run.sh support --- worker/compiler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/worker/compiler.py b/worker/compiler.py index cc0abba39..ccde6c9b2 100644 --- a/worker/compiler.py +++ b/worker/compiler.py @@ -509,8 +509,9 @@ def compile_anything(bot_dir, installTimeLimit=600, timelimit=600, max_error_len print("filename:") print(run_filename) try: - with open(run_filename, 'wb') as f: - f.write(bytes('#%s\n%s\n' % (name, run_cmd), 'UTF-8')) + if os.path.isfile(run_filename) == False: + with open(run_filename, 'wb') as f: + f.write(bytes('#%s\n%s\n' % (name, run_cmd), 'UTF-8')) print("file:") with open(run_filename, 'r') as f: for line in f: From ad7a797d426c625f5403e720ca04b50d6d466681 Mon Sep 17 00:00:00 2001 From: Michael Truell Date: Sat, 3 Dec 2016 10:51:34 -0500 Subject: [PATCH 2/6] Report install.sh error --- worker/compiler.py | 47 +++++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/worker/compiler.py b/worker/compiler.py index ccde6c9b2..f5973e6c8 100644 --- a/worker/compiler.py +++ b/worker/compiler.py @@ -494,8 +494,33 @@ def get_run_lang(submission_dir): return line[1:-1] def compile_anything(bot_dir, installTimeLimit=600, timelimit=600, max_error_len = 3072): + def limitErrors(errors): + # limit length of reported errors + if len(errors) > 0 and sum(map(len, errors)) > max_error_len: + first_errors = [] + cur_error = 0 + length = len(errors[0]) + while length < (max_error_len / 3): # take 1/3 from start + first_errors.append(errors[cur_error]) + cur_error += 1 + length += len(errors[cur_error]) + first_errors.append("...") + length += 3 + end_errors = [] + cur_error = -1 + while length <= max_error_len: + end_errors.append(errors[cur_error]) + cur_error -= 1 + length += len(errors[cur_error]) + end_errors.reverse() + errors = first_errors + end_errors + return errors + if os.path.exists(os.path.join(bot_dir, "install.sh")): _, errors = _run_cmd("chmod +x install.sh; ./install.sh", bot_dir, installTimeLimit) + if errors is not None: + return "Unknown", limitErrors(errors) + detected_language, errors = detect_language(bot_dir) print("detected language") if detected_language: @@ -521,27 +546,7 @@ def compile_anything(bot_dir, installTimeLimit=600, timelimit=600, max_error_len print(e.strerror) return name, None else: - # limit length of reported errors - if len(errors) > 0 and sum(map(len, errors)) > max_error_len: - first_errors = [] - cur_error = 0 - length = len(errors[0]) - while length < (max_error_len / 3): # take 1/3 from start - first_errors.append(errors[cur_error]) - cur_error += 1 - length += len(errors[cur_error]) - first_errors.append("...") - length += 3 - end_errors = [] - cur_error = -1 - while length <= max_error_len: - end_errors.append(errors[cur_error]) - cur_error -= 1 - length += len(errors[cur_error]) - end_errors.reverse() - errors = first_errors + end_errors - - return detected_language.name, errors + return detected_language.name, limitErrors(errors) else: return "Unknown", errors From e4a91bfcccbdee6c4f18ffde9b9b4ab736dd8541 Mon Sep 17 00:00:00 2001 From: Michael Truell Date: Sat, 3 Dec 2016 15:42:40 -0500 Subject: [PATCH 3/6] Add virtualenv to sandbox --- worker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/worker/Dockerfile b/worker/Dockerfile index dfb59e291..e5bcc497c 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -37,3 +37,5 @@ RUN apt-get install -y ocaml RUN sh -c 'echo "$(curl -fsSL https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein)" > /usr/bin/lein' RUN chmod a+x /usr/bin/lein RUN lein + +RUN pip3 install virtualenv From 621bce749ba85ab8ee408d0b712e6e40a87ddc8d Mon Sep 17 00:00:00 2001 From: Michael Truell Date: Sat, 3 Dec 2016 15:46:17 -0500 Subject: [PATCH 4/6] Add git to workre --- worker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/worker/Dockerfile b/worker/Dockerfile index e5bcc497c..35d7a7209 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -39,3 +39,5 @@ RUN chmod a+x /usr/bin/lein RUN lein RUN pip3 install virtualenv + +RUN apt-get install -y git From 7c0b9bafcb44d69d4cacfdb3bdf1f70dca270cd1 Mon Sep 17 00:00:00 2001 From: Michael Truell Date: Sat, 3 Dec 2016 16:49:26 -0500 Subject: [PATCH 5/6] Add virtualenv to test suite --- tests/worker/testWorker.py | 14 ++++++++++++++ tests/worker/virtualenvBot/.gitignore | 4 ++++ tests/worker/virtualenvBot/MyBot.py | 1 + tests/worker/virtualenvBot/hlt.py | 1 + tests/worker/virtualenvBot/install.sh | 10 ++++++++++ tests/worker/virtualenvBot/networking.py | 1 + tests/worker/virtualenvBot/run.sh | 6 ++++++ 7 files changed, 37 insertions(+) create mode 100644 tests/worker/virtualenvBot/.gitignore create mode 120000 tests/worker/virtualenvBot/MyBot.py create mode 120000 tests/worker/virtualenvBot/hlt.py create mode 100755 tests/worker/virtualenvBot/install.sh create mode 120000 tests/worker/virtualenvBot/networking.py create mode 100755 tests/worker/virtualenvBot/run.sh diff --git a/tests/worker/testWorker.py b/tests/worker/testWorker.py index 1354e8fd5..f97362c52 100644 --- a/tests/worker/testWorker.py +++ b/tests/worker/testWorker.py @@ -57,6 +57,20 @@ def testLanguageOverride(self): assert language == expectedLanguage assert errors == None + def testVirtualenv(self): + '''Use a LANGUAGE file to override the detected language''' + VIRTUALENV_BOT_PATH = "virtualenvBot" + + bot_dir = os.path.join(OUR_PATH, VIRTUALENV_BOT_PATH) + + expectedLanguage = "Python" + language, errors = compiler.compile_anything(bot_dir) + if errors is not None: print("Errors: " + "\n".join(errors)) + print("Language: " + language) + + assert language == expectedLanguage + assert errors == None + class GameTests(unittest.TestCase): def testNormalGame(self): '''Test the parsing of the output of runGame.sh''' diff --git a/tests/worker/virtualenvBot/.gitignore b/tests/worker/virtualenvBot/.gitignore new file mode 100644 index 000000000..c3cc4b5b5 --- /dev/null +++ b/tests/worker/virtualenvBot/.gitignore @@ -0,0 +1,4 @@ +lib/ +bin/ +include/ +pip-selfcheck.json diff --git a/tests/worker/virtualenvBot/MyBot.py b/tests/worker/virtualenvBot/MyBot.py new file mode 120000 index 000000000..890e12ef4 --- /dev/null +++ b/tests/worker/virtualenvBot/MyBot.py @@ -0,0 +1 @@ +../../../airesources/Python/MyBot.py \ No newline at end of file diff --git a/tests/worker/virtualenvBot/hlt.py b/tests/worker/virtualenvBot/hlt.py new file mode 120000 index 000000000..28bd5d965 --- /dev/null +++ b/tests/worker/virtualenvBot/hlt.py @@ -0,0 +1 @@ +../../../airesources/Python/hlt.py \ No newline at end of file diff --git a/tests/worker/virtualenvBot/install.sh b/tests/worker/virtualenvBot/install.sh new file mode 100755 index 000000000..178e68bdb --- /dev/null +++ b/tests/worker/virtualenvBot/install.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +virtualenv ./ + +source bin/activate +pip install numpy scipy scikit-learn pillow h5py +pip install --upgrade --no-deps git+git://github.com/Theano/Theano.git +pip install keras +deactivate + diff --git a/tests/worker/virtualenvBot/networking.py b/tests/worker/virtualenvBot/networking.py new file mode 120000 index 000000000..926f24306 --- /dev/null +++ b/tests/worker/virtualenvBot/networking.py @@ -0,0 +1 @@ +../../../airesources/Python/networking.py \ No newline at end of file diff --git a/tests/worker/virtualenvBot/run.sh b/tests/worker/virtualenvBot/run.sh new file mode 100755 index 000000000..e50b4c734 --- /dev/null +++ b/tests/worker/virtualenvBot/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +source bin/activate +python3 MyBot.py +deactivate + From 590160f593dc5a0cc00917a969cb6dde4500cd74 Mon Sep 17 00:00:00 2001 From: Michael Truell Date: Sat, 3 Dec 2016 16:49:44 -0500 Subject: [PATCH 6/6] Try out rming symlink safety --- worker/compiler.py | 1 + worker/worker.py | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/worker/compiler.py b/worker/compiler.py index 2581f4e9b..473c8c17f 100644 --- a/worker/compiler.py +++ b/worker/compiler.py @@ -72,6 +72,7 @@ def _run_cmd(cmd, working_dir, timelimit): try: rawOut, rawErrors = process.communicate(timeout=timelimit) outString = rawOut.decode("utf-8").strip() + print(outString) out = outString.split("\n") if outString.isspace() == False and outString != "" else None errorsString = rawErrors.decode("utf-8").strip() diff --git a/worker/worker.py b/worker/worker.py index 2a0dd7745..431aacf76 100644 --- a/worker/worker.py +++ b/worker/worker.py @@ -82,9 +82,6 @@ def executeCompileTask(user, backend): shutil.move(os.path.join(bufferFolder, filename), os.path.join(workingPath, filename)) os.rmdir(bufferFolder) - # Rm symlinks - os.system("find "+workingPath+" -type l -delete") - language, errors = compile_anything(workingPath) didCompile = True if errors == None else False except Exception as e: