From dbe0e9da5f06a16e53f0c28ee68a1afdacb4a7b1 Mon Sep 17 00:00:00 2001 From: HustleRRR Date: Sun, 2 Jan 2022 13:42:32 +0530 Subject: [PATCH 01/10] enhancement: completing scraper_utils for codechef routes --- codedigger/codechef/scraper.py | 104 +++------------------------ codedigger/codechef/scraper_utils.py | 98 ++++++++++++++++++++++++- 2 files changed, 107 insertions(+), 95 deletions(-) diff --git a/codedigger/codechef/scraper.py b/codedigger/codechef/scraper.py index baf087f..2328391 100644 --- a/codedigger/codechef/scraper.py +++ b/codedigger/codechef/scraper.py @@ -46,104 +46,20 @@ def problemScraper(contest_code): return problem_data -def UserSubmissionDetail(problemcode, contest, user): +def UserSubmissionScraper(problemcode, contest, user): + URL = f"https://www.codechef.com/{contest}/status/{problemcode},{user}" r = requests.get(URL) soup = BeautifulSoup(r.content, 'html5lib') - problemTable = soup.findAll('table', class_="dataTable") - problemRow = problemTable[0].findAll('tr') - problemRow.pop(0) - submissionlist = [] - if len(problemRow) == 0 or problemRow[0].text == 'No Recent Activity': - return submissionlist - - for problem in problemRow: - baseurl = "https://www.codechef.com" - problemDetails = problem.findAll('td') - subid = problemDetails[0].text - subtime = problemDetails[1].text - verdict = problemDetails[3].find('span').get('title') - if len(verdict) == 0: - verdict = (problemDetails[3].find('span').text) - verdict = verdict[:verdict.index('[')] - if int(verdict) == 100: - verdict = "accepted [100/100]" - else: - verdict = "partially accepted [" + verdict + "/100]" - lang = problemDetails[6].text - link = baseurl + problemDetails[7].find('a').get('href') - - subformat = { - 'subid': subid, - 'subtime': subtime, - 'verdict': verdict, - 'lang': lang, - 'link': link, - } - - submissionlist.append(subformat) - - return submissionlist - - -def recentSubmissions(userid): - URL = f"https://www.codechef.com/recent/user?user_handle={userid}" + return soup + + +def recentSubmissionScraper(user_handle): + + URL = f"https://www.codechef.com/recent/user?user_handle={user_handle}" r = requests.get(URL) - r = BeautifulSoup(r.content, 'html5lib') - recentSubs = r.findAll('tbody') - - recentlist = [] - for sub in recentSubs: - subd = sub.findAll('tr') - subd.pop(-1) - try: - query = subd[0].text[:18] - except: - query = "Profile found successfully" - if query == 'No Recent Activity': - break - - for prob in subd: - baseurl = "https://www.codechef.com" - det = prob.findAll('td') - - probid = det[1].find('a').text - probid = probid[:probid.index('<')] - - link = det[1].find('a').get('href') - link = link.replace("\\", "") - link = baseurl + link - - subtime = prob.find('span', class_="tooltiptext") - try: - subtime = subtime.text - subtime = subtime[:subtime.index('<')] - except: - break - - verdict = det[2].find('span').get('title') - if len(verdict) == 0: - verdict = (det[2].find('span').text) - verdict = verdict[:verdict.index('[')] - if int(verdict) == 100: - verdict = "accepted [100/100]" - else: - verdict = "partially accepted [" + verdict + "/100]" - - lang = det[3].text - lang = lang[:lang.index('<')] - - subformat = { - 'probid': probid, - 'subtime': subtime, - 'verdict': verdict, - 'lang': lang, - 'link': link, - } - - recentlist.append(subformat) - - return recentlist + soup = BeautifulSoup(r.content, 'html5lib') + return soup def profilePageScraper(user_handle): diff --git a/codedigger/codechef/scraper_utils.py b/codedigger/codechef/scraper_utils.py index ea42397..77020f7 100644 --- a/codedigger/codechef/scraper_utils.py +++ b/codedigger/codechef/scraper_utils.py @@ -1,4 +1,4 @@ -from codechef.scraper import contestScraper, problemScraper, profilePageScraper +from codechef.scraper import contestScraper, problemScraper, profilePageScraper, recentSubmissionScraper, UserSubmissionScraper def OffsetLoader(contest_type): @@ -167,3 +167,99 @@ def userScraper(user_handle): # user.country_rank = country_rank # user.global_rank = global_rank # user.save() + +def RecentSubmission(user_handle): + soup = recentSubmissionScraper(user_handle) + recentSubs = soup.findAll('tbody') + + recentlist = [] + for sub in recentSubs: + subd = sub.findAll('tr') + subd.pop(-1) + try: + query = subd[0].text[:18] + except: + query = "Profile found successfully" + if query == 'No Recent Activity': + break + + for prob in subd: + baseurl = "https://www.codechef.com" + det = prob.findAll('td') + + probid = det[1].find('a').text + probid = probid[:probid.index('<')] + + link = det[1].find('a').get('href') + link = link.replace("\\", "") + link = baseurl + link + + subtime = prob.find('span', class_="tooltiptext") + try: + subtime = subtime.text + subtime = subtime[:subtime.index('<')] + except: + break + + subtime = subtime.replace("\\", "") + + verdict = det[2].find('span').get('title') + if len(verdict) == 0: + verdict = (det[2].find('span').text) + verdict = verdict[:verdict.index('[')] + if int(verdict) == 100: + verdict = "accepted [100/100]" + else: + verdict = "partially accepted [" + verdict + "/100]" + + lang = det[3].text + lang = lang[:lang.index('<')] + + subformat = { + 'probid': probid, + 'subtime': subtime, + 'verdict': verdict, + 'lang': lang, + 'link': link, + } + + recentlist.append(subformat) + + return recentlist + +def UserSubmissionDetails(problemcode, contest,user_handle): + soup = UserSubmissionScraper(problemcode, contest, user_handle) + problemTable = soup.findAll('table', class_="dataTable") + problemRow = problemTable[0].findAll('tr') + problemRow.pop(0) + submissionlist = [] + if len(problemRow) == 0 or problemRow[0].text == 'No Recent Activity': + return submissionlist + + for problem in problemRow: + baseurl = "https://www.codechef.com" + problemDetails = problem.findAll('td') + subid = problemDetails[0].text + subtime = problemDetails[1].text + verdict = problemDetails[3].find('span').get('title') + if len(verdict) == 0: + verdict = (problemDetails[3].find('span').text) + verdict = verdict[:verdict.index('[')] + if int(verdict) == 100: + verdict = "accepted [100/100]" + else: + verdict = "partially accepted [" + verdict + "/100]" + lang = problemDetails[6].text + link = baseurl + problemDetails[7].find('a').get('href') + + subformat = { + 'subid': subid, + 'subtime': subtime, + 'verdict': verdict, + 'lang': lang, + 'link': link, + } + + submissionlist.append(subformat) + + return submissionlist \ No newline at end of file From 061923a78e320a25d3596291eba740dbc5f1a563 Mon Sep 17 00:00:00 2001 From: HustleRRR Date: Sun, 2 Jan 2022 13:53:18 +0530 Subject: [PATCH 02/10] enhancement: recent submissions route completed --- codedigger/codechef/urls.py | 1 + codedigger/codechef/views.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/codedigger/codechef/urls.py b/codedigger/codechef/urls.py index 15a7328..b62c199 100644 --- a/codedigger/codechef/urls.py +++ b/codedigger/codechef/urls.py @@ -3,5 +3,6 @@ urlpatterns = [ path('upsolve/', views.CodechefUpsolveAPIView.as_view()), + path('recentsub/', views.CodechefRecentSubmissionAPIView.as_view()), path('testing', views.testing) ] diff --git a/codedigger/codechef/views.py b/codedigger/codechef/views.py index 02b8ce1..f226d62 100644 --- a/codedigger/codechef/views.py +++ b/codedigger/codechef/views.py @@ -5,7 +5,7 @@ from user.exception import ValidationException from .models import CodechefContest from .serializers import CodechefUpsolveSerializer -from .scraper_utils import contestgivenScrapper, problems_solved +from .scraper_utils import contestgivenScrapper, problems_solved, RecentSubmission, UserSubmissionDetails from codechef.cron import * # Create your views here. @@ -34,6 +34,20 @@ def get(self, request, username): return Response({'status': 'OK', 'result': result}) +class CodechefRecentSubmissionAPIView(generics.GenericAPIView): + + def get(self, request, username): + + handle = request.GET.get('handle', username) + if handle == None: + raise ValidationException( + 'Any of handle or Bearer Token is required.') + + result = RecentSubmission(handle) + + return Response({'status': 'OK', 'result': result}) + + def testing(request): update_AllContests() return HttpResponse("Successfully Scrapped!") From 2cc798a9be7fcaa5a6c7d90db34d2555dfdd2dd5 Mon Sep 17 00:00:00 2001 From: HustleRRR Date: Sun, 2 Jan 2022 15:28:43 +0530 Subject: [PATCH 03/10] feature: completed codechef problem submission and recent submission routes --- codedigger/codechef/scraper.py | 5 ++-- codedigger/codechef/scraper_utils.py | 41 ++++++++++++++++++++++++++-- codedigger/codechef/urls.py | 1 + codedigger/codechef/views.py | 19 +++++++++++++ 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/codedigger/codechef/scraper.py b/codedigger/codechef/scraper.py index 2328391..41e6edf 100644 --- a/codedigger/codechef/scraper.py +++ b/codedigger/codechef/scraper.py @@ -46,16 +46,15 @@ def problemScraper(contest_code): return problem_data -def UserSubmissionScraper(problemcode, contest, user): +def UserSubmissionScraper(URL): - URL = f"https://www.codechef.com/{contest}/status/{problemcode},{user}" r = requests.get(URL) soup = BeautifulSoup(r.content, 'html5lib') return soup def recentSubmissionScraper(user_handle): - + URL = f"https://www.codechef.com/recent/user?user_handle={user_handle}" r = requests.get(URL) soup = BeautifulSoup(r.content, 'html5lib') diff --git a/codedigger/codechef/scraper_utils.py b/codedigger/codechef/scraper_utils.py index 77020f7..b9e5d4e 100644 --- a/codedigger/codechef/scraper_utils.py +++ b/codedigger/codechef/scraper_utils.py @@ -227,8 +227,45 @@ def RecentSubmission(user_handle): return recentlist -def UserSubmissionDetails(problemcode, contest,user_handle): - soup = UserSubmissionScraper(problemcode, contest, user_handle) +def allProblemsSolved(user_handle): + + soup = profilePageScraper(user_handle) + print(user_handle) + problems_solved = [] + + all_contests = soup.find('article') + contests_list = all_contests.find_all('p') + cont = (contests_list[0].find('strong').contents)[0][:-1] + + # if cont == "Practice": + # probs = contests_list[0].find_all('a') + # for prob in probs: + # upsolved_problems.append(prob.contents[0]) + + probs = all_contests.find_all('a') + for prob in probs: + link = prob['href'] + name = prob.contents[0] + + problems_solved.append((name,link)) + + return problems_solved + +def UserSubmissionDetails(problemcode, user_handle): + + url = "" + solvedByUser = allProblemsSolved(user_handle) + + for solved in solvedByUser: + if solved[0] == problemcode: + url = solved[1] + break + + if len(url) == 0 : + return [] + + baseurl = f'https://www.codechef.com/' + soup = UserSubmissionScraper(baseurl + url) problemTable = soup.findAll('table', class_="dataTable") problemRow = problemTable[0].findAll('tr') problemRow.pop(0) diff --git a/codedigger/codechef/urls.py b/codedigger/codechef/urls.py index b62c199..18012bc 100644 --- a/codedigger/codechef/urls.py +++ b/codedigger/codechef/urls.py @@ -4,5 +4,6 @@ urlpatterns = [ path('upsolve/', views.CodechefUpsolveAPIView.as_view()), path('recentsub/', views.CodechefRecentSubmissionAPIView.as_view()), + path('problemsub//', views.CodechefUserSubmissionAPIView.as_view()), path('testing', views.testing) ] diff --git a/codedigger/codechef/views.py b/codedigger/codechef/views.py index f226d62..ee3c27d 100644 --- a/codedigger/codechef/views.py +++ b/codedigger/codechef/views.py @@ -48,6 +48,25 @@ def get(self, request, username): return Response({'status': 'OK', 'result': result}) +class CodechefUserSubmissionAPIView(generics.GenericAPIView): + + def get(self, request, username, problem): + + handle = request.GET.get('handle', username) + problemcode = request.GET.get('problemcode', problem) + if handle == None: + raise ValidationException( + 'Any of handle or Bearer Token is required.') + + if problemcode == None: + raise ValidationException( + 'Any of valid problem code is required.') + + result = UserSubmissionDetails(problemcode, handle) + + return Response({'status': 'OK', 'result': result}) + + def testing(request): update_AllContests() return HttpResponse("Successfully Scrapped!") From 5a7bfbaceb2930178fcef87144f80ebe6c63b7a4 Mon Sep 17 00:00:00 2001 From: HustleRRR Date: Sun, 2 Jan 2022 15:33:44 +0530 Subject: [PATCH 04/10] feature: codechef route for list of problem of a contest --- codedigger/codechef/urls.py | 1 + codedigger/codechef/views.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/codedigger/codechef/urls.py b/codedigger/codechef/urls.py index 18012bc..b2db213 100644 --- a/codedigger/codechef/urls.py +++ b/codedigger/codechef/urls.py @@ -5,5 +5,6 @@ path('upsolve/', views.CodechefUpsolveAPIView.as_view()), path('recentsub/', views.CodechefRecentSubmissionAPIView.as_view()), path('problemsub//', views.CodechefUserSubmissionAPIView.as_view()), + path('contestproblems/', views.CodechefContestProblemsAPIView.as_view()), path('testing', views.testing) ] diff --git a/codedigger/codechef/views.py b/codedigger/codechef/views.py index ee3c27d..fbecab0 100644 --- a/codedigger/codechef/views.py +++ b/codedigger/codechef/views.py @@ -67,6 +67,20 @@ def get(self, request, username, problem): return Response({'status': 'OK', 'result': result}) +class CodechefContestProblemsAPIView(generics.GenericAPIView): + + def get(self, request, contest): + + contest_id = request.GET.get('contest_id', contest) + + if contest_id == None: + raise ValidationException( + 'A valid Contest ID is required.') + + result = ProblemData(contest_id) + + return Response({'status': 'OK', 'result': result}) + def testing(request): update_AllContests() return HttpResponse("Successfully Scrapped!") From add670f8231457adbcb1b16e9940da5c4980e314 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 2 Jan 2022 12:53:28 +0000 Subject: [PATCH 05/10] refactor: automatic code reformatting to PEP8 by yapf --- codedigger/codechef/scraper_utils.py | 11 +++++++---- codedigger/codechef/urls.py | 9 ++++++--- codedigger/codechef/views.py | 9 ++++----- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/codedigger/codechef/scraper_utils.py b/codedigger/codechef/scraper_utils.py index b9e5d4e..2ebe263 100644 --- a/codedigger/codechef/scraper_utils.py +++ b/codedigger/codechef/scraper_utils.py @@ -168,6 +168,7 @@ def userScraper(user_handle): # user.global_rank = global_rank # user.save() + def RecentSubmission(user_handle): soup = recentSubmissionScraper(user_handle) recentSubs = soup.findAll('tbody') @@ -227,6 +228,7 @@ def RecentSubmission(user_handle): return recentlist + def allProblemsSolved(user_handle): soup = profilePageScraper(user_handle) @@ -241,16 +243,17 @@ def allProblemsSolved(user_handle): # probs = contests_list[0].find_all('a') # for prob in probs: # upsolved_problems.append(prob.contents[0]) - + probs = all_contests.find_all('a') for prob in probs: link = prob['href'] name = prob.contents[0] - problems_solved.append((name,link)) + problems_solved.append((name, link)) return problems_solved + def UserSubmissionDetails(problemcode, user_handle): url = "" @@ -261,7 +264,7 @@ def UserSubmissionDetails(problemcode, user_handle): url = solved[1] break - if len(url) == 0 : + if len(url) == 0: return [] baseurl = f'https://www.codechef.com/' @@ -299,4 +302,4 @@ def UserSubmissionDetails(problemcode, user_handle): submissionlist.append(subformat) - return submissionlist \ No newline at end of file + return submissionlist diff --git a/codedigger/codechef/urls.py b/codedigger/codechef/urls.py index b2db213..d4aa170 100644 --- a/codedigger/codechef/urls.py +++ b/codedigger/codechef/urls.py @@ -3,8 +3,11 @@ urlpatterns = [ path('upsolve/', views.CodechefUpsolveAPIView.as_view()), - path('recentsub/', views.CodechefRecentSubmissionAPIView.as_view()), - path('problemsub//', views.CodechefUserSubmissionAPIView.as_view()), - path('contestproblems/', views.CodechefContestProblemsAPIView.as_view()), + path('recentsub/', + views.CodechefRecentSubmissionAPIView.as_view()), + path('problemsub//', + views.CodechefUserSubmissionAPIView.as_view()), + path('contestproblems/', + views.CodechefContestProblemsAPIView.as_view()), path('testing', views.testing) ] diff --git a/codedigger/codechef/views.py b/codedigger/codechef/views.py index fbecab0..1072f35 100644 --- a/codedigger/codechef/views.py +++ b/codedigger/codechef/views.py @@ -57,10 +57,9 @@ def get(self, request, username, problem): if handle == None: raise ValidationException( 'Any of handle or Bearer Token is required.') - + if problemcode == None: - raise ValidationException( - 'Any of valid problem code is required.') + raise ValidationException('Any of valid problem code is required.') result = UserSubmissionDetails(problemcode, handle) @@ -74,13 +73,13 @@ def get(self, request, contest): contest_id = request.GET.get('contest_id', contest) if contest_id == None: - raise ValidationException( - 'A valid Contest ID is required.') + raise ValidationException('A valid Contest ID is required.') result = ProblemData(contest_id) return Response({'status': 'OK', 'result': result}) + def testing(request): update_AllContests() return HttpResponse("Successfully Scrapped!") From db18eb49c31ec872b1453ce1d46422dd540bc98b Mon Sep 17 00:00:00 2001 From: HustleRRR Date: Tue, 18 Jan 2022 13:51:08 +0530 Subject: [PATCH 06/10] enhancement: only one instance of problem will be saved. --- codedigger/codechef/model_utils.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/codedigger/codechef/model_utils.py b/codedigger/codechef/model_utils.py index bfba88f..ee1d8c3 100644 --- a/codedigger/codechef/model_utils.py +++ b/codedigger/codechef/model_utils.py @@ -6,16 +6,19 @@ def create_or_update_codechefProblem(problemdata): for problem in problemdata: Prob, created = Problem.objects.get_or_create( - name=problem['Name'], - prob_id=problem['ProblemCode'], - url=problem['ProblemURL'], - contest_id=problem['ContestId'], - platform=problem['Platform']) + prob_id = problem['ProblemCode'], + platform = problem['Platform'], + defaults = { + 'name' : problem['Name'], + 'url' : problem['ProblemURL'], + 'contest_id' : problem['ContestId'], + }, + ) + cont = CodechefContest.objects.get(contestId=problem['ContestId']) - prob = Problem.objects.get(prob_id=problem['ProblemCode'], - contest_id=problem['ContestId']) + ccprob, created = CodechefContestProblems.objects.get_or_create( - contest=cont, problem=prob) + contest = cont, problem = Prob) def create_or_update_codechefContest(contest): From dcb7a0f5337f1fa141ee29489192b5f503138820 Mon Sep 17 00:00:00 2001 From: HustleRRR Date: Tue, 18 Jan 2022 14:19:42 +0530 Subject: [PATCH 07/10] enhancement: modified contest scraper and util for contest type scrapping --- codedigger/codechef/cron.py | 2 +- codedigger/codechef/scraper.py | 2 +- codedigger/codechef/scraper_utils.py | 29 ++++++++++++++++++++++++++-- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/codedigger/codechef/cron.py b/codedigger/codechef/cron.py index 1f79687..6301676 100644 --- a/codedigger/codechef/cron.py +++ b/codedigger/codechef/cron.py @@ -4,7 +4,7 @@ def update_AllContests(): # Creates new contests and problems in Database - all_contests = ContestData('past') + all_contests = ContestData('all', 'past') for contest in all_contests: create_or_update_codechefContest(contest) contest_problems_info = ProblemData(contest['ContestCode']) diff --git a/codedigger/codechef/scraper.py b/codedigger/codechef/scraper.py index 41e6edf..6148bfa 100644 --- a/codedigger/codechef/scraper.py +++ b/codedigger/codechef/scraper.py @@ -21,7 +21,7 @@ def divisionScraper(contest_id): def contestScraper(offset, contest_type): - query_contest_url = f"https://www.codechef.com/api/list/contests/{contest_type}?sort_by=START&sorting_order=desc&offset={offset}&mode=premium" + query_contest_url = f"https://www.codechef.com/api/list/contests/" + contest_type + "?sort_by=START&sorting_order=desc&offset=" + str(offset) + "&mode=premium" # Query URL might change in future. contest_data = requests.get(query_contest_url) diff --git a/codedigger/codechef/scraper_utils.py b/codedigger/codechef/scraper_utils.py index 2ebe263..ebbd167 100644 --- a/codedigger/codechef/scraper_utils.py +++ b/codedigger/codechef/scraper_utils.py @@ -30,9 +30,9 @@ def getContestDivision(contest_id): return subcontests -def ContestData(type): +def ContestData(cont_type, cont_time): - contests_data = OffsetLoader(type) + contests_data = OffsetLoader(cont_time) all_contests = [] dateDict = { "Jan": "January", @@ -48,7 +48,32 @@ def ContestData(type): "Nov": "November", "Dec": "December" } + + longContestCode = ['JAN', 'FEB', 'MARCH', 'APRIL', 'MAY', 'JUNE', 'JULY', 'AUG', 'SEPT', 'OCT', 'NOV', 'DEC'] + for contest in contests_data: + + if cont_type == 'starters': + if contest['contest_code'][:5] != 'START': + continue + + if cont_type == 'lunchtime': + if contest['contest_code'][:5] != 'LTIME': + continue + + if cont_type == 'cookoff': + if contest['contest_code'][:4] != 'COOK': + continue + + if cont_type == 'long': + found = 0 + for month in longContestCode: + if contest['contest_code'][:len(month)] == month: + found = 1 + break + if found == 0: + continue + childContests = getContestDivision(contest['contest_code']) for contest_id in childContests: From 56eb9eef51da6a4a2735b2b3263684abcdc7d7e9 Mon Sep 17 00:00:00 2001 From: HustleRRR Date: Sat, 22 Jan 2022 17:10:17 +0530 Subject: [PATCH 08/10] enhancement: route for codechef contests fetching with scraper --- codedigger/codechef/urls.py | 2 ++ codedigger/codechef/views.py | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/codedigger/codechef/urls.py b/codedigger/codechef/urls.py index d4aa170..e2e5b23 100644 --- a/codedigger/codechef/urls.py +++ b/codedigger/codechef/urls.py @@ -9,5 +9,7 @@ views.CodechefUserSubmissionAPIView.as_view()), path('contestproblems/', views.CodechefContestProblemsAPIView.as_view()), + path('contests//', + views.CodechefContestsAPIView.as_view()), path('testing', views.testing) ] diff --git a/codedigger/codechef/views.py b/codedigger/codechef/views.py index 1072f35..2008d23 100644 --- a/codedigger/codechef/views.py +++ b/codedigger/codechef/views.py @@ -5,7 +5,7 @@ from user.exception import ValidationException from .models import CodechefContest from .serializers import CodechefUpsolveSerializer -from .scraper_utils import contestgivenScrapper, problems_solved, RecentSubmission, UserSubmissionDetails +from .scraper_utils import contestgivenScrapper, problems_solved, RecentSubmission, UserSubmissionDetails, ContestData from codechef.cron import * # Create your views here. @@ -80,6 +80,23 @@ def get(self, request, contest): return Response({'status': 'OK', 'result': result}) +class CodechefContestsAPIView(generics.GenericAPIView): + + def get(self, request, time, typec): + + ctime = request.GET.get('ctime', time) + ctype = request.GET.get('ctype', typec) + if ((ctime != 'past') & (ctime != 'present') & (ctime != 'future')): + raise ValidationException( + 'A valid parameter for contest time is required') + + if ((ctype != 'lunchtime') & (ctime != 'cookoff') & (ctime != 'starters') & (ctime != 'long')): + ctype = 'all' + + result = ContestData(ctype, ctime) + return Response({'status': 'OK', 'result':result}) + + def testing(request): update_AllContests() return HttpResponse("Successfully Scrapped!") From 7335ee39784917a770107ba723f95f2b84aae474 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 22 Jan 2022 11:44:28 +0000 Subject: [PATCH 09/10] refactor: automatic code reformatting to PEP8 by yapf --- codedigger/codechef/model_utils.py | 14 +++++++------- codedigger/codechef/scraper.py | 3 ++- codedigger/codechef/scraper_utils.py | 9 ++++++--- codedigger/codechef/urls.py | 2 +- codedigger/codechef/views.py | 5 +++-- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/codedigger/codechef/model_utils.py b/codedigger/codechef/model_utils.py index ee1d8c3..d886b02 100644 --- a/codedigger/codechef/model_utils.py +++ b/codedigger/codechef/model_utils.py @@ -6,19 +6,19 @@ def create_or_update_codechefProblem(problemdata): for problem in problemdata: Prob, created = Problem.objects.get_or_create( - prob_id = problem['ProblemCode'], - platform = problem['Platform'], - defaults = { - 'name' : problem['Name'], - 'url' : problem['ProblemURL'], - 'contest_id' : problem['ContestId'], + prob_id=problem['ProblemCode'], + platform=problem['Platform'], + defaults={ + 'name': problem['Name'], + 'url': problem['ProblemURL'], + 'contest_id': problem['ContestId'], }, ) cont = CodechefContest.objects.get(contestId=problem['ContestId']) ccprob, created = CodechefContestProblems.objects.get_or_create( - contest = cont, problem = Prob) + contest=cont, problem=Prob) def create_or_update_codechefContest(contest): diff --git a/codedigger/codechef/scraper.py b/codedigger/codechef/scraper.py index 6148bfa..23fe9aa 100644 --- a/codedigger/codechef/scraper.py +++ b/codedigger/codechef/scraper.py @@ -21,7 +21,8 @@ def divisionScraper(contest_id): def contestScraper(offset, contest_type): - query_contest_url = f"https://www.codechef.com/api/list/contests/" + contest_type + "?sort_by=START&sorting_order=desc&offset=" + str(offset) + "&mode=premium" + query_contest_url = f"https://www.codechef.com/api/list/contests/" + contest_type + "?sort_by=START&sorting_order=desc&offset=" + str( + offset) + "&mode=premium" # Query URL might change in future. contest_data = requests.get(query_contest_url) diff --git a/codedigger/codechef/scraper_utils.py b/codedigger/codechef/scraper_utils.py index 0222da0..49cc913 100644 --- a/codedigger/codechef/scraper_utils.py +++ b/codedigger/codechef/scraper_utils.py @@ -49,14 +49,17 @@ def ContestData(cont_type, cont_time): "Dec": "December" } - longContestCode = ['JAN', 'FEB', 'MARCH', 'APRIL', 'MAY', 'JUNE', 'JULY', 'AUG', 'SEPT', 'OCT', 'NOV', 'DEC'] + longContestCode = [ + 'JAN', 'FEB', 'MARCH', 'APRIL', 'MAY', 'JUNE', 'JULY', 'AUG', 'SEPT', + 'OCT', 'NOV', 'DEC' + ] for contest in contests_data: if cont_type == 'starters': if contest['contest_code'][:5] != 'START': continue - + if cont_type == 'lunchtime': if contest['contest_code'][:5] != 'LTIME': continue @@ -64,7 +67,7 @@ def ContestData(cont_type, cont_time): if cont_type == 'cookoff': if contest['contest_code'][:4] != 'COOK': continue - + if cont_type == 'long': found = 0 for month in longContestCode: diff --git a/codedigger/codechef/urls.py b/codedigger/codechef/urls.py index e2e5b23..c4239c6 100644 --- a/codedigger/codechef/urls.py +++ b/codedigger/codechef/urls.py @@ -9,7 +9,7 @@ views.CodechefUserSubmissionAPIView.as_view()), path('contestproblems/', views.CodechefContestProblemsAPIView.as_view()), - path('contests//', + path('contests//', views.CodechefContestsAPIView.as_view()), path('testing', views.testing) ] diff --git a/codedigger/codechef/views.py b/codedigger/codechef/views.py index 31f5ea2..c0960dd 100644 --- a/codedigger/codechef/views.py +++ b/codedigger/codechef/views.py @@ -114,11 +114,12 @@ def get(self, request, time, typec): raise ValidationException( 'A valid parameter for contest time is required') - if ((ctype != 'lunchtime') & (ctime != 'cookoff') & (ctime != 'starters') & (ctime != 'long')): + if ((ctype != 'lunchtime') & (ctime != 'cookoff') & + (ctime != 'starters') & (ctime != 'long')): ctype = 'all' result = ContestData(ctype, ctime) - return Response({'status': 'OK', 'result':result}) + return Response({'status': 'OK', 'result': result}) def testing(request): From 8c5d63642c6d23e57da62a9bc4e741ae730e9cff Mon Sep 17 00:00:00 2001 From: HustleRRR Date: Sat, 22 Jan 2022 17:22:01 +0530 Subject: [PATCH 10/10] enhancement: codechef contests route with scraper --- codedigger/codechef/urls.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/codedigger/codechef/urls.py b/codedigger/codechef/urls.py index c4239c6..565f2fc 100644 --- a/codedigger/codechef/urls.py +++ b/codedigger/codechef/urls.py @@ -2,7 +2,9 @@ from . import views urlpatterns = [ - path('upsolve/', views.CodechefUpsolveAPIView.as_view()), + path('upsolve', + views.CodechefUpsolveAPIView.as_view(), + name="codechef-upsolve"), path('recentsub/', views.CodechefRecentSubmissionAPIView.as_view()), path('problemsub//',