From 41e5a52a5a15e9731dd094695efdaaa178342bcb Mon Sep 17 00:00:00 2001 From: Steven Date: Tue, 23 Aug 2011 13:35:58 -0700 Subject: [PATCH 1/3] Added a simple RateMyProfessor dialog when clicking on a professor name. It still needs work because clicking an instructor still adds the class to the calendar. --- antplanner.py | 28 ++++++++++++++- app.yaml | 6 ++-- scraper.py | 25 +++++++++++++- static/js/main.js | 81 +++++++++++++++++++++++++++++++++++++++----- templates/index.html | 8 ++++- 5 files changed, 133 insertions(+), 15 deletions(-) diff --git a/antplanner.py b/antplanner.py index 30cb2f0..3be015f 100644 --- a/antplanner.py +++ b/antplanner.py @@ -10,6 +10,8 @@ from google.appengine.api import memcache from google.appengine.api import users +import logging + urls = ( '/', 'index', '/search', 'search', @@ -19,7 +21,8 @@ '/admin', 'admin', '/admin/flush-cache', 'adminFlushCache', '/admin/latest-web-soc', 'latestWebSoc', - '/admin/delete-old-schedules', 'deleteOldSchedules' + '/admin/delete-old-schedules', 'deleteOldSchedules', + '/prof', 'getProf' ) render = web.template.render('templates/') @@ -107,6 +110,29 @@ class loadSchedule(): def GET(self): return schedule.load_schedule(web.input().username) +class getProf(): + def GET(self): + p = web.input() + logging.debug(p) + #data = memcache.get("PROF") + if p is None or p.name is None: + return '{"Empty Request": ""}' + #if data is None: + try: + name = urllib.quote_plus(p.name) + logging.debug('Escaped name: ' + name) + raw_page = urlfetch.fetch("http://www.ratemyprofessors.com/SearchProfs.jsp?letter=" + urllib.quote_plus(p.name), + method=urlfetch.GET, + deadline=10) + data = scraper.strip_professors(raw_page.content) + #memcache.add("PROF", data, 60 * 60) + except urlfetch.DownloadError: + data = '{"RateMyProfessors.com request exceeded 10 seconds": ""}' + except urlfetch.Error: + data = '{"RateMyProfessors.com is not available at the moment": ""}' + + return data + if __name__ == "__main__": app = web.application(urls, globals()) app.cgirun() diff --git a/app.yaml b/app.yaml index 7c89692..a5e287b 100644 --- a/app.yaml +++ b/app.yaml @@ -1,5 +1,5 @@ -application: antplanner -version: release-0-9-8-1 +application: antplanner-fork +version: release-0-9-8-2 api_version: 1 runtime: python @@ -10,4 +10,4 @@ handlers: script: antplanner.py login: admin - url: .* - script: antplanner.py \ No newline at end of file + script: antplanner.py diff --git a/scraper.py b/scraper.py index 9b1fdd7..a79cb31 100644 --- a/scraper.py +++ b/scraper.py @@ -1,5 +1,7 @@ from lib.BeautifulSoup import BeautifulSoup import re +from django.utils import simplejson as json +import logging def strip_search(html): form_html = BeautifulSoup(html).find('form', action='http://websoc.reg.uci.edu/') @@ -28,4 +30,25 @@ def strip_websoc_version(html): return 'Couldn\'t find a match' else: return version_matches[0] - \ No newline at end of file + + +def strip_professors(html): + p = BeautifulSoup(html).find('table', {'id': 'rmp_table'}) + if p is None: + logging.debug(html[500:]) + return '{"Error while parsing table at RateMyProfessors.com":""}' + else: + profs = list() + trs = p.findAll('tr') + for tr in trs: + if (str(tr).find('sid=1074') != -1): + logging.debug(str(tr)) + tds = tr.findAll('td') + logging.debug(str(tds[1])) + anchor = tds[1].find('a') + name = anchor.renderContents().strip() + href = anchor['href'].strip() + prof = { name: href } + logging.debug(prof) + profs.append(prof) + return json.dumps(profs) \ No newline at end of file diff --git a/static/js/main.js b/static/js/main.js index 6981062..85d8cf8 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -307,6 +307,7 @@ function WindowManager() { function SOCParser() { this.SCHEDULE_CODE_INDEX = 0; this.SCHEDULE_TYPE_INDEX = 1; + this.SCHEDULE_PROF_INDEX = 4; this.SCHEDULE_TIME_INDEX = 5; this.getTimeString = function(element) { @@ -326,29 +327,30 @@ function SOCParser() { }; this.getCourseCode = function(element) { - var courseCode = $(element).find('td').eq(this.SCHEDULE_CODE_INDEX).html(); - return courseCode; + return $(element).find('td').eq(this.SCHEDULE_CODE_INDEX).html(); }; this.getCourseType = function(element) { - var courseType = $(element).find('td').eq(this.SCHEDULE_TYPE_INDEX).html(); - return courseType; + return $(element).find('td').eq(this.SCHEDULE_TYPE_INDEX).html(); }; + this.getCourseProf = function(element) { + return $(element).find('td').eq(this.SCHEDULE_PROF_INDEX).html(); + } + this.getCourseString = function(element) { - var courseString = $(element).prevAll().find('.CourseTitle:last').html(); - return courseString; + return $(element).prevAll().find('.CourseTitle:last').html(); } }; -function SOC() { +function SOC() { this.initSOC = function(bridge) { var list = $('.course-list', frames['school'].document); //hover over valid course $("tr[valign*='top']", list).hover( function() { - $(this).css({'color': 'red', 'cursor': 'pointer'}); + $(this).css({'color': 'blue', 'cursor': 'pointer'}); }, function() { $(this).css({'color': 'black', 'cursor': 'default'}); @@ -402,9 +404,60 @@ function SOC() { bridge.addEvent(calEvents[i]); } }); + + // click on instructor + $("tr[valign*='top'] td:nth-child(5)", list).click(function() { + //var socParser = new SOCParser(); + var prof = $(this).html(); + showProfessors(prof); + }); } } +function showProfessors(name) { + // name is the query string for professors + name = prof.split(',')[0]; // get first word only (aka last name) + if (name == '' || name.toUpperCase() == 'STAFF') + return; + + var profTable = $('#prof-select'); + profTable.html('Please wait while we load RateMyProfessors.com'); + profTable.dialog('open'); + + $.ajax({ + url: "/prof", + type: 'get', + data: 'name=' + name, + dataType: 'json', + beforeSend: function() { + $('body').css({'cursor': 'wait'}); + if(!name) { + return false; + } + }, + success: function(data) { + console.log(data); + + var trs = 'ProfessorDepartment# RatingsQualityEasinessHot'; + if (data.length > 0) { + for (var i=0; i'+a+''; + trs += '' + } + } else { + trs = 'No reviews found for professor '+name+''; + } + profTable.html(trs); + }, + complete: function(jqXHR, textStatus) { + $('body').css({'cursor': 'auto'}); + } + }); +} + $(document).ready(function() { var courseManager = new CourseManager(); var apCalendar = new APCalendar(); @@ -541,7 +594,17 @@ $(document).ready(function() { }); return false; }); + + $('#prof-select').dialog({ autoOpen: false, + modal: true, + title: 'Select a Professor', + width: 235, + resizable: false, + closeOnEscape: true, + draggable: false + }).css('font-size', '14px'); + }); //surprise! -if (top === self){}else{$(document).ready(function(){$('html').html('
')});} \ No newline at end of file +if (top === self){}else{$(document).ready(function(){$('html').html('
')});} diff --git a/templates/index.html b/templates/index.html index c4f5720..6d8c809 100644 --- a/templates/index.html +++ b/templates/index.html @@ -17,6 +17,7 @@ @@ -59,8 +61,12 @@

A better WebSoc for UCI

- + + + +
Please wait while we load RateMyProfessors.com
From 62c043ae01a552f27d9e0fe28b16872d8901c0c0 Mon Sep 17 00:00:00 2001 From: Steven Date: Tue, 23 Aug 2011 17:25:42 -0700 Subject: [PATCH 2/3] RateMyProfessor feature is now narrowing down the results very well. In fact, if RMP does not have a first initial, there won't be a match. For example GOODRICH, M. is not listed with a first initial on RMP. --- antplanner.py | 10 ++++----- scraper.py | 55 +++++++++++++++++++++++++++++++++------------ static/css/main.css | 11 +++++++++ static/js/main.js | 41 +++++++++++++++++++-------------- 4 files changed, 81 insertions(+), 36 deletions(-) diff --git a/antplanner.py b/antplanner.py index 3be015f..358dc5d 100644 --- a/antplanner.py +++ b/antplanner.py @@ -113,18 +113,18 @@ def GET(self): class getProf(): def GET(self): p = web.input() - logging.debug(p) + #logging.debug(p) #data = memcache.get("PROF") if p is None or p.name is None: return '{"Empty Request": ""}' #if data is None: try: - name = urllib.quote_plus(p.name) - logging.debug('Escaped name: ' + name) - raw_page = urlfetch.fetch("http://www.ratemyprofessors.com/SearchProfs.jsp?letter=" + urllib.quote_plus(p.name), + q = urllib.quote_plus(p.name[0]) + #logging.debug('Query param: ' + q) + raw_page = urlfetch.fetch("http://www.ratemyprofessors.com/SelectTeacher.jsp?the_dept=All&sid=1074&orderby=TLName&letter=" + q, method=urlfetch.GET, deadline=10) - data = scraper.strip_professors(raw_page.content) + data = scraper.strip_professors(raw_page.content, unicode(p.name)) #memcache.add("PROF", data, 60 * 60) except urlfetch.DownloadError: data = '{"RateMyProfessors.com request exceeded 10 seconds": ""}' diff --git a/scraper.py b/scraper.py index a79cb31..ceeddee 100644 --- a/scraper.py +++ b/scraper.py @@ -32,23 +32,50 @@ def strip_websoc_version(html): return version_matches[0] -def strip_professors(html): - p = BeautifulSoup(html).find('table', {'id': 'rmp_table'}) - if p is None: +def strip_professors(html, name): + table = BeautifulSoup(html).find('div', {'id': 'ratingTable'}) + if table is None: logging.debug(html[500:]) return '{"Error while parsing table at RateMyProfessors.com":""}' else: profs = list() - trs = p.findAll('tr') - for tr in trs: - if (str(tr).find('sid=1074') != -1): - logging.debug(str(tr)) - tds = tr.findAll('td') - logging.debug(str(tds[1])) - anchor = tds[1].find('a') - name = anchor.renderContents().strip() - href = anchor['href'].strip() - prof = { name: href } - logging.debug(prof) + name = name.upper() + split = name.split(','); + qLastName = split[0].strip() + qFirstName = split[1].strip() + if (qFirstName == None or qFirstName == ''): + qFirstName = '!' + rows = table.findAll('div', {'class': re.compile(r".*\bentry\b.*")}) + for row in rows: + divName = row.find('div', {'class': 'profName'}) + anchor = divName.find('a') + profName = unicode(anchor.renderContents().strip(), 'utf-8', 'ignore').upper() + split = profName.split(','); + lastName = split[0].strip() + firstName = split[1].strip() + if (firstName == None or firstName == ''): + firstName = '!' + #logging.debug(qLastName + ' =? ' + lastName + ' && ' + qFirstName + ' =? ' + firstName) + if lastName == qLastName and firstName[0] == qFirstName[0]: + href = 'http://www.ratemyprofessors.com/' + anchor['href'].strip() + profDept = row.find('div', {'class': 'profDept'}).renderContents().strip() + profRatings = row.find('div', {'class': 'profRatings'}).renderContents().strip() + profQuality = row.find('div', {'class': 'profAvg'}).renderContents().strip() + profEasiness = row.find('div', {'class': 'profEasy'}).renderContents().strip() + profHot = row.find('div', {'class': re.compile(r".*\bprofHot\b.*")}).renderContents().strip() + if profHot == 'Hot': + profHot = '✓' + else: + profHot = ' ' + + prof = {'name': profName, + 'href': href, + 'dept': profDept, + 'ratings': profRatings, + 'quality': profQuality, + 'easiness': profEasiness, + 'hot': profHot + } + #logging.debug(prof) profs.append(prof) return json.dumps(profs) \ No newline at end of file diff --git a/static/css/main.css b/static/css/main.css index 1e30cc4..a730d79 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -111,3 +111,14 @@ iframe#school { width: 50%; float: left; } + +/* professor modal popup */ +#prof-select { + font-size: 14px; + border-collapse: collapse; + margin:10px; +} +#prof-select td, #prof-select th { + border: 1px solid gray; + padding: 5px; +} diff --git a/static/js/main.js b/static/js/main.js index 85d8cf8..615c455 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -357,7 +357,7 @@ function SOC() { } ); - //click on course + //click on course (this should be placed on all tr's except instructor) $("tr[valign*='top']", list).click(function() { var socParser = new SOCParser(); var courseUtils = new CourseUtils(); @@ -407,21 +407,30 @@ function SOC() { // click on instructor $("tr[valign*='top'] td:nth-child(5)", list).click(function() { - //var socParser = new SOCParser(); var prof = $(this).html(); showProfessors(prof); }); + + // hover + $("tr[valign*='top'] td:nth-child(5)", list).hover( + function() { + $(this).css({'color': 'green', 'cursor': 'pointer'}); + }, + function() { + $(this).css({'color': 'inherit', 'cursor': 'inherit'}); + } + ); } } function showProfessors(name) { // name is the query string for professors - name = prof.split(',')[0]; // get first word only (aka last name) - if (name == '' || name.toUpperCase() == 'STAFF') + lastName = name.split(",")[0]; // get first word only (aka last name) + if (lastName == '' || lastName.toUpperCase() == 'STAFF') return; var profTable = $('#prof-select'); - profTable.html('Please wait while we load RateMyProfessors.com'); + profTable.html('Please wait while we load RateMyProfessors.com'); profTable.dialog('open'); $.ajax({ @@ -436,22 +445,20 @@ function showProfessors(name) { } }, success: function(data) { - console.log(data); - - var trs = 'ProfessorDepartment# RatingsQualityEasinessHot'; + var trs = 'ProfessorDepartment# RatingsQualityEasinessHot'; if (data.length > 0) { for (var i=0; i'+a+''; - trs += '' + var p = data[i]; + trs += ''+p.name+''+p.dept+''+p.ratings+''+p.quality+''+p.easiness+''+p.hot+''; } } else { - trs = 'No reviews found for professor '+name+''; + trs += ''+name+' 0 found   '; } profTable.html(trs); }, + error: function(jqXHR, textStatus, errorThrown) { + profTable.html('An error occured:'+textStatus+''+errorThrown+''); + }, complete: function(jqXHR, textStatus) { $('body').css({'cursor': 'auto'}); } @@ -597,12 +604,12 @@ $(document).ready(function() { $('#prof-select').dialog({ autoOpen: false, modal: true, - title: 'Select a Professor', - width: 235, + title: 'RateMyProfessors.com', + width: 600, resizable: false, closeOnEscape: true, draggable: false - }).css('font-size', '14px'); + }); }); From 76d060685a2dd033ad6a7ce844a61d5329cb265a Mon Sep 17 00:00:00 2001 From: Steven Date: Wed, 24 Aug 2011 11:13:18 -0700 Subject: [PATCH 3/3] Courses with multiple instructors will show ratings for each instructor. Added better error messages. --- antplanner.py | 6 ++-- scraper.py | 36 ++++++++++++++------- static/css/main.css | 5 +++ static/js/main.js | 76 +++++++++++++++++++++++---------------------- 4 files changed, 72 insertions(+), 51 deletions(-) diff --git a/antplanner.py b/antplanner.py index 358dc5d..7f9c8f1 100644 --- a/antplanner.py +++ b/antplanner.py @@ -116,7 +116,7 @@ def GET(self): #logging.debug(p) #data = memcache.get("PROF") if p is None or p.name is None: - return '{"Empty Request": ""}' + return get_rmp_error('Empty Request','The professor must have a last name in order to find ratings.') #if data is None: try: q = urllib.quote_plus(p.name[0]) @@ -127,9 +127,9 @@ def GET(self): data = scraper.strip_professors(raw_page.content, unicode(p.name)) #memcache.add("PROF", data, 60 * 60) except urlfetch.DownloadError: - data = '{"RateMyProfessors.com request exceeded 10 seconds": ""}' + data = get_rmp_error('urlfetch.DownloadError','RateMyProfessors.com request exceeded 10 seconds') except urlfetch.Error: - data = '{"RateMyProfessors.com is not available at the moment": ""}' + data = get_rmp_error('urlfetch.Error','RateMyProfessors.com is not available at the moment') return data diff --git a/scraper.py b/scraper.py index ceeddee..31dd3f8 100644 --- a/scraper.py +++ b/scraper.py @@ -31,15 +31,26 @@ def strip_websoc_version(html): else: return version_matches[0] +def get_rmp_error(title, message): + data = { + 'name': title, + 'href': 'javascript:void(0);', + 'dept': message, + 'ratings': '0', + 'quality': '0', + 'easiness': '0', + 'hot': 'Ø' + } + return json.dumps(data) def strip_professors(html, name): table = BeautifulSoup(html).find('div', {'id': 'ratingTable'}) if table is None: logging.debug(html[500:]) - return '{"Error while parsing table at RateMyProfessors.com":""}' + return get_rmp_error('Parse Error','Could not find "ratingTable" at RateMyProfessors.com') else: profs = list() - name = name.upper() + #name = name.upper() split = name.split(','); qLastName = split[0].strip() qFirstName = split[1].strip() @@ -68,14 +79,17 @@ def strip_professors(html, name): else: profHot = ' ' - prof = {'name': profName, - 'href': href, - 'dept': profDept, - 'ratings': profRatings, - 'quality': profQuality, - 'easiness': profEasiness, - 'hot': profHot - } + prof = { + 'name': profName, + 'href': href, + 'dept': profDept, + 'ratings': profRatings, + 'quality': profQuality, + 'easiness': profEasiness, + 'hot': profHot + } #logging.debug(prof) profs.append(prof) - return json.dumps(profs) \ No newline at end of file + return json.dumps(profs) + + diff --git a/static/css/main.css b/static/css/main.css index a730d79..72c9294 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -122,3 +122,8 @@ iframe#school { border: 1px solid gray; padding: 5px; } + +#prof-select a { + color: #688FE7; + font-size: 14px; +} diff --git a/static/js/main.js b/static/js/main.js index 615c455..4962f23 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -350,7 +350,7 @@ function SOC() { //hover over valid course $("tr[valign*='top']", list).hover( function() { - $(this).css({'color': 'blue', 'cursor': 'pointer'}); + $(this).css({'color': 'red', 'cursor': 'pointer'}); }, function() { $(this).css({'color': 'black', 'cursor': 'default'}); @@ -411,10 +411,10 @@ function SOC() { showProfessors(prof); }); - // hover + // instructor hover $("tr[valign*='top'] td:nth-child(5)", list).hover( function() { - $(this).css({'color': 'green', 'cursor': 'pointer'}); + $(this).css({'color': 'blue', 'cursor': 'pointer'}); }, function() { $(this).css({'color': 'inherit', 'cursor': 'inherit'}); @@ -423,46 +423,48 @@ function SOC() { } } -function showProfessors(name) { +function showProfessors(nameString) { // name is the query string for professors - lastName = name.split(",")[0]; // get first word only (aka last name) - if (lastName == '' || lastName.toUpperCase() == 'STAFF') - return; - var profTable = $('#prof-select'); profTable.html('Please wait while we load RateMyProfessors.com'); profTable.dialog('open'); - $.ajax({ - url: "/prof", - type: 'get', - data: 'name=' + name, - dataType: 'json', - beforeSend: function() { - $('body').css({'cursor': 'wait'}); - if(!name) { - return false; - } - }, - success: function(data) { - var trs = 'ProfessorDepartment# RatingsQualityEasinessHot'; - if (data.length > 0) { - for (var i=0; i'+p.name+''+p.dept+''+p.ratings+''+p.quality+''+p.easiness+''+p.hot+''; - } - } else { - trs += ''+name+' 0 found   '; - } + var trs = 'ProfessorDepartment# RatingsQualityEasinessHot'; + nameList = nameString.split('
'); + for (var n=0; n 0 found   '; profTable.html(trs); - }, - error: function(jqXHR, textStatus, errorThrown) { - profTable.html('An error occured:'+textStatus+''+errorThrown+''); - }, - complete: function(jqXHR, textStatus) { - $('body').css({'cursor': 'auto'}); - } - }); + } else { + $.ajax({ + url: "/prof", + type: 'get', + data: 'name=' + name, + dataType: 'json', + beforeSend: function() { + $('body').css({'cursor': 'wait'}); + }, + success: function(data) { + if (data.length > 0) { + for (var i=0; i'+p.name+''+p.dept+''+p.ratings+''+p.quality+''+p.easiness+''+p.hot+''; + } + } else { + trs += ''+name+' 0 found   '; + } + profTable.html(trs); + }, + error: function(jqXHR, textStatus, errorThrown) { + profTable.html('An error occured:'+textStatus+''+errorThrown+''); + }, + complete: function(jqXHR, textStatus) { + $('body').css({'cursor': 'auto'}); + } + }); + } + } } $(document).ready(function() {