| import urllib2 |
| import json |
| import re |
| import pickle |
| from BeautifulSoup import BeautifulSoup |
|
|
| def cleanhtml(raw): |
| cleanr = re.compile('<.*?>|\\n') |
| raw = re.sub(cleanr, ' ', raw) |
| raw = BeautifulSoup(raw).getText() |
| raw = raw.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c", '"').replace(u"\u201d", '"') |
| raw = raw.replace(' .', '.').replace(' ,', ',') |
| raw = ' '.join(raw.split()) |
| return raw |
|
|
| def extract_links(source): |
| links = set([]); |
| i = source.find('<article'); |
| source = source[i:] |
| j = source.find('</section>'); |
| source = source[:j] |
| while source.find('href')>0: |
| i = source.find('href') + 6 |
| source = source[i:] |
| j = source.find('"') |
| links.add(source[:j]) |
| return links |
|
|
| def parse_link(link): |
| html = urllib2.urlopen(link).read() |
| i = html.find('<h1 class="title') |
| html = html[i:] |
| i = html.find('>') + 1 |
| html = html[i:] |
| j = html.find('</h1>') |
| question = html[:j] |
|
|
| i = html.find('<p>') |
| html = html[i:] |
| j = html.find('<div id="mobile-social" class="clearfix">') |
| answer = html[:j] |
|
|
| return cleanhtml(question), cleanhtml(answer) |
|
|
| faq_source = urllib2.urlopen('https://sexetc.org/sex-ed/info-center/faqs/?pageNum=100').read(); |
| term_source = urllib2.urlopen('https://sexetc.org/sex-ed/sex-terms/?pageNum=100').read(); |
| links = list(extract_links(faq_source) | extract_links(term_source)) |
| mapping = {} |
| i = 0 |
| for link in links: |
| q, a = parse_link(link) |
| mapping[q] = a |
| pickle.dump(mapping, open('answers.p', 'w')) |
| i += 1 |
| print i |
| json.dump(mapping, open('../answers.json', 'w'), indent=2, sort_keys=True) |
|
|
