File size: 1,558 Bytes
9ef3204 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | import urllib2
import json
import re
import pickle
from BeautifulSoup import BeautifulSoup
def cleanhtml(raw):
cleanr = re.compile('<.*?>|\\n')
raw = re.sub(cleanr, ' ', raw)
raw = BeautifulSoup(raw).getText()
raw = raw.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c", '"').replace(u"\u201d", '"')
raw = raw.replace(' .', '.').replace(' ,', ',')
raw = ' '.join(raw.split())
return raw
def extract_links(source):
links = set([]);
i = source.find('<article');
source = source[i:]
j = source.find('</section>');
source = source[:j]
while source.find('href')>0:
i = source.find('href') + 6
source = source[i:]
j = source.find('"')
links.add(source[:j])
return links
def parse_link(link):
html = urllib2.urlopen(link).read()
i = html.find('<h1 class="title')
html = html[i:]
i = html.find('>') + 1
html = html[i:]
j = html.find('</h1>')
question = html[:j]
i = html.find('<p>')
html = html[i:]
j = html.find('<div id="mobile-social" class="clearfix">')
answer = html[:j]
return cleanhtml(question), cleanhtml(answer)
faq_source = urllib2.urlopen('https://sexetc.org/sex-ed/info-center/faqs/?pageNum=100').read(); # FAQs
term_source = urllib2.urlopen('https://sexetc.org/sex-ed/sex-terms/?pageNum=100').read(); # terms
links = list(extract_links(faq_source) | extract_links(term_source))
mapping = {}
i = 0
for link in links:
q, a = parse_link(link)
mapping[q] = a
pickle.dump(mapping, open('answers.p', 'w'))
i += 1
print i
json.dump(mapping, open('../answers.json', 'w'), indent=2, sort_keys=True)
|