-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathweb_scrape.py
37 lines (24 loc) · 826 Bytes
/
web_scrape.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
from bs4 import BeautifulSoup
import requests
import csv
source = requests.get('http://coreyms.com').text
soup = BeautifulSoup(source)
csv_file = open('cms_scrape.csv', 'w')
csv_writer = csv.writer(csv_file)
csv_writer.writerow(['headline', 'summary', 'video_link'])
for article in soup.find_all('article'):
headline = article.h2.a.text
print(headline)
summary = article.find('div', class_='entry-content').p.text
print(summary)
try:
vid_src = article.find('iframe', class_='youtube-player')['src']
vid_id = vid_src.split('/')[4]
vid_id = vid_id.split('?')[0]
yt_link = f'https://youtube.com/watch?v={vid_id}'
except Exception as e:
yt_link = None
print(yt_link)
print()
csv_writer.writerow([headline, summary, yt_link])
csv_file.close()