Q: Having trouble in scraping table data using beautiful soup I would like to scrape the table data from this site. I've tried the code below but for whatever reason, BS4 seems unable to fetch the table data:
import bs4 as bs
import urllib.request

sauce = urllib.request.urlopen('https://drafty.cs.brown.edu/csprofessors').read()
soup = bs.BeautifulSoup(sauce, 'lxml')
table = soup.find('table',  attrs={"id": "table"})
table_rows = table.find_all('tr')

for tr in table_rows:
    td = tr.find_all('td')
    row = [i.text for i in td]
    print(row)


I would really appreciate your help :)

A: You used wrong tag and id name to find the right table. The following should work:
import bs4 as bs
import urllib.request

sauce = urllib.request.urlopen('https://drafty.cs.brown.edu/csprofessors').read()
soup = bs.BeautifulSoup(sauce, 'lxml')
table = soup.find('template', attrs={"id":"table-data"})
for tr in table.find_all('tr'):
    td = tr.find_all('td')
    row = [i.text for i in td]
    print(row)


A: import requests
from bs4 import BeautifulSoup as bs4


url = ('https://drafty.cs.brown.edu/csprofessors')
response = requests.get(url)
if response.ok:
    data = list()
    soup = bs4(response.text, 'html.parser')
    fullnames = soup.select('td:nth-child(1)')
    university = soup.select('td:nth-child(2)')
    join_year = soup.select('td:nth-child(3)')
    sub_field = soup.select('td:nth-child(4)')
    bachelors = soup.select('td:nth-child(5)')
    doctorate = soup.select('td:nth-child(6)')
    for item in range(1, len(fullnames) + 1):
        data.append(
            [
                {
                'fullnames': fullnames,
                'university': university,
                'join_year': join_year,
                'sub_field': sub_field,
                'bachelors': bachelors,
                'doctorate': doctorate
                }
            ]
        )


A: You can simply use selenium combined with pandas to scrape the table. Here is how you do it:
import pandas as pd
from selenium import webdriver
import time

url = 'https://drafty.cs.brown.edu/csprofessors'

driver = webdriver.Chrome()

driver.get(url)

time.sleep(2)

driver.find_element_by_xpath('//*[@id="welcome-screen"]/div/div/div[1]/button').click()

time.sleep(1)

page = driver.page_source

df = pd.read_html(page)[0]

print(df)

