Q: I want to extract IMDb movie IDs using python Here is my code:
So I wanted to extract all the bollywood movies, and the project requires, movie titles, cast, crew, IMDB id etc.... I am not able to get all the IMDb IDs with the error nonetype. When I used it on one page only it was working quite well, however, when I use it on multiple pages it shows an error. Please help
#importing the libraries needed 
import pandas as pd
import numpy as np
import requests
import re
from bs4 import BeautifulSoup
from time import sleep
from random import randint

#declaring the list of empty variables, So that we can append the data overall

movie_name = []
year = []
time=[]
rating=[]
votes = []
description = []
director_s = []
starList= []
imdb_id = []

#the whole core of the script
url = "https://www.imdb.com/search/title/?title_type=feature&primary_language=hi&sort=num_votes,desc&start=1&ref_=adv_nxt"
page = requests.get(url)
soup = BeautifulSoup(page.text, 'html.parser')
movie_data = soup.findAll('div', attrs = {'class': 'lister-item mode-advanced'})

for store in movie_data:
    name = store.h3.a.text
    movie_name.append(name)
    
    year_of_release = store.h3.find('span', class_ = "lister-item-year text-muted unbold").text
    year.append(year_of_release)
        
    runtime = store.p.find("span", class_ = 'runtime').text if store.p.find("span", class_ = 'runtime') else " "
    time.append(runtime)
        
    rate = store.find('div', class_ = "inline-block ratings-imdb-rating").text.replace('\n', '') if store.find('div', class_ = "inline-block ratings-imdb-rating") else " "
    rating.append(rate)
        
    value = store.find_all('span', attrs = {'name': "nv"})
        
    vote = value[0].text if store.find_all('span', attrs = {'name': "nv"}) else " "
    votes.append(vote)
        
    # Description of the Movies 
    describe = store.find_all('p', class_ = 'text-muted')
    description_ = describe[1].text.replace('\n', '') if len(describe) > 1 else ' '
    description.append(description_)
        
    ## Director  
    ps = store.find_all('p')
    for p in ps:
        if 'Director'in p.text:
            director =p.find('a').text
    
    director_s.append(director)
    
    ## ID
    imdbID = store.find('span','rating-cancel').a['href'].split('/')[2]
    imdb_id.append(imdbID)

    ## actors
    star = store.find("p", attrs={"class":""}).text.replace("Stars:", "").replace("\n", "").replace("Director:", "").strip()
    starList.append(star)


Error:
AttributeError                            Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_17576/2711511120.py in <module>
     63 
     64         ## IDs
---> 65         imdbID = store.find('span','rating-cancel').a['href'].split('/')[2] if store.find('span','rating-cancel').a['href'].split('/')[2] else ' '
     66         imdb_id.append(imdbID)
     67 

AttributeError: 'NoneType' object has no attribute 'a'


A: Change your condition to the following, cause first you have to check if <span> exists:
imdbID = store.find('span','rating-cancel').a.get('href').split('/')[2] if store.find('span','rating-cancel') else ' '

Example
Check the url, here are some of the <span> missing:
import requests
from bs4 import BeautifulSoup

#the whole core of the script
url = "https://www.imdb.com/search/title/?title_type=feature&primary_language=hi&sort=my_ratings,desc"
page = requests.get(url)
soup = BeautifulSoup(page.text, 'html.parser')
movie_data = soup.find_all('div', attrs = {'class': 'lister-item mode-advanced'})

for store in movie_data:
    imdbID = store.find('span','rating-cancel').a.get('href').split('/')[2] if store.find('span','rating-cancel') else ' '
    print(imdbID)

Output
tt9900050
tt9896506
tt9861220
tt9810436
tt9766310
tt9766294
tt9725058
 
tt9700334
 
tt9680166
 
tt9602804
 
 


Even better scrape the id via image tag cause these is always there even if there is only the placholder:
imdbID = store.img.get('data-tconst')

