Q: Writing a for loop that appends data to a unique DF and continues the loop in the range specified This code scrapes one match in the range only, I need the code to loop through each range, scrape the data specified, add it to one df and move on until it's finished in the range.
For some reason the code is stopping at one loop and not continuing on as desired.
import requests
from bs4 import BeautifulSoup
import json
import pandas as pd

colnames = ["x", "y", "xg", "team"]
df = pd.DataFrame(index=colnames)

for id in range(16376,16379):
    understat = f'https://understat.com/match/{id}'
    res = requests.get(understat)
    # parsing the webpage, use .content
    soup = BeautifulSoup(res.content, "lxml")
    scripts = soup.find_all('script')

    # get only shots data, and strip data so we only have json data

    strings = scripts[1].string
    index_start = strings.index("('") + 2
    index_end = strings.index("')")
    json_data = strings[index_start:index_end]
    json_data = json_data.encode('utf8').decode('unicode_escape')
    data1 = json.loads(json_data)

    x = []
    y = []
    xg = []
    team = []
    data_home = data1["h"]
    data_away = data1["a"]


    for index in range(len(data_home)):
        for key in data_home[index]:
            if key == "X":
                x.append(data_home[index][key])
            if key == "Y":
                y.append(data_home[index][key])
            if key == "xG":
                xg.append(data_home[index][key])
            if key == "h_team":
               team.append(data_home[index][key])
               df_h = (x,y,xg,team)


    for index in range(len(data_away)):
        for key in data_away[index]:
            if key == "X":
                x.append(data_away[index][key])
            if key == "Y":
                y.append(data_away[index][key])
            if key == "xG":
                xg.append(data_away[index][key])
            if key == "a_team":
               team.append(data_away[index][key])
               df_a = (x, y, xg, team)

            continue
    # create the df
colnames = ["x", "y", "xg", "team"]

df = pd.DataFrame([x, y, xg, team], index=colnames)
df = df.T


A: I think the problem is you override the lists x,y,xg,team and use only the last iteration to create the df.
If your end goal is to create One dataframe contain the data from all of the iteration, you should try those steps.
Steps:

*

*Create a list before the first loop to keep all the dataframes. list_of_dfs = []

*Create df at the end of each iteration. Just by inserting your df creation into the loop (One indentation)

*Append the created dataframe to the list of dataframes. Just after the df creation: list_of_dfs.append(df)

*after the loops end, concat all the dataframes from the list into one dataframe. You can use: final_df = pd.concat(list_of_dfs, axis=1)
Full code should go like this:
import requests
from bs4 import BeautifulSoup
import json
import pandas as pd

colnames = ["x", "y", "xg", "team"]
df = pd.DataFrame(index=colnames)
list_of_dfs = []

for id in range(16376,16379):
    understat = f'https://understat.com/match/{id}'
    res = requests.get(understat)
    # parsing the webpage, use .content
    soup = BeautifulSoup(res.content, "lxml")
    scripts = soup.find_all('script')

    # get only shots data, and strip data so we only have json data

    strings = scripts[1].string
    index_start = strings.index("('") + 2
    index_end = strings.index("')")
    json_data = strings[index_start:index_end]
    json_data = json_data.encode('utf8').decode('unicode_escape')
    data1 = json.loads(json_data)

    x = []
    y = []
    xg = []
    team = []
    data_home = data1["h"]
    data_away = data1["a"]


    for index in range(len(data_home)):
        for key in data_home[index]:
            if key == "X":
                x.append(data_home[index][key])
            if key == "Y":
                y.append(data_home[index][key])
            if key == "xG":
                xg.append(data_home[index][key])
            if key == "h_team":
               team.append(data_home[index][key])
               df_h = (x,y,xg,team)


    for index in range(len(data_away)):
        for key in data_away[index]:
            if key == "X":
                x.append(data_away[index][key])
            if key == "Y":
                y.append(data_away[index][key])
            if key == "xG":
                xg.append(data_away[index][key])
            if key == "a_team":
               team.append(data_away[index][key])
               df_a = (x, y, xg, team)

            continue
    # create the df
    colnames = ["x", "y", "xg", "team"]

    df = pd.DataFrame([x, y, xg, team], index=colnames)
    df = df.T
    list_of_dfs.append(df)

final_df = pd.concat(list_of_dfs, axis=0)

