Arknights - scrape operator level up stats

save_scraped_data_in_df.py

# Imports
# (...)

# selenium set up
# (...)

# scrape dictionary of characters and output as pickle
# (...)

# Running dataframe to collect operator data into
main_df = pd.DataFrame(
    {
        "operator": list(),
        "rarity": list(),
        "class": list(),
        "promotion_level": list(),
        "level": list(),
        "hp": list(),
        "attack": list(),
        "defense": list(),
        "resistance": list(),
        "redeployment_time": list(),
        "dp_cost": list(),
        "block_count": list(),
        "attack_interval": list(),
        "cn_release_date": list(),
        "global_release_date": list(),
        "is_limited": list()
    }
)

# Refactored function
# Given the number of levels, it levels up the character that many times and reads its stats each time
def get_stats_per_level(num_levels: int, df: pd.DataFrame) -> pd.DataFrame:

    for i in range(num_levels):
        
        # Logic to scrape data seen before
        # (...)
        
        # To add the new row of data, it can be added as a dictionary
        row_to_append = {
            "operator": operator,
            "rarity": operator_rarity,
            "class": operator_class,
            "promotion_level": promotion_level,
            "level": operator_level,
            "hp": op_hp,
            "attack": op_atk,
            "defense": op_def,
            "resistance": op_res,
            "redeployment_time": op_redeploy,
            "dp_cost": op_cost,
            "block_count": op_block,
            "attack_interval": op_atk_interv,
            "cn_release_date": cn_release_date,
            "global_release_date": global_release_date,
            "is_limited": operator_is_limited
        }
        # And now append that dictionary as a new row
        df = df.append(row_to_append, ignore_index=True)
        
        # Same code as before but I kept in the snippet because it's short
        # Get the arrow to increase operator level
        increase_level_button = driver.find_element_by_class_name("fa-arrow-right")
        # Click the button (increase 1 level)
        increase_level_button.click()

    # And now the dataframe with the latest data is returned
    # Return the DF with all the data accumulated so far
    return df

# Remaining code seen before to scrape data
for operator in op_dict:
# (...)
    if len(rank_buttons) == 1:
        # Get stats for all E0 levels
        main_df = get_stats_per_level(30, main_df)

    elif len(rank_buttons) == 2:
        # Get stats for all E0 levels
        main_df = get_stats_per_level(40, main_df)
        # Now promote to E1 i.e. click the E1 button
        rank_buttons[1].click()
        main_df = get_stats_per_level(55, main_df)

    elif len(rank_buttons) == 3:
        # Get stats for all E0 levels
        main_df = get_stats_per_level(50, main_df)
        # Now promote to E1 i.e. click the E1 button
        rank_buttons[1].click()
        main_df = get_stats_per_level(80, main_df)
        # Now promote to E2 i.e. click the E2 button
        rank_buttons[2].click()
        main_df = get_stats_per_level(90, main_df)

# Scraping completed, time to output the CSV
# Remove duplicate rows because of the issue discussed earlier
main_df.drop_duplicates(inplace=True)
# main_df.to_csv("arknights_operator_stats.csv", index=False)
main_df.to_csv("test.csv", index=False)

# Close the Selenium Chrome driver
driver.quit()