Data-Analysis-Jupyter/medium/bargraphs.py

from bs4 import BeautifulSoup
from selenium import webdriver
from dateutil import parser
from datetime import datetime, timedelta
import pandas as pd
import math
import time

from webdriver_manager.chrome import ChromeDriverManager


def get_all_pages(driver, xpath, months, suffix):

    # Initially starting at today
    latest_date_in_graph = datetime.now().date()

    print("Starting on ", latest_date_in_graph)

    views = []
    dates = []

    # Iterate through the graphs
    for m in range(months + 1):
        graph_views = []
        graph_dates = []
        # Extract the bar graph
        bargraph = BeautifulSoup(driver.page_source).find_all(
            attrs={"class": "bargraph"}
        )[0]

        # Get all the bars in the bargraph
        bardata = bargraph.find_all(attrs={"class": "bargraph-bar"})
        # Sort the bar data by x position (which will be date order) with most recent first
        bardata = sorted(bardata, key=lambda x: float(x.get("x")), reverse=True)
        bardata = [bar.get("data-tooltip") for bar in bardata]
        latest_day = int(bardata[0].split("\xa0")[-1])

        # Some months are not overlapping
        if latest_day != latest_date_in_graph.day:
            latest_date_in_graph -= timedelta(days=1)

        # Iterate through the bars which now are sorted in reverse date order (newest to oldest)
        for i, data in enumerate(bardata):
            graph_views.append(float(data.split(" ")[0].replace(",", "")))
            graph_dates.append(latest_date_in_graph - timedelta(days=i))

        views.extend(graph_views)
        dates.extend(graph_dates)
        # Find the earliest date in the graph
        earliest_date_in_graph = graph_dates[-1]

        # Update the latest date in the next graph
        latest_date_in_graph = earliest_date_in_graph

        # Go to the previous graph
        driver.find_element_by_xpath(xpath).click()
        time.sleep(2)
        print(f"{100 * m /(months):.0f}% complete.", end="\r")

    results = (
        pd.DataFrame({"date": pd.to_datetime(dates), suffix: views})
        .groupby("date")
        .sum()
    )
    results = results.loc[
        results[results["views"] != 0.0].index.min() :,
    ]
    print("First views on ", str(results.index.min().date()))

    # Save using the date as the file name
    fname = f"data/{str(datetime.now().date())}_{suffix}"
    results.to_parquet(fname)
    print("Stats saved to ", fname)

    return results


if __name__ == "__main__":
    # Go to the website
    driver = webdriver.Chrome(ChromeDriverManager().install())
    driver.get("https://medium.com/me/stats")
    # Wait for user to log in
    input("Waiting for you to log in. Press enter when ready: ")

    # Find earliest date
    earliest_article_date = parser.parse(
        input("Enter earliest article date as string: ")
    ).date()
    days = (datetime.now().date() - earliest_article_date).total_seconds() / (
        60 * 60 * 24
    )
    months = math.ceil(days / 30)

    # Get the xpath from user
    xpath = input("Paste xpath with no quotation marks: ")
    # Gather the results
    results = get_all_pages(driver, xpath, months, suffix="views")
    print("Refresh page and click on reads")

    # Get the xpath from user
    xpath = input("Paste xpath with no quotation marks: ")
    # Gather the results
    results = get_all_pages(driver, xpath, months, suffix="reads")

    print("Refresh page and click on fans")
    # Get the xpath from user
    xpath = input("Paste xpath with no quotation marks: ")
    # Gather the results
    results = get_all_pages(driver, xpath, months, suffix="fans")
    print("Complete. All results saved in data directory.")