from io import StringIO
import pandas as pd
import requests
import itertools
import datetime as datetime
from time import sleep

def get_charts_data(datelist):
  result_charts_list = []

  for charts_date in datelist:
    endpoint_url = "https://spotifycharts.com/regional/jp/daily/"+ str(charts_date) + "/download"

    sleep(0.5)
    try:
      res = requests.get(endpoint_url, timeout=(1800.0, 180.5))
    except:
      raise
    if res.status_code == 200:
      str_content=str(res.content,'utf-8')
      data = StringIO(str_content) 

      df=pd.read_csv(data,header=1)
      rename_df = df.rename(
          columns={
              'Position': 'position', 
              'Track Name': 'track_name', 
              'Artist': 'artist', 
              'Streams': 'streams', 
              'URL': 'url'
              })
      rename_df["date"] = str(charts_date)
      result_charts_list.append(rename_df.values.tolist())
  return result_charts_list

def main():
  START_DATE = "2016-12-31"
  END_DATE = "2020-11-05"

  date_from = datetime.datetime.strptime(START_DATE,'%Y-%m-%d')
  date_to = datetime.datetime.strptime(END_DATE,'%Y-%m-%d')
  datelist = []

  while (date_from-date_to)!=datetime.timedelta(days=0):
    date_from = date_from + datetime.timedelta(days=1)
    datelist.append(date_from.strftime('%Y-%m-%d'))
  charts_list = get_charts_data(datelist)
  result_df = pd.DataFrame(list(itertools.chain.from_iterable(charts_list)), index=None,columns=['position','track_name','artist','streams','url','date'])

if __name__ == '__main__':
    main()