import sys, time, os, requests, re, pandas as pd, time, datetime
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import ElementNotVisibleException, NoSuchElementException
from datetime import datetime, timedelta
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import json

#Excelからキーワード読み込み
kwd_list_file = "./kwd_list/kwd_list.xlsx"
kwd_list = pd.read_excel(kwd_list_file, sheet_name="MTG")

#キーワードリストからスクレイピング
def mercari_output(kwd_list):
    #option セッティング
    chrome_options = Options()
    #chrome_options.add_argument("--headless")
    chrome_options.binary_location='C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe'
    chrome_driver_path = ("C:\\Users\\wada\\Documents\\chrome_driver\\chromedriver.exe")

    caps = DesiredCapabilities.CHROME
    caps["goog:loggingPrefs"] = {"performance": "ALL"} 
    driver = webdriver.Chrome(chrome_driver_path, chrome_options=chrome_options, desired_capabilities=caps)
    
    dfs = []
    for i in range(len(kwd_list)):
        kwd = kwd_list.name.iloc[i]
        card_id = kwd_list["横断カード名ID"].iloc[i] 
        #以下のURLフォーマットはURLマスタから取得
        url = "https://jp.mercari.com/search?keyword={}&category_id=1288&sort=created_time&order=desc"
        driver.get(url.format(kwd))
        time.sleep(3)
        netlog = driver.get_log("performance")
        logs = [json.loads(lr["message"])["message"] for lr in netlog]
        for log in logs:
            if log["method"] ==  "Network.responseReceived" and  "json" in log["params"]["response"]["mimeType"]:
                if log["params"]["response"]["url"] == "https://api.mercari.jp/v2/entities:search":
                    output = driver.execute_cdp_cmd("Network.getResponseBody", {"requestId": log["params"]["requestId"]})
                    break
        tmp_output = json.loads(output["body"])["items"]
        tmp_output = pd.DataFrame(tmp_output)
        tmp_output = tmp_output.assign(got_time = datetime.now().strftime("%Y%m%d%H%M%S"),
                                        kwd = kwd,
                                        card_id = card_id, )
        dfs.append(tmp_output)
    driver.close()
    return pd.concat(dfs)



def ebay_closed_scrape(kwd_list):
    #option セッティング
    chrome_options = Options()
    #chrome_options.add_argument("--headless")
    chrome_options.binary_location='C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe'
    chrome_driver_path = ("C:\\Users\\wada\\Documents\\chrome_driver\\chromedriver.exe")

    driver = webdriver.Chrome(chrome_driver_path, chrome_options=chrome_options)
    dfs = []

    for i in range(len(kwd_list)):
        kwd = kwd_list.name.iloc[i]
        card_id = kwd_list["横断カード名ID"].iloc[i] 
        #以下のURLフォーマットはURLマスタから取得
        url = "https://www.ebay.com/sch/i.html?_dcat=183454&_fsrp=1&rt=nc&_from=R40&_nkw={}&_sacat=183454&LH_TitleDesc=0&LH_Complete=1&LH_Sold=1&Game=Magic%253A%2520The%2520Gathering"
        driver.get(url.format(kwd))
        
        
        item_tags = driver.find_elements_by_xpath("//div[@class='s-item__info clearfix']")
        item_tags = item_tags[1:]
              
        closed_times = []
        prices =[]
        titles = []
        infos = []
        item_statuses =[]
        urls = []
        sellers = []
        
        for j in range(len(item_tags)):
            item_tag = item_tags[j]            
            closed_times.append(item_tag.find_element_by_xpath(".//div[@class='s-item__title--tag']/span[@class='POSITIVE']").text)
            prices.append(item_tag.find_element_by_xpath(".//span[@class='s-item__price']/span[contains(@class,'POSITIVE')]").text)
            titles.append(item_tag.find_element_by_xpath(".//div[@class='s-item__title']/span[@role='heading']").text) 
            #item_statuses.append(item_tag.find_element_by_xpath(".//div[@class='s-item__subtitle']/span").text)
            try:
                item_status = item_tag.find_element_by_xpath(".//div[@class='s-item__subtitle']/span").text
            except NoSuchElementException:
                item_status = ""
            item_statuses.append(item_status)
            
            infos.append("".join(list(map(lambda x: x.text, item_tag.find_elements_by_xpath(".//div[@class='s-item__detail s-item__detail--primary']")))))
            urls.append(item_tag.find_element_by_xpath(".//a[@href]").get_attribute("href").split("?")[0])
            sellers.append(item_tag.find_element_by_xpath(".//span[@class='s-item__seller-info-text']").text)
        
        tmp_df = pd.DataFrame([closed_times, prices, titles, item_statuses, infos, urls, sellers], 
                              index=[ "closed_time", "price", "title", "item_status", "info", "url", "seller"] ).T.assign(kwd=kwd, card_id = card_id, got_time = datetime.now().strftime("%Y%m%d%H%M%S"))
        
        driver.close()
        dfs.append(tmp_df)
        return pd.concat(dfs)