【2018年10月時点】産経ニュースから経済カテゴリのニュースの情報をスクレイピング【python】

def get_code(shijo="東証1部"):
    shijolist = []
    sappend = shijolist.append
    q = jsm.Quotes()
    for fieldcode in data:
        for BrandData in data[fieldcode]:
            if(BrandData.market == shijo):
                sappend(BrandData.ccode)
    return shijolist

def complist(code):
    pd.read_csv("stock_list.csv",header = 0,engine = "python")
    compdata = compdata[compdata["銘柄コード"].isin(code)]
    return compdata

def scrape(year="2018"):
    dflist = []
    pageSize = 1
    for i in range(1,pageSize+1):
        url = "https://www.sankei.com/economy/newslist/business-n" + str(i) + ".html"
        bshtml = gethtml(url)
        news = getnewses(bshtml,year)
        dflist.append(news)
    newsdata = dflist[0]
    for i in range(pageSize):
        newsdata = pd.concat([newsdata,dflist[i]])
    return newsdata

def getnewses(bshtml,year):
    newsdata = pd.DataFrame(index = [],columns = ["title","when","contents"])
    section = bshtml.find_all("div",class_= "entry_content")
    
    for elem1 in section:
            news = []
            a = elem1.select("a")
            
            test = a[0].get("href")
            contents_url = test
            contents = gethtml(contents_url)
            c_section = contents.find_all("div",class_="post_content")
            arts = c_section[0].find_all("p")
            
            tbody = ""
            for art in arts:
                if(art.string):
                    tbody += art.string
            span = elem1.find("div",class_ ="entry_meta")
            try:
                when_raw = str(span.select("time")[0].string)
                timelist = when_raw.split(" ")
                month,day = timelist[0].split(".")
                hour,minute = timelist[1].split(":")
                when = pd.datetime(int(year),int(month),int(day),int(hour))
            except:
                print("no time data")
                continue
                
            news.append(a[0].string)
            news.append(when)
            news.append(tbody)
            news = pd.Series(news,index = newsdata.columns)
            newsdata = newsdata.append(news,ignore_index = True)
    return newsdata

def matchtitle(newsdata,compdata):
    """一部上場している企業一覧とタイトル一覧から，企業名が入っているタイトルを抽出"""
    selectnews =  pd.DataFrame(index = [],columns = ["title","when","contents","company","code"]) 
    for compname in compdata["銘柄名"]:
        for newsname in newsdata["title"]:
            if(compname == newsname[:len(compname)]):
                appenddata = newsdata[newsdata["title"] == newsname]
                appenddata["company"]  = compname
                appenddata["code"] = int(compdata.loc[compdata.loc[:,"銘柄名"] == compname,"銘柄コード"])
                selectnews = selectnews.append(appenddata,ignore_index = True)
 
    selectnews[["code"]] = selectnews[["code"]].astype(int)            
    return selectnews

def delete_duplicatedNews(newsdata):
    new_newsdata = pd.DataFrame(index = [],columns = newsdata.columns)
    for no_old,row_old in newsdata.iterrows():
        isAdd = True
        for no_new ,row_new in new_newsdata.iterrows():
            if(row_old["wakachi"] == row_new["wakachi"]):
                isAdd = False
        if(isAdd):
            new_newsdata = new_newsdata.append(row_old,ignore_index = True)
    return new_newsdata