50行代码爬取Top500图书导入TXT文档

一时失言乱红尘 2023-01-06 12:00 97阅读 0赞

## 50行代码爬取Top500图书导入TXT文档 ##

import re   #正则表达式，进行文字提取
    import requests
    import json
    
    def main(page):
        #声明爬取网址
        baseurl = "http://bang.dangdang.com/books/fivestars/01.00.00.00.00.00-recent30-0-0-1-" + str(page)
        #爬取网页内容
        datalist = getData(baseurl)
        #保存网页数据
        savepath = "Top500_book.txt"
        saveData(datalist,savepath)
    
    #得到数据
    def getData(baseurl):
        html = askURL(baseurl)
        datalist = parse_result(html)
        return datalist
    #对源码进行解析
    def parse_result(html):
       pattern = re.compile('<li>.*?list_num.*?(\d+).</div>.*?<img src="(.*?)".*?class="name".*?title="(.*?)">.*?class="star">.*?class="tuijian">(.*?)</span>.*?class="publisher_info">.*?target="_blank">(.*?)</a>.*?class="biaosheng">.*?<span>(.*?)</span></div>.*?<p><span\sclass="price_n">¥(.*?)</span>.*?</li>',re.S)
       items = re.findall(pattern,html)
       for item in items:
           yield {
        
               'range': item[0],
               'iamge': item[1],
               'title': item[2],
               'recommend': item[3],
               'author': item[4],
               'times': item[5],
               'price': item[6]
           }
    #获取网页源码
    def askURL(url):
        try:
            response = requests.get(url)
            if response.status_code == 200:
                return response.text
        except requests.RequestException:
            return None
    #保存数据到txt文本文档
    def saveData(datalst,savepath):
        print("save....")
        for item in datalst:
            with open(savepath, 'a', encoding='UTF-8') as f:
                f.write(json.dumps(item, ensure_ascii=False) + '\n')
                f.close()
    
    if __name__ == '__main__':
        #for循环实现翻页
        for i in range(1,26):
            main(i)

【运行结果】  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2dldHNfcw_size_16_color_FFFFFF_t_70]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2dldHNfcw_size_16_color_FFFFFF_t_70]: /images/20221119/79639d5fa611493f8f5142bdf589c8e5.png