python获取个人博客所有标题

﹏ヽ暗。殇╰゛Y 2023-06-30 02:17 113阅读 0赞

import requests
    import xlwt
    from bs4 import BeautifulSoup
    import urllib.request
    import time
    import lxml
    import os
    import time
    
    opener = urllib.request.build_opener()
    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
    print("爬取博客开始")
    #添加
    main_url='https://blog.csdn.net/qq_36958104'
    
    page=1
    #url = main_url + '/article/list/' + str(page)
    #https://blog.csdn.net/qq_36958104/article/list/1
    #    qq_36958104
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'}
    ss=None
    flag=0
    count=1
    page_flag=1
    #for count in range(2000):
    while(count<10):
        try:
            url = main_url + '/article/list/' + str(count)
            r=requests.get(url,headers=headers,timeout=10)
            # print(r.text)
            # r.raise_for_status()    
            # print("网页编码是："+r.encoding)
            # # print(r.)
            # # 获取页面内容
            # # prin5t(r.text)
            # 给excel放数据的  一个就是一行
            all_lists = []
            book = xlwt.Workbook(encoding='utf-8')#创建工作簿
            soup = BeautifulSoup(r.text, 'lxml')
            # 下面的这种方法会报错   TypeError("'NoneType' object is not callable")
            # ss=soup.findall("class",class_="article-item-box csdn-tracking-statistics")
            # 找到所有class为article-item-box csdn-tracking-statistics的节点
            row0 = ['文章标题','文章链接']#定义表头，即Excel中第一行标题
            sheet = book.add_sheet('bokewangzhi',cell_overwrite_ok=True)
            sheet.write(0,0,row0[0])#写入表
            
            sheet.write(0,1,row0[1])#写入表
            for s in soup.findAll(name="div", attrs={"class" :"article-item-box csdn-tracking-statistics"}):
                for ss in s.findAll(name="h4"):
                        sss=ss.find(name="a",href=True);
                        #print("文章标题："+ss.getText().replace("","").strip()+"\n文章链接："+sss['href'])
                        print("文章标题："+ss.getText().replace("","").strip())
                        opener.open(sss['href'])#打开网页刷访问
                        #print(sss['href'])#打印网址链接                   
                        list=[ss.getText().replace("原创","").strip()]
                        all_lists.append(list)
                        #print(list)
            #第一行开始
            i=1
            for all_list in all_lists:
                j=0
                for data in all_list:
                        sheet.write(i,j,data)#迭代列，并写入数据，#重新设置，需要cell_overwrite_ok=True
                        j+=1
                i+=1
            book.save("bokewangzhi.xls")
            count=count+1
            print("第",count,"页数据")
            #time.sleep(10)
            if count>5:
               count=0
               flag+=1
               print('爬取次数统计',flag*200)
               print("重新开始")
               del count
               del i
               del all_list
               del list
               time.sleep(60)
        except Exception as e:
         print("出现异常------异常信息："+repr(e));
         #os.system('python pachong3.py') #启动dos
        except urllib.error.HTTPError:
         print('urllib.error.HTTPError')
         time.sleep(60)
        except urllib.error.URLError:
         print('urllib.error.URLError')
         time.sleep(60)

# python获取个人博客所有标题 #

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM2OTU4MTA0_size_16_color_FFFFFF_t_70][]

import requests
    import xlwt
    from bs4 import BeautifulSoup
    import urllib.request
    import time
    import lxml
    import os
    import time
    
    opener = urllib.request.build_opener()
    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
    print("爬取博客开始")
    url="https://blog.csdn.net/qq_36958104"
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'}
    ss=None
    flag=0
    
    for count in range(20000):
        try:
            r=requests.get(url,headers=headers,timeout=30)
            # print(r.text)
            # r.raise_for_status()    
            # print("网页编码是："+r.encoding)
            # # print(r.)
            # # 获取页面内容
            # # print(r.text)
            # 给excel放数据的  一个就是一行
            all_lists = []
            book = xlwt.Workbook(encoding='utf-8')#创建工作簿
            soup = BeautifulSoup(r.text, 'lxml')
            # 下面的这种方法会报错   TypeError("'NoneType' object is not callable")
            # ss=soup.findall("class",class_="article-item-box csdn-tracking-statistics")
            # 找到所有class为article-item-box csdn-tracking-statistics的节点
            row0 = ['文章标题','文章链接']#定义表头，即Excel中第一行标题
            sheet = book.add_sheet('AINY',cell_overwrite_ok=True)
            sheet.write(0,0,row0[0])#写入表
            
            sheet.write(0,1,row0[1])#写入表
            for s in soup.findAll(name="div", attrs={"class" :"article-item-box csdn-tracking-statistics"}):
                for ss in s.findAll(name="h4"):
                        sss=ss.find(name="a",href=True);
                        print("文章标题："+ss.getText().replace("","").strip()+"\n文章链接："+sss['href'])
                        
                        print(sss['href'])#打印网址链接
                        flag+=1
                        print('爬取次数统计',flag)                   
                        list=[ss.getText().replace("原创","").strip(),sss['href']]
                        all_lists.append(list)
            #第一行开始
            i=1
            for all_list in all_lists:
                j=0
                for data in all_list:
                        sheet.write(i,j,data)#迭代列，并写入数据，#重新设置，需要cell_overwrite_ok=True
                        j+=1
                i+=1
            book.save("bokewangzhi.xls")
            print("导出成功！")
            print("等待一分钟之后再次爬取数据！")
            print('%d',count)
            time.sleep(60)
        except Exception as e:
         print("出现异常------异常信息："+repr(e));
         os.system('python pachong3.py') #启动dos
        except urllib.error.HTTPError:
         print('urllib.error.HTTPError')
         time.sleep(60)
        except urllib.error.URLError:
         print('urllib.error.URLError')
         time.sleep(60)

opener.open(sss\['href'\])

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM2OTU4MTA0_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20200114091819472.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzM2OTU4MTA0,size_16,color_FFFFFF,t_70