用python爬取链家二手房信息，并把数据存入mongodb

旧城等待， 2023-02-11 04:42 34阅读 0赞

爬取网站：[https://gz.lianjia.com/ershoufang/pg][https_gz.lianjia.com_ershoufang_pg]

爬取链家二手房信息，数据存mongodb，代码如下：

import requests
    from fake_useragent import UserAgent
    from lxml import etree
    import time
    import random
    
    from pymongo import MongoClient
    
    class LianjiaSpider(object):
    
        def __init__(self):
            self.url = 'https://gz.lianjia.com/ershoufang/pg{}/'
    
        # 获取页面，异常处理
        def get_html(self, url):
            ua = UserAgent()
            headers = { 'User-Agent': ua.random}
            response = requests.get(url=url, headers=headers)
            return response.content.decode('utf-8')
    
        # 解析页面
        def parse_html(self, url):
            html = self.get_html(url)
            # with open('test.html', 'w', encoding='utf-8') as f:
            # f.write(html)
            parse_obj = etree.HTML(html)
    
            li_list = parse_obj.xpath('//ul[@class="sellListContent"]/li')
            
            # 2.for循环一次遍历每个房源li节点，获取1房源的所有注释
            house_list = []
            item = { }
            for li in li_list:
                # 名称[0]:第一个房源,strip移除空格
                item['name'] = li.xpath('.//div[@class="positionInfo"]/a[1]/text()')[0].strip()
                # 信息 户型+面积+方向+装修+楼层+时间+类型
                # 2室1厅 | 51.99平米 | 西 | 精装 | 中楼层(共18层) | 2000年建 | 塔楼
                info_list = li.xpath('.//div[@class="houseInfo"]/text()')[0].split('|')
                item['model'] = info_list[0].strip()
                item['area'] = info_list[1].strip()
                item['direction'] = info_list[2].strip()
                item['perfect'] = info_list[3].strip()
                item['floor'] = info_list[4].strip()
                item['time'] = info_list[5].strip()[:-2]
                item['type'] = info_list[-1].strip() if len(info_list) == 6 else None
                # 地区 总价 单价
                item['address'] = li.xpath('.//div[@class="positionInfo"]/a[2]/text()')[0].strip()
                item['total price'] = li.xpath('.//div[@class="totalPrice"]/span/text()')[0].strip()
                item['unit price'] = li.xpath('.//div[@class="unitPrice"]/span/text()')[0].strip()
                house_list.append(item)
    
            lianjia_info = dict()
            lianjia_info['url'] = self.url
            lianjia_info['描述'] = '链家二手房信息'
            lianjia_info['信息'] = house_list
            return lianjia_info
    
        def save_data(self, data):
            client = MongoClient('mongodb://localhost:27017')
            collection = client["student_de"]
            collection['class_one'].insert(data)
    
        def run(self):
            for pg in range(1, 3):
                url = self.url.format(pg)
                data = self.parse_html(url)
                self.save_data(data)
                time.sleep(random.randint(1, 3))
    
    
    if __name__ == '__main__':
        spider = LianjiaSpider()
        spider.run()

执行成功，结果如下图：

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2N1aV95b25naHVh_size_16_color_FFFFFF_t_70]

[https_gz.lianjia.com_ershoufang_pg]: https://gz.lianjia.com/ershoufang/pg
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2N1aV95b25naHVh_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/2020052119393620.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2N1aV95b25naHVh,size_16,color_FFFFFF,t_70