python小爬虫(爬取职位信息和博客文章信息)

心已赠人 2022-04-08 15:11 456阅读 0赞

### 1.python爬取招聘信息 ###

简单爬取智联招聘职位信息（仅供学习）

# ！/usr/bin/env python
    # -*-coding:utf-8-*-
    """
    @Author  : xiaofeng
    @Time    : 2018/12/18 16:31
    @Desc : Less interests,More interest. (爬取智联招聘职位数据)
    @Project : python_appliction
    @FileName: zhilianzhaopin.py
    @Software: PyCharm
    @Blog    ：https://blog.csdn.net/zwx19921215
    """
    
    import pymysql as db
    import requests
    
    # mysql配置信息
    mysql_config = {
        'host': '101.0.2.110',
        'user': 'test',
        'password': 'test',
        'database': 'xiaofeng',
        'charset': 'utf8'
    }
    
    # url
    url = 'https://data.highpin.cn/api/JobSearch/Search'
    
    """
    爬取智联招聘职位数据
    @:param page 页码
    @:param position 职位关键字
    """
    
    
    def zhilian(page, position):
        # 封装头信息
        headers = {
            'Referer': 'https://www.highpin.cn/zhiwei/',
            'Origin': 'https://www.highpin.cn',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
            'Accept': 'application/json, text/javascript, * / *; q=0.01',
        }
        # 表单信息
        datas = {
            'Q': position,
            'pageIndex': page
        }
        resp = requests.post(url, data=datas, headers=headers)
        result = resp.json()
        return result
    
    
    """
    控制台输出
    """
    
    
    def print_data(result):
        body = result['body']['JobList']
        print(body)
    
    
    """
    数据入库
    """
    
    
    def insert(result):
        print("insert......")
        database = db.connect(**mysql_config)
        for item in result:
            print(item)
            sql = "INSERT INTO zhilian(JobID,JobTitle,ReferrerType,CompanyName,AnnualSalaryMin," \
                  "AnnualSalaryMax,JobLactionStr,JobLactionID,JobTags\
            ,JobDegree,JobDegreeId,WorkExperience,WorkExperienceID,CompanyIndustry,CompanyIndustryID," \
                  "CompanyType,CompanyTypeID,PublishDate,CompanyScale,SalaryWhite) \
                  VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
    
            # list convert to str
            JobLactionID = str(item['JobLactionID'])
            CompanyIndustryID = str(item['CompanyIndustryID'])
    
            if 'JobTags' in item:
                JobTags = str(item['JobTags'])
            else:
                JobTags = ''
            cursor = database.cursor()
            cursor.execute(sql, (
                item['JobID'], item['JobTitle'], item['ReferrerType'], item['CompanyName'], item['AnnualSalaryMin'],
                item['AnnualSalaryMax'],
                item['JobLactionStr'], JobLactionID, JobTags, item['JobDegree'], item['JobDegreeId'],
                item['WorkExperience'],
                item['WorkExperienceID'], item['CompanyIndustry'], CompanyIndustryID, item['CompanyType'],
                item['CompanyTypeID'], item['PublishDate'], item['CompanyScale'], item['SalaryWhite']))
            database.commit()
            cursor.close()
        database.close()
    
    
    def main(position):
        result = zhilian(1, position)
        page_count = result['body']['PageCount']
        print("---------------共", page_count, "页-------------")
        page = 1
        while page <= page_count:
            print('----------------第', page, '页-----------------')
            result = zhilian(page, position)
            # print_data(result)
            body = result['body']['JobList']
            insert(body)
            page = page + 1
    
    
    if __name__ == '__main__':
        main('java')

控制台输出信息

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3eDE5OTIxMjE1_size_16_color_FFFFFF_t_70][]

入库数据

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3eDE5OTIxMjE1_size_16_color_FFFFFF_t_70 1][]

### 2.python爬取csdn博客文章 ###

python简单爬取csdn博客文章列表（仅供学习）

步骤：

1.分页获取博客url

2.解析html 获取指定信息

# ！/usr/bin/env python
    # -*-coding:utf-8-*-
    """
    @Author  : xiaofeng
    @Time    : 2018/12/20 11:30
    @Desc : Less interests,More interest.（爬取csdn博客文章列表）
    @Project : python_appliction
    @FileName: csdn.py
    @Software: PyCharm
    @Blog    ：https://blog.csdn.net/zwx19921215
    """
    import requests
    from lxml import html
    
    # 声明头信息
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0'
    }
    
    """
    解析html，获取相关数据
    @:param url 网页路径
    """
    
    
    def parse_html(url):
        response = requests.get(url=url, headers=headers)
        text = html.fromstring(response.text)
        hrefs = text.xpath('//h4/a/@href')
        title = text.xpath('//h4/a/text()')
        """
        文章摘要、发布日期、阅读数、评论数等其余属性自行按需获取即可
        """
        # 移出第一个元素链接，不知道为啥 csdn 博客默认都多了一条数据，也就多了一个链接
        hrefs.pop(0)
        titles = []
        # 格式化标题
        for item in title:
            st = str(item).replace('\n', '').strip()
            if st != '':
                titles.append(st)
    
        # 组合输出
        # 移出第一个元素，不知道为啥 csdn 博客默认都多了一条数据 “帝都的凛冬”
        titles.pop(0)
        i = 0
        for item in titles:
            results = {
                '标题': titles[i],
                '链接': hrefs[i]
            }
            i = i + 1
            print(results)
    
    
    """
    自动判断页数
    @:param page_url 页面路径
    @:param page 页号
    """
    
    
    def get_page(page_url, page):
        url = page_url + str(page)
        print('url=', url)
        response = requests.get(url=url, headers=headers)
        text = html.fromstring(response.text)
        next_page = text.xpath(
            '//div[@class="ui-paging-container"]/ul/li[@class="js-page-next js-page-action ui-pager"]/text()')
        if next_page:
            parse_html(url)
            page = page + 1
            get_page(page_url, page)
        else:
            return -1
    
    
    """
    分页爬取相关数据
    @:param page_url 页面路径
    @:param page 页号
    """
    
    
    def get_page2(page_url, page):
        url = page_url + str(page)
        while page <= 10:
            print('\n')
            print("----------------------第", page, "页--------------------")
            print('url=', url)
            print('\n')
            parse_html(url)
            page = page + 1
            url = page_url + str(page)
    
    
    if __name__ == '__main__':
        page_url = 'https://blog.csdn.net/zwx19921215/article/list/'
        get_page2(page_url, 1)

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3eDE5OTIxMjE1_size_16_color_FFFFFF_t_70 2][]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3eDE5OTIxMjE1_size_16_color_FFFFFF_t_70]: /images/20220404/03a6201ee2b34dcdab0941cb46d8eecb.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3eDE5OTIxMjE1_size_16_color_FFFFFF_t_70 1]: /images/20220404/ffc36fc7b6b14fbba19b58bfff02f8f0.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3eDE5OTIxMjE1_size_16_color_FFFFFF_t_70 2]: /images/20220404/cdc24eae18d84f82b532528b688087db.png