【百度地图】使用百度地图API爬取小区名字并获取小区范围边界

小鱼儿 2023-06-29 10:45 112阅读 0赞

# 获取小区名字 #

# -*- coding: utf-8 -*-
    
    import numpy as np  # 导入数值计算扩展模块
    import requests     # 导入网页请求模块
    import pymysql      # 导入pymysql模块
    import time, math
    
    print("connecting mysql......\n")
    db = pymysql.connect("localhost", "root", "123456", "community", charset='utf8')  # 链接mysql数据库community
    print("connect successfully,start creating table community_zh in database community\n")
    cursor = db.cursor()  # 创建游标对象
    
    cursor.execute("DROP TABLE IF EXISTS community_zh")  # 如果community_zh存在则drop掉
    c_sql = """CREATE TABLE community_zh( province char(10), city char(10), area char(10), community char(30), address char(50), lat float(9,6), lng float(9,6), uid char(25), detail int(1) )"""
    
    cursor.execute(c_sql)   # 创建一个表
    cursor.execute("ALTER TABLE community_zh CONVERT TO CHARACTER SET utf8 COLLATE utf8_general_ci;")  # 如果保存内容包含中文需要修改字符集为utf-8
    print("table community_zh has been created\n")
    
    
    unit = 60 #60*60个栅格
    lat_partion = [round(x,6) for x in list(np.linspace(21.874881,22.418767, unit))]    # 维度划分，保留6位小数
    lng_partion = [round(y,6) for y in list(np.linspace(113.112379,113.641301, unit))]  # 经度划分，保留6位小数
    
    def get_community():            # 自定义按区域获取小区名称的函数
        for i in range(53, 59):      # 横向栅格索引
            # time.sleep(10)
            for j in range(0, 59):  # 纵向栅格索引
                not_max_page = True # 没有到达最大页面时循环
                page_num = 0        # 从第一页开始
    
                while not_max_page:
                    url = "http://api.map.baidu.com/place/v2/search?query=小区&page_size=20&page_num=" + str(page_num) + "\
                    &bounds=" + str(lat_partion[i]) + "," + str(lng_partion[j])+"," + str(lat_partion[i+1]) + "," + str(lng_partion[j+1]) + \
                    "&output=json&ak="  # 构造请求网址,其中你的ak是你要向百度申请的密钥
                    header = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'}  # 构造请求头
                    response = requests.get(url, headers = header)  # 发出请求
                    time.sleep(1)
    
                    answer = response.json()  # json化
                    total = int(answer["total"])
                    max_page = math.ceil(total / 20)
    
                    if answer['status'] == 0:  # 如果正常返回
                        print("\n# Rectangle (%d,%d) Page %s" % (i, j, page_num))
                        print("# Amount: %s" % len(answer['results']))
                        print("# Total: %s" % total)
    
                        page_num += 1
                        if page_num > max_page:  # 到达最大页时推出循环到下一个单元
                            break
    
                        for k in range(len(answer['results'])): 
                            province = answer['results'][k]['province']    # 省份
                            if province == "澳门特别行政区":
                                break
                            city = answer['results'][k]['city']            # 城市
                            area = answer['results'][k]['area']            # 区域
                            community = answer['results'][k]['name']       # 小区名称
                            address = answer['results'][k]['address']      # 地址
                            lat = answer['results'][k]['location']['lat']  # 纬度
                            lng = answer['results'][k]['location']['lng']  # 经度
                            uid = answer['results'][k]['uid']              # 唯一标识
                            detail = answer['results'][k]['detail']        # 是否具有详情内容
                            insert_data = ("INSERT INTO community_zh(province, city, area, community, address, lat, lng, uid, detail)""VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s)")
                            community_data = ([province, city, area, community, address, lat, lng, uid, detail])  # 组成一条记录
                            print(community_data)
                            cursor.execute(insert_data, community_data)  # 执行插入操作
                            db.commit()  # 主动提交
                    else:
                        print("* The rectangle (%d,%d) contains no community"%(i, j)) 
                        break
    
            
    if __name__=='__main__':
        get_community()

# 获取小区边界 #

# -*- coding: utf-8 -*-
    
    import requests     # 导入网页请求模块
    import pymysql      # 导入pymysql模块
    import time, math
    
    
    # 百度米转百度经纬度
    def meter2Degree(x, y):
        url = "http://api.map.baidu.com/geoconv/v1/?coords="+ x + "," + y + "&from=6&to=5&output=json&ak="
        header = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'}  # 构造请求头
        response = requests.get(url, headers = header)  # 发出请求
        answer = response.json()  # json化
        result = answer["result"]
        lng = result[0]["x"]
        lat = result[0]["y"]
        return lng, lat
    
    # 提取百度米坐标字符串，转为经纬度坐标串
    def coordinateToPoints(coordinates):
        points = ""
        if coordinates and coordinates.index("-") >= 0:
            coordinates = coordinates.split("-")
            temp_coordinates = coordinates[1]
            if temp_coordinates and temp_coordinates.index(",") >= 0:
                temp_coordinates = temp_coordinates.replace(";", "").split(",")
                temp_points = []
                for i in range(0, len(temp_coordinates), 2): 
                    x = temp_coordinates[i]
                    y = temp_coordinates[i + 1]
                    point = { }
                    point["x"] = x
                    point["y"] = y
                    temp_points.append(point)
    
                for point in temp_points:
                    x = point["x"]
                    y = point["y"]
                    lng, lat = meter2Degree(x, y)
                    points += str(lng) + "," + str(lat) + ";"
    
        return points
    
    # 获取边界
    def getBorder(uid):
        url = "http://map.baidu.com/?pcevaname=pc4.1&qt=ext&ext_ver=new&l=12&uid=" + uid
        header = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'}  # 构造请求头
        response = requests.get(url, headers = header)  # 发出请求
        answer = response.json()  # json化
        content = answer["content"]
        points = ""
    
        if "geo" in content and content["geo"] != None and content["geo"] != "":
            geo = content["geo"]
            points = coordinateToPoints(geo)
        
        return points
    
    
    if __name__ == "__main__":
        print("connecting mysql......\n")
        db = pymysql.connect("localhost", "root", "123456", "community", charset='utf8')  # 链接mysql数据库community
        print("connect successfully,start creating table community_zh in database community\n")
        cursor = db.cursor()  # 创建游标对象
        cursor.execute("select community,uid from community_zh")
        rows = cursor.fetchall()
        
        start_flag = False
        with open("./community_border.txt", "a") as wf:
            for row in rows:
                name = row[0]
                uid = row[1]
                if name == "香溪兴苑(小区)":
                    start_flag = True
                if start_flag:
                    points = getBorder(uid)
                    time.sleep(1)
                    content = name + "\t" + points + "\n"
                    print(content)
                    wf.write(content)