php操作selenuim+webdriver爬取基金历史数据

秒速五厘米 2023-03-06 05:30 29阅读 0赞

### 本文教大家怎么用php操作selenuim的webdriver来爬取基金数据 ###

先搭建selenium环境：selenium需要jdk环境，自己安装上去就行了，java -version可以查看你是否安装了，我下载的selenium是

目前的稳定版本 selenium-server-standalone-3.141.59.jar  可以在selenuim的官网下载：[https://www.selenium.dev/downloads/][https_www.selenium.dev_downloads]

我用的是chrome浏览器，版本 79.0.3945.36（正式版本） （64 位）然后找到对应的webdriver版本，找到你的：

[https://sites.google.com/a/chromium.org/chromedriver/downloads][https_sites.google.com_a_chromium.org_chromedriver_downloads]

注意：你的webdriver一定要跟浏览器的版本一样哦！！！

把你下载的chromedriver.exe跟selenium-server-standalone-3.141.59.jar 放在同一个文件夹

然后启动selenium :

java -jar selenium-server-standalone-3.141.59.jar

### 爬取基金数据思路: ###

1.先爬目前市面上所有的基金，把数据存到数据库

2.遍历所有基金，然后分别爬取基金的历史数据

3.工具：yaf框架+selenuim+webdriver爬取基金历史数据，

4.关于selenium和webdriver怎么使用的可以参考https://github.com/facebook/php-webdriver

5.注意事项，不要爬太快，慢慢爬，我是每翻一页就休眠0.1秒，后来我取消了休眠，爬几百个基金后就报错了，又重新启动他。

然后又爬几百个又报错了，还是翻一页休眠0.1秒吧，这样慢慢爬就没有问题了。

基金历史数据：

![20200805174744778.png][]

基金数量：

![20200805174845258.png][]

### **Talk is cheap , Show you my code .** ###

/**
     * 爬取目前所有的基金数量,目前有一万多个    php index.php "index/juhe/fundHistory"
     */ 
    public function fundTotalAction(){
            $url = "http://fund.eastmoney.com/js/fundcode_search.js";
    
            $res = https_request($url);
    
            $out = "";
            preg_match("/\[\[(.*)\]\]/",$res,$out);
            if(!isset($out[0])||!$out[0]){
                dd("spide数据有问题");
            }
    
            $funds = json_decode($out[0],true);
    
            foreach($funds as $k=>$v){
                if(!isset($v[0])||!isset($v[1])||!isset($v[2])||!isset($v[3])||!isset($v[4])){
                    continue;
                }
                $insert = array(
                    "fund_id" =>$v[0],
                    "fund_code" =>$v[1],
                    "fund_name" =>$v[2],
                    "fund_type" =>$v[3],
                    "pingyin" =>$v[3],
                    "spide_time" =>$v[0],
                    "spider_status" =>$v[0],
                );
    
                //查询基金是否存在,存在则更新,不存在就插入
                $fund_id = Db::name("fund_total")->where("fund_id='{$v[0]}'")->value("fund_id");
                if($fund_id){
                    Db::name("fund_total")->where("fund_id='{$v[0]}'")->update($insert);
                }else{
                    Db::name("fund_total")->insert($insert);
                }
                unset($insert);
            }
    
        }
        /**
         * 爬取遍历所有基金,分别爬取每个基金的历史数据爬了2天多，爬出来是800多万条
         * php index.php "index/juhe/fundHistory"
         */
        public function fundHistoryAction(){
    
            //$source = Db::name("fund")->where("status=1")->select();
            //$source = Db::name("fund_total")->where("id",">",8830)->select();
            $source = Db::name("fund_total")->select();
            //dd($source);
            if(!$source){
                dd("没有需要pa的数据");
            }
            $fundids = Select2array($source,'fund_id');
            header("Content-Type: text/html; charset=UTF-8");
            $host = 'http://localhost:4444/wd/hub'; // this is the default
            $driver = RemoteWebDriver::create($host, \Facebook\WebDriver\Remote\DesiredCapabilities::chrome(), 5000);
            //dd($fundids);
    
            $start = time();
            #set_time_limit(120);//用cli模式执行,不要设置时间
            header("Content-Type: text/html; charset=UTF-8");
    
            foreach($fundids as $k=>$v){
                $fund_id = $v;
                $rand = mt_rand(12372573208,42372573208);
                $time = time();
                $url = "https://stock.finance.sina.com.cn/fundInfo/api/openapi.php/CaihuiFundInfoService.getNav?callback=jQuery1112051806{$rand}_{$time}111&symbol={$fund_id}&datefrom=&dateto=&page=1&_={$time}254";
    
                $driver->get($url);
                $driver->wait(60);
                $html = $driver->getPageSource();
    
                preg_match("/\d$(.*)$/",$html,$output);
    
                if(!isset($output[1])||!$output[1]){
                    file_log("fund_history",$fund_id." 没有匹配到数据1");
                    continue;
                }
                $detail = json_decode($output[1],true);
                if(!isset($detail['result']['data']['data'])||!$detail['result']['data']['data']){
                    file_log("fund_history",$fund_id." 没有获取到数据1");
                    continue;
                }
    
                //总条数
                $totalCount = isset($detail['result']['data']['total_num'])?$detail['result']['data']['total_num']:'';
                $fundDetail = isset($detail['result']['data']['data'])?$detail['result']['data']['data']:'';
                //p($totalCount);dd($fundDetail);
                foreach($fundDetail as $k2=>$v2){
                    usleep(100000);//延迟0.1秒,还是得延迟啊，不延迟的话爬着爬着就被网站xx的哦！
                    $fundDay = date("Y-m-d",strtotime($v2['fbrq']));
                    $insert = array(
                        "fund_id"=>$fund_id,
                        "fbrq"=>$fundDay,
                        "jjjz"=>$v2['jjjz'],
                        "ljjz"=>$v2['ljjz'],
                    );
    
                    $where = "fund_id={$fund_id} and fbrq = '{$fundDay}'";
                    //查询这条数据是否存在,如果存在就更新,不存在就插入
                    $history_id = Db::name("fund_history")->where($where)->value("history_id");
                    if($history_id){
                        Db::name("fund_history")->where($where)->update($insert);
                    }else{
                        Db::name("fund_history")->insert($insert);
                    }
    
                    unset($insert);
                }
    
                //查看是否有更多的历史数据
                $totalPage = ceil($totalCount/20);
                for($i=2;$i<=$totalPage;$i++){
                    $rand = mt_rand(12372573208,42372573208);
                    $time = time();
                    $url = "https://stock.finance.sina.com.cn/fundInfo/api/openapi.php/CaihuiFundInfoService.getNav?callback=jQuery1112051806{$rand}_{$time}111&symbol={$fund_id}&datefrom=&dateto=&page={$i}&_={$time}254";
    
                    $driver->get($url);
                    $driver->wait(60);
    
                    $html = $driver->getPageSource();
    
                    preg_match("/\d$(.*)$/",$html,$output);
    
                    if(!isset($output[1])||!$output[1]){
                        file_log("fund_history",$fund_id." 没有匹配到数据2");
                        continue;
                    }
                    $detail = json_decode($output[1],true);
                    if(!isset($detail['result']['data']['data'])||!$detail['result']['data']['data']){
                        file_log("fund_history",$fund_id." 没有获取到数据2");
                        continue;
                    }
    
                    //数据详情
                    $fundDetail = isset($detail['result']['data']['data'])?$detail['result']['data']['data']:'';
    
                    foreach($fundDetail as $k3=>$v3){
                        if(!isset($v3['jjjz'])||!isset($v3['ljjz'])){
                            file_log("fund_history",$fund_id." 跳出了循环2");
                            continue;
                        }
                        $fundDay = date("Y-m-d",strtotime($v3['fbrq']));
                        $insert2 = array(
                            "fund_id"=>$fund_id,
                            "fbrq"=>$fundDay,
                            "jjjz"=>$v3['jjjz'],
                            "ljjz"=>$v3['ljjz'],
                        );
    
                        //查询这条数据是否存在,如果存在就更新,不存在就插入
                        $where = "fund_id={$fund_id} and fbrq = '{$fundDay}'";
                        $history_id = Db::name("fund_history")->where($where)->value("history_id");
                        if($history_id){
                            Db::name("fund_history")->where($where)->update($insert2);
                        }else{
                            Db::name("fund_history")->insert($insert2);
                        }
                        unset($insert2);
                    }
                }
                
            }
    
            $driver->quit();
            $useTime = time()-$start;
            echo "消耗时间{$useTime}秒";exit();
        }

[https_www.selenium.dev_downloads]: https://www.selenium.dev/downloads/
[https_sites.google.com_a_chromium.org_chromedriver_downloads]: https://sites.google.com/a/chromium.org/chromedriver/downloads
[20200805174744778.png]: /images/20230208/7b25fbe7f23d4f018258835f4dff54ed.png
[20200805174845258.png]: /images/20230208/03e1cf9dea4e4b108618a21d2ad89d45.png