使用线程池和CountDownLatch计数器来进行抓取的简单事例2

布满荆棘的人生 2022-05-25 00:37 58阅读 0赞

import java.util.List;
    import java.util.concurrent.CountDownLatch;
    import java.util.concurrent.ExecutorService;
    import java.util.concurrent.Executors;
    import org.apache.log4j.Logger;
    import util.ProxyUtil;
    import util.StringUtil;
    public class DictEnToChar implements Runnable{
    static Logger log = Logger.getLogger(DictEnToChar.class);
    static int threadnum = 5;
    public static void main(String[] args) {
    	if(args.length==1){
    		try{
    			threadnum = Integer.parseInt(args[0]);
    		}catch(Exception e){
    			e.printStackTrace();
    			System.out.println("请输入线程个数.为整型参数....");
    		}
    	}
    	DictEnToChar dic = new DictEnToChar();
    	for(char c='A'; c<='Z'; c++){
    		dic.forTotalPage("http://www.*****.com/hotproduct_list_"+c+"_(*).html",c);
    	}
    	DictEn.main(args);
    }
    /**
     * 对每个列表页进行循环
     */
    static int totalPage;//每个cas下标的总页数
    static int startpage;//每个cas下标的开始页
    static String threadurl;//列表页的地址
    static CountDownLatch cdl = new CountDownLatch(threadnum);//计数器,每个cas的下标全部抓完才能抓取下一个下标
    static ExecutorService threadpool = Executors.newCachedThreadPool();//线程池
    public void forTotalPage(String url,int index){
    	String content = ProxyUtil.getProxyStr(StringUtil.changeURL(url, 1), "utf-8", "", "CAS Number");
    	String maxstr = StringUtil.getFirststr(content, "Page 1 of", "<span", 1).trim();
    	try{
    		totalPage = Integer.parseInt(maxstr);
    	}catch(Exception e){
    		e.printStackTrace();
    		System.out.println(index+"获得最大页数失败............");
    	}
    	System.out.println("产品名称的下标为"+index+"的最大页数是:"+totalPage);
    	
    	cdl = new CountDownLatch(threadnum);//计数重置
    	startpage=0;//赋初值
    	threadurl = url;//赋初值
    	DictEnToChar dicrun = new DictEnToChar();
    	for(int num=0; num<threadnum; num++){
    		threadpool.execute(dicrun);//在线程池中启动线程
    	}
    	try {
    		cdl.await();//如果不使用await,那么这个方法便会执行结束
    	} catch (InterruptedException e) {
    		e.printStackTrace();}
    	System.out.println("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$");
    }
    //线程执行方法
    public void run(){
    	while(startpage < totalPage){
    		synchronized(DictEn.class){
    			try {
    				Thread.sleep(50);//进行睡眠，防止下面的线程不安全
    			} catch (InterruptedException e) {
    				e.printStackTrace();}
    			startpage++;
    			System.out.println(Thread.currentThread().getName()+"#页数#"+startpage);
    		}//在这里会有线程安全问题，故在上面的锁内进行睡眠
    		getLinkList(StringUtil.changeURL(threadurl, startpage));
    	}
    	cdl.countDown();
    	System.out.println(Thread.currentThread().getName()+"-----------------线程结束-----------------还有线程个数："+cdl.getCount());
    }
    /**
     * 得到每个列表页的链接
     */
    public void getLinkList(String url){
    	System.out.println(Thread.currentThread().getName()+"#当前列表页:"+url);
    	log.info("lookchem列表:"+url);
    	
    	String content=ProxyUtil.getProxyStr(url, "utf-8", "", "html");
    	List<String> list=StringUtil.getalltagslist(content, "<table width=\"100%\" border=\"0\" cellspacing=\"0\"", "</table>");
    	for(String s:list){
    		String cas = StringUtil.getFirststr(s, "<td width=\"120\" valign=\"middle\" >", "</td>", 1).trim();
    		//在这里先判断数据库中是否存在该数据
    		if(!StringUtil.isCasNo(cas) || !DictEnDao.checkCas(cas)){
    			System.out.println(cas+":Cas重复..............");
    			continue;
    		}
    		String u = StringUtil.getFirststr(s, "href=\"", "\"", 1);
    		getContent(u);//访问详细页面
    	}
    }