java使用爬虫_如何用Java写一个爬虫

今天药忘吃喽~ 2022-11-06 13:56 334阅读 0赞

展开全部

以下是一个使用java实现的简单爬虫核心代码32313133353236313431303231363533e59b9ee7ad9431333363383966：

public void crawl() throws Throwable \{

while (continueCrawling()) \{

CrawlerUrl url = getNextUrl(); //获取待爬取队列中的下一个URL

if (url != null) \{

printCrawlInfo();

String content = getContent(url); //获取URL的文本信息

//聚焦爬虫只爬取与主题内容相关的网页，这里采用正则匹配简单处理

if (isContentRelevant(content, this.regexpSearchPattern)) \{

saveContent(url, content); //保存网页至本地

//获取网页内容中的链接，并放入待爬取队列中

Collection urlStrings = extractUrls(content, url);

addUrlsToUrlQueue(url, urlStrings);

\} else \{

System.out.println(url + " is not relevant ignoring ...");

//延时防止被对方屏蔽

Thread.sleep(this.delayBetweenUrls);

closeOutputStream();

private CrawlerUrl getNextUrl() throws Throwable \{

CrawlerUrl nextUrl = null;

while ((nextUrl == null) && (!urlQueue.isEmpty())) \{

CrawlerUrl crawlerUrl = this.urlQueue.remove();

//doWeHavePermissionToVisit：是否有权限访问该URL，友好的爬虫会根据网站提供的"Robot.txt"中配置的规则进行爬取

//isUrlAlreadyVisited：URL是否访问过，大型的搜索引擎往往采用BloomFilter进行排重，这里简单使用HashMap

//isDepthAcceptable：是否达到指定的深度上限。爬虫一般采取广度优先的方式。一些网站会构建爬虫陷阱(自动生成一些无效链接使爬虫陷入死循环)，采用深度限制加以避免

if (doWeHavePermissionToVisit(crawlerUrl)

&& (!isUrlAlreadyVisited(crawlerUrl))

&& isDepthAcceptable(crawlerUrl)) \{

nextUrl = crawlerUrl;

// System.out.println("Next url to be visited is " + nextUrl);

return nextUrl;

private String getContent(CrawlerUrl url) throws Throwable \{

//HttpClient4.1的调用与之前的方式不同

HttpClient client = new DefaultHttpClient();

HttpGet httpGet = new HttpGet(url.getUrlString());

StringBuffer strBuf = new StringBuffer();

HttpResponse response = client.execute(httpGet);

if (HttpStatus.SC\_OK == response.getStatusLine().getStatusCode()) \{

HttpEntity entity = response.getEntity();

if (entity != null) \{

BufferedReader reader = new BufferedReader(

new InputStreamReader(entity.getContent(), "UTF-8"));

String line = null;

if (entity.getContentLength() > 0) \{

strBuf = new StringBuffer((int) entity.getContentLength());

while ((line = reader.readLine()) != null) \{

strBuf.append(line);

if (entity != null) \{

nsumeContent();

//将url标记为已访问

markUrlAsVisited(url);

return strBuf.toString();

public static boolean isContentRelevant(String content,

Pattern regexpPattern) \{

boolean retValue = false;

if (content != null) \{

//是否符合正则表达式的条件

Matcher m = regexpPattern.matcher(content.toLowerCase());

retValue = m.find();

return retValue;

public List extractUrls(String text, CrawlerUrl crawlerUrl) \{

Map urlMap = new HashMap();

extractHttpUrls(urlMap, text);

extractRelativeUrls(urlMap, text, crawlerUrl);

return new ArrayList(urlMap.keySet());

private void extractHttpUrls(Map urlMap, String text) \{

Matcher m = (text);

while (m.find()) \{

String url = m.group();

String\[\] terms = url.split("a href=\\"");

for (String term : terms) \{

// System.out.println("Term = " + term);

if (term.startsWith("http")) \{

int index = term.indexOf("\\"");

if (index > 0) \{

term = term.substring(0, index);

urlMap.put(term, term);

System.out.println("Hyperlink: " + term);

private void extractRelativeUrls(Map urlMap, String text,

CrawlerUrl crawlerUrl) \{

Matcher m = relativeRegexp.matcher(text);

URL textURL = crawlerUrl.getURL();

String host = textURL.getHost();

while (m.find()) \{

String url = m.group();

String\[\] terms = url.split("a href=\\"");

for (String term : terms) \{

if (term.startsWith("/")) \{

int index = term.indexOf("\\"");

if (index > 0) \{

term = term.substring(0, index);

String s = //" + host + term;

urlMap.put(s, s);

System.out.println("Relative url: " + s);

public static void main(String\[\] args) \{

try \{

String url = "";

Queue urlQueue = new LinkedList();

String regexp = "java";

urlQueue.add(new CrawlerUrl(url, 0));

NaiveCrawler crawler = new NaiveCrawler(urlQueue, 100, 5, 1000L,

regexp);

// boolean allowCrawl = crawler.areWeAllowedToVisit(url);

// System.out.println("Allowed to crawl: " + url + " " +

// allowCrawl);

crawler.crawl();

\} catch (Throwable t) \{

System.out.println(t.toString());

t.printStackTrace();

![2Q==][2Q]

已赞过

已踩过<

你对这个回答的评价是？

收起

[2Q]:

java使用爬虫_如何用Java写一个爬虫

发表评论取消回复

还没有评论，来说两句吧...

相关阅读

相关用python写一个爬虫

相关用python写一个爬虫小程序

相关用python写一个爬虫代码

相关如何使用Java进行简单爬虫

相关 java爬虫(java爬虫框架)

相关用JAVA写一个超级简单的爬虫例子

相关 java爬虫写一个百度图片下载器

相关 java使用爬虫_如何用Java写一个爬虫

相关如何使用Python写一个爬虫软件

相关【nodeJs】用cheerio写一个爬虫吧