Java crawls the web data to get all the link instances Shared in the web page

  • 2020-04-01 02:43:28
  • OfStack

rendering

< img border = 0 id = theimg onclick = window. The open this. (SRC) SRC = "/ / files.jb51.net/file_images/article/201312/20131231092541.jpg? 2013113192629 ">


import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class HtmlParser {
 
 String htmlUrl;
 
 ArrayList<String> hrefList = new ArrayList();
 
 String charSet;
 public HtmlParser(String htmlUrl) {
  //Constructor stub automatically generated by TODO
  this.htmlUrl = htmlUrl;
 }
 
 public ArrayList<String> getHrefList() throws IOException {
  parser();
  return hrefList;
 }
 
 private void parser() throws IOException {
  URL url = new URL(htmlUrl);
  HttpURLConnection connection = (HttpURLConnection) url.openConnection();
  connection.setDoOutput(true);
  String contenttype = connection.getContentType();
  charSet = getCharset(contenttype);
  InputStreamReader isr = new InputStreamReader(
    connection.getInputStream(), charSet);
  BufferedReader br = new BufferedReader(isr);
  String str = null, rs = null;
  while ((str = br.readLine()) != null) {
   rs = getHref(str);
   if (rs != null)
    hrefList.add(rs);
  }
 }
 
 private String getCharset(String str) {
  Pattern pattern = Pattern.compile("charset=.*");
  Matcher matcher = pattern.matcher(str);
  if (matcher.find())
   return matcher.group(0).split("charset=")[1];
  return null;
 }
 
 private String getHref(String str) {
  Pattern pattern = Pattern.compile("<a href=.*</a>");
  Matcher matcher = pattern.matcher(str);
  if (matcher.find())
   return matcher.group(0);
  return null;
 }
 public static void main(String[] arg) throws IOException {
  HtmlParser a = new HtmlParser("http://news.163.com/");
  ArrayList<String> hrefList = a.getHrefList();
  for (int i = 0; i < hrefList.size(); i++)
   System.out.println(hrefList.get(i));
 }
}


Related articles: