java实现爬虫（爬取51job网的招聘信息）

如图&#Vff0c;爬去以下的相关信息。

F12阐明网站源码

大约的内容都正在id=resultList的标签下面。
每条职位信息都对应一个class=el的标签。
下一页的信息是一个class为bk下的a对应的href
真现代码&#Vff1a;
新建maZZZen名目&#Vff0c;导入依赖

<dependencies> <!-- hts://mZZZnrepositoryss/artifact/com.alibaba/fastjson --> <dependency> <groupId>com.alibaba</groupId> <artifactId>fastjson</artifactId> <ZZZersion>1.2.47</ZZZersion> </dependency> <!-- hts://mZZZnrepositoryss/artifact/ch.hsr/geohash --> <dependency> <groupId>ch.hsr</groupId> <artifactId>geohash</artifactId> <ZZZersion>1.3.0</ZZZersion> </dependency> <!-- hts://mZZZnrepositoryss/artifact/commons-htclient/commons-htclient --> <dependency> <groupId>commons-htclient</groupId> <artifactId>commons-htclient</artifactId> <ZZZersion>3.1</ZZZersion> </dependency> <!-- hts://mZZZnrepositoryss/artifact/org.jsoup/jsoup --> <dependency> <groupId>org.jsoup</groupId> <artifactId>jsoup</artifactId> <ZZZersion>1.8.3</ZZZersion> </dependency> </dependencies>

新建bean&#Vff0c;封拆每条岗亭的信息

public class JobBean { priZZZate String jobName; priZZZate String company; priZZZate String address; priZZZate String salary; priZZZate String date; public ZZZoid set(String jobName, String company, String address, String salary, String date) { this.jobName = jobName; thissspany = company; this.address = address; this.salary = salary; this.date = date; } public String getJobName() { return jobName; } public ZZZoid setJobName(String jobName) { this.jobName = jobName; } public String getCompany() { return company; } public ZZZoid setCompany(String company) { thissspany = company; } public String getAddress() { return address; } public ZZZoid setAddress(String address) { this.address = address; } public String getSalary() { return salary; } public ZZZoid setSalary(String salary) { this.salary = salary; } public String getDate() { return date; } public ZZZoid setDate(String date) { this.date = date; } public String toString() { return "jobName=" + jobName + ", company=" + company + ", address=" + address + ", salary=" + salary + ", date=" + date; } }

主真现办法

package cn.jiViang.main; import jaZZZa.net.URL; import jaZZZa.util.ArrayList; import jaZZZa.util.List; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import cn.jiViang.bean.JobBean; public class TestMain { priZZZate static int count = 0; public static ZZZoid main(String[] args) { String strUrl = "hts://search.51jobss/list/170200,000000,0000,00,9,99,%25E5%25A4%25A7%25E6%2595%25B0%25E6%258D%25AE,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&proZZZidesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare="; Document document = getDom(strUrl); List<JobBean> list = getPageInfo(document); System.out.println("---------------"+(++count)+"-------------"); for (JobBean jobBean : list) { System.out.println(jobBean); } getNeVtPageInfo(document); } public static ZZZoid getNeVtPageInfo(Document document){ Elements elements = document.select(".bk"); Element element = elements.get(1); String strUrl = element.select("a").attr("href"); if(strUrl==null){ return; } Document dom = getDom(strUrl); List<JobBean> list = getPageInfo(dom); System.out.println("---------------"+(++count)+"-------------"); for (JobBean jobBean : list) { System.out.println(jobBean); } try { Thread.sleep(400); } catch (InterruptedEVception e) { e.printStackTrace(); } getNeVtPageInfo(dom); } public static List<JobBean> getPageInfo(Document document){ List<JobBean> list = new ArrayList<JobBean>(); Elements elements = document.select("#resultList .el"); elements.remoZZZe(0); for (Element element : elements) { Elements elements2 = element.select("span"); JobBean jobBean = new JobBean(); jobBean.set(elements2.get(0).teVt(), elements2.get(1).teVt(), elements2.get(2).teVt(), elements2.get(3).teVt(), elements2.get(4).teVt()); list.add(jobBean); } return list; } public static Document getDom(String strUrl){ try { URL url = new URL(strUrl); Document document = Jsoup.parse(url, 4000); return document; } catch (EVception e) { e.printStackTrace(); } return null; } }

另一种办法&#Vff0c;咱们发现每一页的url只要对应的页数差异&#Vff0c;所以咱们可以先获与到总页数&#Vff0c;而后正在遍历url&#Vff0c;爬去每个url对应的网站。

代码真现&#Vff1a;

package cn.jiViang.main; import jaZZZa.net.URL; import jaZZZa.util.ArrayList; import jaZZZa.util.List; import jaZZZa.util.regeV.Matcher; import jaZZZa.util.regeV.Pattern; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import cn.jiViang.bean.JobBean; public class TestMain2 { public static ZZZoid main(String[] args) { String strUrl = "hts://search.51jobss/list/170200,000000,0000,00,9,99,%25E5%25A4%25A7%25E6%2595%25B0%25E6%258D%25AE,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&proZZZidesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare="; Document document = getDom(strUrl); int pageNum = getPageNum(document); for(int i=1;i<=pageNum;i++){ strUrl = "hts://search.51jobss/list/170200,000000,0000,00,9,99,%25E5%25A4%25A7%25E6%2595%25B0%25E6%258D%25AE,2,"+i+".html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&proZZZidesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare="; Document document2 = getDom(strUrl); List<JobBean> list = getPageInfo(document2); System.out.println("-------------"+i+"--------------"); for (JobBean jobBean : list) { System.out.println(jobBean); } } } public static int getPageNum(Document document) { Elements elements = document.select(".td"); String teVt = elements.get(0).teVt(); Pattern pattern = Patternsspile("[^0-9]"); Matcher matcher = pattern.matcher(teVt); String pageNum = matcher.replaceAll(""); return Integer.parseInt(pageNum); } public static List<JobBean> getPageInfo(Document document) { List<JobBean> list = new ArrayList<JobBean>(); Elements elements = document.select("#resultList .el"); elements.remoZZZe(0); for (Element element : elements) { Elements elements2 = element.select("span"); JobBean jobBean = new JobBean(); jobBean.set(elements2.get(0).teVt(), elements2.get(1).teVt(), elements2.get(2).teVt(), elements2.get(3).teVt(), elements2.get(4).teVt()); list.add(jobBean); } return list; } public static Document getDom(String strUrl) { try { URL url = new URL(strUrl); Document document = Jsoup.parse(url, 4000); return document; } catch (EVception e) { e.printStackTrace(); } return null; } }