Using Java to collect jingdong mall administrative division data example
- 2020-04-01 03:09:53
- OfStack
package com.test.html;
import com.alibaba.fastjson.JSON;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
public class JD_Address
{
private static int timeout = 300000;
private static int count = 0;
public static void main(String[] args) throws IOException, InterruptedException
{
String province_url = "http://trade.jd.com/dynamic/consignee/getProvinces.action";
String city_url = "http://trade.jd.com/dynamic/consignee/getCitys.action?consigneeParam.provinceId=";
String county_url = "http://trade.jd.com/dynamic/consignee/getCountys.action?consigneeParam.cityId=";
String town_url = "http://trade.jd.com/dynamic/consignee/getTowns.action?consigneeParam.countyId=";
//getList(town_url + "4139");
List<Address> provinces = getList(province_url);
for (Address province : provinces)
{
List<Address> citys = getList(city_url + province.getId());
province.setChildren(citys);
for (Address city : citys)
{
List<Address> countys = getList(county_url + city.getId());
city.setChildren(countys);
for (Address county : countys)
{
List<Address> towns = getList(town_url + county.getId());
county.setChildren(towns);
}
}
}
System.out.println("=======");
FileUtils.writeStringToFile(new File(" Save the location /address2.js"), "var ds=" + JSON.toJSONString(provinces));
}
private static List<Address> getList(String url) throws IOException, InterruptedException
{
List<Address> list = new ArrayList<Address>();
Document doc = Jsoup.parse(new URL(url), timeout);
Elements elements = doc.select("option[value~=\d+]");
for (Element element : elements)
{
Address address = new Address();
address.setName(element.text().replace("*", StringUtils.EMPTY));
address.setId(element.attr("value"));
list.add(address);
//System.out.println(address.getName());
}
++count;
if (count % 100 == 0)
{
System.out.println(count);
}
if (count % 500 == 0)
{
Thread.sleep(5000);
}
return list;
}
}
class Address
{
private String name;
private String id;
private List<Address> children;
public String getName()
{
return name;
}
public void setName(String name)
{
this.name = name;
}
public List<Address> getChildren()
{
return children;
}
public void setChildren(List<Address> children)
{
this.children = children;
}
public String getId()
{
return id;
}
public void setId(String id)
{
this.id = id;
}
}
JavaScript code]
[
{"children": [
{"children": [
{"id": "2799", "name": " Within the third ring road "},
{"id": "2819", "name": " Between the third and fourth rings "},
{"id": "2839", "name": " Between the fourth ring and the fifth ring "},
{"id": "2840", "name": " Between the fifth and sixth rings "},
{"id": "4137", "name": " GuanZhuang "},
{"id": "4139", "name": " beiyuan "},
{"id": "4211", "name": " Dingfu village "}
], "id": "72", "name": " Chaoyang district "},
{"children": [
{"id": "2848", "name": " Within the third ring road "},
{"id": "2849", "name": " Between the third and fourth rings "},
{"id": "2850", "name": " Between the fourth ring and the fifth ring "},
{"id": "2851", "name": " Between the fifth and sixth rings "},
{"id": "2852", "name": " Six ring "},
{"id": "4134", "name": " XiSanQi "},
{"id": "4209", "name": " Flag col. "}
], "id": "2800", "name": " Haidian district, "},
{"children": [
{"id": "2827", "name": " Inner ring to second ring "},
{"id": "2853", "name": " Two to three rings "}
], "id": "2801", "name": " Xicheng district "},
{"children": [
{"id": "2821", "name": " From the inner ring to the third ring "}
], "id": "2802", "name": " The dongcheng district "},
{"children": [
{"id": "2829", "name": " One ring to two rings "},
{"id": "2842", "name": " Two to three rings "}
], "id": "2803", "name": " Chongwen district "},
{"children": [
{"id": "2828", "name": " From the inner ring to the third ring "}
], "id": "2804", "name": " Xuanwu district "},
{"children": [
{"id": "2832", "name": " Between the fourth ring and the fifth ring "},
{"id": "2854", "name": " Two to three rings "},
{"id": "2855", "name": " Between the third and fourth rings "},
{"id": "34544", "name": " Between the fifth and sixth rings "},
{"id": "34545", "name": " Six ring outside "}
], "id": "2805", "name": " Fengtai district "},
{"children": [
{"id": "2831", "name": " The fourth ring to the fifth ring "},
{"id": "4187", "name": " Shijingshan district "},
{"id": "4188", "name": " Eight science and technology parks "}
], "id": "2806", "name": " The shijingshan district "},
{"children": [
{"id": "6491", "name": " Within the city "},
{"id": "2843", "name": " On the outskirts of "}
], "id": "2807", "name": " vallely "},
{"children": [
{"id": "6492", "name": " Within the city "},
{"id": "2844", "name": " On the outskirts of "}
], "id": "2808", "name": " Fangshan district "},
{"children": [
{"id": "4175", "name": " Between the fifth and sixth rings "},
{"id": "37643", "name": " Outside the sixth ring road (at home) "},
{"id": "51150", "name": " Outside the sixth ring road (zhangjiawan town, taihu town, and hang county town) "},
{"id": "51151", "name": " Beyond the sixth ring road (songzhuang town) "},
{"id": "35698", "name": " Outside the sixth ring road (other areas) "}
], "id": "2809", "name": " Tongzhou district "},
{"children": [
{"id": "4194", "name": " Between the fourth and fifth rings "},
{"id": "6501", "name": " Between the fifth and sixth rings "},
{"id": "4205", "name": " Six ring "},
{"id": "51081", "name": " Yizhuang economic development zone "}
], "id": "2810", "name": " Daxing district "},
{"children": [
{"id": "51125", "name": " North town of stone runnel "},
{"id": "51126", "name": " North business town "},
{"id": "51127", "name": " BeiXiaoYing Town "},
{"id": "51128", "name": " Big sun village "},
{"id": "51129", "name": " Korea camp town "},
{"id": "51130", "name": " The street light "},
{"id": "51131", "name": " Housha valley area "},
{"id": "51132", "name": " Airport street "},
{"id": "51133", "name": " LiQiao Town "},
{"id": "51134", "name": " Li Sui town "},
{"id": "51135", "name": " Longwan town, "},
{"id": "51136", "name": " Ma Po region "},
{"id": "51137", "name": " Forest town, "},
{"id": "51138", "name": " Color of south town, "},
{"id": "51139", "name": " South France and trust "},
{"id": "51140", "name": " Niulan mountain area "},
{"id": "51141", "name": " Benevolence and regional "},
{"id": "51142", "name": " Victory street "},
{"id": "51143", "name": " The rock garden street "},
{"id": "51144", "name": " Shuangfeng street "},
{"id": "51145", "name": " Tianzhu region "},
{"id": "51146", "name": " Wangquan street "},
{"id": "51147", "name": " Yang Town region "},
{"id": "51148", "name": " Zhang town, "},
{"id": "51149", "name": " ZhaoQuanYing Town "}
], "id": "2812", "name": " Shunyi district "},
{"children": [
{"id": "6115", "name": " Within the city "},
{"id": "2847", "name": " On the outskirts of "}
], "id": "2814", "name": " Huairou district "},
{"children": [
{"id": "6667", "name": " The city "},
{"id": "2862", "name": " Outside the city "}
], "id": "2816", "name": " Miyun area "},
{"children": [
{"id": "4135", "name": " Within six ring "},
{"id": "4136", "name": " The city "},
{"id": "2906", "name": " Outside the city "}
], "id": "2901", "name": " Changping district "},
{"children": [
{"id": "6666", "name": " The city "},
{"id": "2954", "name": " Outside the city "}
], "id": "2953", "name": " Pinggu district "},
{"children": [
{"id": "6009", "name": " The spring professional , Beijing new high speed south , Kang Zhang Ludong , Beijing silver Lucy "},
{"id": "3066", "name": " The spring road , Beijing new high speed north , Kang Zhang Luxi , Beijing road east of silver "}
], "id": "3065", "name": " Yanqing county "}
], "id": "1", "name": " Beijing "},
{"children": [
{"children": [
{"id": "79", "name": " Within the inner ring "},
{"id": "80", "name": " Between the inner ring "},
{"id": "81", "name": " Between the outer ring "}
], "id": "78", "name": " Huangpu district "},
{"children": [], "id": "2811", "name": " Luwan district "},
{"children": [
{"id": "2863", "name": " Within the inner ring "},
{"id": "2865", "name": " Between the inner ring "}
], "id": "2813", "name": " Xuhui district "},
{"children": [
{"id": "2870", "name": " Within the inner ring "},
{"id": "2871", "name": " Between the inner ring "},
{"id": "2872", "name": " Between the outer ring "}
], "id": "2815", "name": " Changning district "},
{"children": [
{"id": "2873", "name": " Within the inner ring "},
{"id": "2874", "name": " Between the inner ring "},
{"id": "2875", "name": " Between the outer ring "}
], "id": "2817", "name": " Jing'an District "},
{"children": [], "id": "2820", "name": " zhabei "},
{"children": [
{"id": "2856", "name": " Between the inner ring "},
{"id": "2882", "name": " Within the inner ring "},
{"id": "2883", "name": " Between the outer ring "}
], "id": "2822", "name": " Hongkou district "},
{"children": [], "id": "2823", "name": " Yangpu district "},
{"children": [
{"id": "2889", "name": " Within the central "},
{"id": "2890", "name": " Between the outer ring "},
{"id": "2891", "name": " Outside the outer ring "}
], "id": "2824", "name": " Baoshan district "},
{"children": [
{"id": "2892", "name": " Within the central "},
{"id": "2893", "name": " Outside the outer ring "},
{"id": "2896", "name": " Between the outer ring "}
], "id": "2825", "name": " Minhang district "},
{"children": [
{"id": "2864", "name": " Outside the outer ring "}
], "id": "2826", "name": " Jiading district, "},
{"children": [
{"id": "2894", "name": " Within the central "},
{"id": "2895", "name": " Outside the outer ring "},
{"id": "2897", "name": " Between the outer ring "}
], "id": "2830", "name": " Pudong new area "},
{"children": [
{"id": "2869", "name": " Outside the outer ring "}
], "id": "2833", "name": " Qingpu, "},
{"children": [
{"id": "2866", "name": " Outside the outer ring "}
], "id": "2834", "name": " Songjiang district "},
{"children": [
{"id": "2868", "name": " Outside the outer ring "}
], "id": "2835", "name": " jinshan "},
{"children": [
{"id": "2903", "name": " ZhuQiao town "},
{"id": "2904", "name": " The town is sure to be "},
{"id": "2935", "name": " HuiNa town "},
{"id": "2937", "name": " The town of Cambridge "},
{"id": "2938", "name": " XuanQiao town "},
{"id": "2939", "name": " The academy town "},
{"id": "2940", "name": " Large town, "},
{"id": "2941", "name": " Zhoupu town "},
{"id": "2942", "name": " Luchao port town, "},
{"id": "2943", "name": " Mud towns "},
{"id": "2944", "name": " Six town "},
{"id": "2945", "name": " Hangtou town, "},
{"id": "2946", "name": " Wan Xiang town "},
{"id": "2947", "name": " The old port town, "},
{"id": "4159", "name": " Shen gang street "},
{"id": "4180", "name": " Lingang new city "}
], "id": "2836", "name": " Nanhui district "},
{"children": [
{"id": "2888", "name": " Outside the outer ring "}
], "id": "2837", "name": " The district "},
{"children": [
{"id": "2876", "name": " Within the inner ring "},
{"id": "2877", "name": " Between the inner ring "},
{"id": "2878", "name": " Between the outer ring "},
{"id": "3110", "name": " Taopu saemaul undong "}
], "id": "2841", "name": " Putuo district "},
{"children": [
{"id": "50779", "name": " Fort town "},
{"id": "50780", "name": " The town of temple "},
{"id": "50781", "name": " Yingyue ( "},
{"id": "50782", "name": " City bridge town, "},
{"id": "50783", "name": " Dongping town "},
{"id": "50784", "name": " Hong Kong west town, "},
{"id": "50785", "name": " Port along the town "},
{"id": "50786", "name": " The construction of the town "},
{"id": "50787", "name": " Green China town "},
{"id": "50788", "name": " The town of samsung "},
{"id": "50789", "name": " Shu new town "},
{"id": "50790", "name": " To the town, "},
{"id": "50791", "name": " The new town "},
{"id": "50792", "name": " XinHe Town "},
{"id": "50793", "name": " Zte's town "},
{"id": "50794", "name": " Changxing township "},
{"id": "50795", "name": " Horizontal sand township "},
{"id": "50796", "name": " XinCun Township "}
], "id": "2919", "name": " for "}
], "id": "2", "name": " Shanghai "},
{"children": [
{"children": [
{"id": "39620", "name": " All the "}
], "id": "51035", "name": " Dong li "},
{"children": [
{"id": "2984", "name": " All the "}
], "id": "51036", "name": " Heping district "},
{"children": [
{"id": "2987", "name": " All the "}
], "id": "51037", "name": " hebei "},
{"children": [
{"id": "3000", "name": " All the "}
], "id": "51038", "name": " Hedong district "},
{"children": [
{"id": "2985", "name": " All the "}
], "id": "51039", "name": " Hexi district "},
{"children": [
{"id": "2986", "name": " All the "}
], "id": "51040", "name": " Hongqiao district "},
{"children": [
{"id": "98", "name": " All the "}
], "id": "51041", "name": " jixian "},
{"children": [
{"id": "36157", "name": " All the "}
], "id": "51042", "name": " fruits "},
{"children": [
{"id": "2907", "name": " All the "}
], "id": "51043", "name": " Nankai district "},
{"children": [
{"id": "25708", "name": " All the "}
], "id": "51044", "name": " Tanggu district "},
{"children": [
{"id": "25712", "name": " From green , Connie , A famous martial art , Big temple town, , Outer haitai and outer ring "},
{"id": "25711", "name": " In other parts of "}
], "id": "51045", "name": " Xiqing district "},
{"children": [
{"id": "22846", "name": " Yangcun town, lower zhuzhuang "},
{"id": "22847", "name": " In other parts of "}
], "id": "51046", "name": " Wu qing "},
{"children": [
{"id": "25704", "name": " Yanshui gu town, haihe education park, haihe science and technology park "},
{"id": "36171", "name": " Shuanggang, xinzhuang "},
{"id": "36172", "name": " In other parts of the "}
], "id": "51047", "name": " jinnan "},
{"children": [
{"id": "23672", "name": " Hangu district street, hangu development zone "},
{"id": "23673", "name": " In other parts of "}
], "id": "51048", "name": " hangu "},
{"children": [], "id": "51049", "name": " Dagang district "},
{"children": [
{"id": "6646", "name": " In the outer ring "},
{"id": "36167", "name": " Outer ring outer double street town, hebei institute of technology new school, qudian industrial park "},
{"id": "36168", "name": " Other areas outside the outer ring "}
], "id": "51050", "name": " Beichen district "},
{"children": [
{"id": "22848", "name": " Chengguan town, majiadian development zone, tianbao industrial park "},
{"id": "22849", "name": " In other parts of "}
], "id": "51051", "name": " Baodi district "},
{"children": [
{"id": "23674", "name": " Lutai town, economic development zone, trade development zone "},
{"id": "23675", "name": " In other parts of "}
], "id": "51052", "name": " Ninghe county "}
], "id": "3", "name": " tianjin "},
{"children": [
{"children": [
{"id": "9775", "name": " Chenjiaba street "},
{"id": "9776", "name": " Bell drum tower street "},
{"id": "9777", "name": " Zhoujiaba street "},
{"id": "9778", "name": " Baianba street "},
{"id": "9779", "name": " Gao zhantang street "},
{"id": "9780", "name": " Shuang heu street "},
{"id": "9781", "name": " The dragon is the street "},
{"id": "9782", "name": " Arched street "},
{"id": "9783", "name": " Shahe street "},
{"id": "9784", "name": " The white street "},
{"id": "9785", "name": " Covering the streets "},
{"id": "9786", "name": " The town of clay "},
{"id": "9787", "name": " The Aries town "},
{"id": "9788", "name": " Big week town "},
{"id": "9789", "name": " The town of marbles "},
{"id": "9790", "name": " Water town "},
{"id": "9791", "name": " This town, "},
{"id": "9792", "name": " The peak, "},
{"id": "9793", "name": " The town of sorghum "},
{"id": "9794", "name": " HouShan Town "},
{"id": "9795", "name": " The routes, "},
{"id": "9796", "name": " Handsome boy, "},
{"id": "9797", "name": " The town of lonza "},
{"id": "9798", "name": " Liaocheng town "},
{"id": "9799", "name": " SunJia Town "},
{"id": "9800", "name": " Taian town "},
{"id": "9801", "name": " So the dragon town "},
{"id": "9802", "name": " Day town "},
{"id": "9803", "name": " WuLing Town "},
{"id": "9804", "name": " Xiangshui town "},
{"id": "9805", "name": " Xiao zhou town, "},
{"id": "9806", "name": " The town of SAN tin "},
{"id": "9807", "name": " New towns "},
{"id": "9808", "name": " XiongJia Town "},
{"id": "9809", "name": " The town of more than "},
{"id": "9810", "name": " Changling town "},
{"id": "9811", "name": " Long ping town "},
{"id": "9812", "name": " Long beach town "},
{"id": "9813", "name": " Cook's town "},
{"id": "9814", "name": " � cross the town "},
{"id": "9815", "name": " Mr. Town "},
{"id": "9816", "name": " The column as a "},
{"id": "9817", "name": " Yanshan township "},
{"id": "9818", "name": " Xikou town "},
{"id": "9819", "name": " General ZiXiang "},
{"id": "9820", "name": " Township to treasure "},
{"id": "9821", "name": " Iron feng township "},
{"id": "9822", "name": " Cortex phellodendri township "},
{"id": "9823", "name": " Nine pool township "},
{"id": "9824", "name": " Pear township "},
{"id": "9825", "name": " GuoCun Township "},
{"id": "9826", "name": " Constant or township "}
], "id": "113", "name": " Wanzhou district "},
{"children": [
{"id": "9893", "name": " Litchi street "},
{"id": "9894", "name": " Benevolence street in London "},
{"id": "9895", "name": " North street "},
{"id": "9896", "name": " Jiangdong street "},
{"id": "9897", "name": " Chung righteousness street "},
{"id": "9898", "name": " Li Du town "},
{"id": "9899", "name": " BaiTao Town "},
{"id": "9900", "name": " The town of yum! "},
{"id": "9901", "name": " A great town, "},
{"id": "9902", "name": " JiaoShi town "},
{"id": "9903", "name": " Rush town "},
{"id": "9904", "name": " Dragon bridge town, "},
{"id": "9905", "name": " LongTan Town "},
{"id": "9906", "name": " MaWu Town "},
{"id": "9907", "name": " South Tuo town "},
{"id": "9908", "name": " Qingyang town, "},
{"id": "9909", "name": " Qingxi town "},
{"id": "9910", "name": " Stone Tuo town "},
{"id": "9911", "name": " New wonderful town "},
{"id": "9912", "name": " Righteousness and town "},
{"id": "9913", "name": " Increase township "},
{"id": "9914", "name": " Jane creek town, "},
{"id": "9915", "name": " ZhenAn town "},
{"id": "9916", "name": " To Korea town "},
{"id": "9917", "name": " Township land slope "},
{"id": "9918", "name": " Wuling shan "},
{"id": "9919", "name": " 27 township "},
{"id": "9920", "name": " Hometown native place "},
{"id": "9921", "name": " The jungle township "},
{"id": "9922", "name": " Big wood township "},
{"id": "9923", "name": " Huimin township "},
{"id": "9924", "name": " Hotel township "},
{"id": "9925", "name": " Jubao village "},
{"id": "9926", "name": " Volume hole township "},
{"id": "9927", "name": " Two hui township "},
{"id": "9928", "name": " Rowan township "},
{"id": "9929", "name": " Ming hometown "},
{"id": "9930", "name": " Righteousness township "},
{"id": "9931", "name": " Fruit township "},
{"id": "9932", "name": " Stone and township "},
{"id": "9933", "name": " Shilong town, "},
{"id": "9934", "name": " Taihe town "},
{"id": "9935", "name": " tiantai "},
{"id": "9936", "name": " The pleasure of his country "},
{"id": "9937", "name": " XinCun Township "}
], "id": "114", "name": " fuling "},
{"children": [
{"id": "9938", "name": " LiangShan Town "},
{"id": "9939", "name": " Cypress home town "},
{"id": "9940", "name": " Green hills "},
{"id": "9941", "name": " The town of observatory "},
{"id": "9942", "name": " The town of ferro "},
{"id": "9943", "name": " GeXing Town "},
{"id": "9944", "name": " And Lin town "},
{"id": "9945", "name": " Tiger town "},
{"id": "9946", "name": " HuiLong Town "},
{"id": "9947", "name": " Gold belt town "},
{"id": "9948", "name": " JuKui town "},
{"id": "9949", "name": " The town of comity "},
{"id": "9950", "name": " Longmen town "},
{"id": "9951", "name": " Mingda town "},
{"id": "9952", "name": " PanLong Town "},
{"id": "9953", "name": " Screen kam town "},
{"id": "9954", "name": " RenXian town "},
{"id": "9955", "name": " ShiAn Town "},
{"id": "9956", "name": " WenHua Town "},
{"id": "9957", "name": " The new town "},
{"id": "9958", "name": " Yin ping town "},
{"id": "9959", "name": " Yuan Yi town "},
{"id": "9960", "name": " Yunlong town, "},
{"id": "9961", "name": " Zhushan town "},
{"id": "9962", "name": " Township ascen "},
{"id": "9963", "name": " Iron township "},
{"id": "9964", "name": " Purple according to township "},
{"id": "9965", "name": " Song of "},
{"id": "9966", "name": " Longsheng town "},
{"id": "9967", "name": " North township "},
{"id": "9968", "name": " ChengDong Township "},
{"id": "9969", "name": " After ping township "},
{"id": "39680", "name": " In the county "}
], "id": "115", "name": " Liangping county "},