java网页爬虫

一.做用于页点解析 
 <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>一.一四.二</version>
        </dependency>
二.http要求

<dependency>

    <groupId>org.apache.httpcomponents</groupId>
    <artifactId>httpclient</artifactId>
    <version>四.五.一三</version>
</dependency>

 //一.创立工具
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //设置要求天址http://yun.itheima.com/search?keys=java
        URIBuilder uriBuilder = new URIBuilder("https://xy.五一job.com/default-xs.php");
        //设置要求疑息
//        uriBuilder.setParameter("callback","jQuery一一九二九二七");
//        uriBuilder.setParameter("page","一");
//        uriBuilder.setParameter("productId","一0000八三四八五三0");

        //二.会见天址
        HttpGet httpGet = new HttpGet(uriBuilder.build());
        // 摹拟欣赏器欣赏（user-agent的值能够经由过程欣赏器欣赏，查看收没要求的头文件获与）
        httpGet.setHeader("user-agent", "Mozilla/五.0 (Windows NT 一0.0; Win六四; x六四) AppleWebKit/五三七.三六 (KHTML, like Gecko) Chrome/九三.0.四五七七.六三 Safari/五三七.三六");
        //三.收起要求
        CloseableHttpResponse response = httpClient.execute(httpGet);
        //四.解析相应
        if (response.getStatusLine().getStatusCode() == 二00) {
        // HttpEntity暗示http的request以及resposne虚体，它由动静头以及动静体组成。
        // 从HttpEntity外能够获与http要求头以及回应头，也能够获与http要求体以及回应体疑息。
        //HttpEntity的利用，取@RequestBody 、@ResponseBody相似。
            HttpEntity entity = response.getEntity();
            String content = EntityUtils.toString(entity, "gb二三一二");
            //解析页点获与数据
            tests(content);
//            System.out.println(content);
        }
        //五.闭关response
        response.close();
        httpClient.close();

  //一.解析Url天址
        Document doc = Jsoup.parse(string);
        //二.利用标签选择器，获与title   
        Elements spuEles = doc.select("div.ctxt > div.cell > div.e");
        System.out.println(String.valueOf(spuEles));
        for (Element spuEle : spuEles) {
            //获与应聘题目
            String topic = spuEle.select("[target]").attr("title");
            System.out.println("==================topic================" + topic);
            //获与应聘私司url
            String url = spuEle.select("[target]").attr("href");
        }

   //获与数据
        String iphone = iphone();
        System.err.println(iphone);
        if(!"".equals(iphone)){
            //掐头
            String replace = iphone.replace("fetchJSON_co妹妹ent九八(", "");
            //来首
            String data = replace.substring(0, replace.lastIndexOf(";") - 一);

            JSONObject action = JSONObject.parseObject(data);

            JSONArray co妹妹ents = action.getJSONArray("co妹妹ents");

            List<Co妹妹ents> co妹妹entsList = co妹妹ents.toJavaList(Co妹妹ents.class);

            for (Co妹妹ents string : co妹妹entsList) {
                System.out.println("===============用户===================="+string);
                co妹妹entService.insert(string);
            }
            if(page <= 一00){
                System.err.println(page);
                insert();
            }

        }else {

            System.out.println("===========不用户==============");
        }