From 6684650c44c7daa027ca17529b2eb26f36730a50 Mon Sep 17 00:00:00 2001 From: fanyong Date: Mon, 19 Oct 2020 10:01:38 +0800 Subject: [PATCH] =?UTF-8?q?2020-10-19=20=E4=BF=AE=E5=A4=8DRequest#continue?= =?UTF-8?q?Request=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../example/RequestInterceptionExample.java | 33 ++-- example/src/main/resources/testSelect.html | 13 -- .../ruiyun/jvppeteer/core/page/Request.java | 141 ++++++++++-------- 3 files changed, 96 insertions(+), 91 deletions(-) delete mode 100644 example/src/main/resources/testSelect.html diff --git a/example/src/main/java/com/ruiyun/example/RequestInterceptionExample.java b/example/src/main/java/com/ruiyun/example/RequestInterceptionExample.java index 533239d9..54ddf55e 100644 --- a/example/src/main/java/com/ruiyun/example/RequestInterceptionExample.java +++ b/example/src/main/java/com/ruiyun/example/RequestInterceptionExample.java @@ -25,25 +25,28 @@ public static void main(String[] args) throws Exception { arrayList.add("--disable-setuid-sandbox"); Browser browser = Puppeteer.launch(options); Page page = browser.newPage(); - page.emulate(Device.IPHONE_X); -// System.out.println("sdaad"); + PageNavigateOptions options1 = new PageNavigateOptions(); //如果不设置 domcontentloaded 算页面导航完成的话，那么goTo方法会超时，因为图片请求被拦截了，页面不会达到loaded阶段 options1.setWaitUntil(Collections.singletonList("domcontentloaded")); - page.goTo("https://www.baidu.com/?tn=98012088_10_dg&ch=3",options1); + + page.setRequestInterception(true); //拦截请求 -// page.onRequest(request -> { -// if ("image".equals(request.resourceType()) || "media".equals(request.resourceType())) { -// //遇到多媒体或者图片资源请求，拒绝，加载页面加载 -// System.out.println(request.url()+": "+request.resourceType()+": abort"); -// request.abort(); -// } else {//其他资源放行 -// request.continueRequest(); -// } -// }); -// page.setRequestInterception(true); - - page.goTo("https://item.taobao.com/item.htm?id=541605195654",options1); + page.onRequest(request -> { + if ("image".equals(request.resourceType())) { + //遇到多媒体或者图片资源请求，拒绝，加载页面加载 + System.out.println(request.url()+": "+request.resourceType()+": abort"); + request.abort(); + } else if(request.url().contains("wd=31")){ + request.continueRequest(); + } + else{//其他资源放行 + request.continueRequest(); + } + }); + + + page.goTo("https://www.baidu.com/s?cl=3&tn=baidutop10&fr=top1000&wd=31%E7%9C%81%E5%8C%BA%E5%B8%82%E6%96%B0%E5%A2%9E%E5%A2%83%E5%A4%96%E8%BE%93%E5%85%A513%E4%BE%8B&rsv_idx=2&rsv_dl=fyb_n_homepage&hisfilter=1",options1); } } diff --git a/example/src/main/resources/testSelect.html b/example/src/main/resources/testSelect.html deleted file mode 100644 index 601eaf89..00000000 --- a/example/src/main/resources/testSelect.html +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/src/main/java/com/ruiyun/jvppeteer/core/page/Request.java b/src/main/java/com/ruiyun/jvppeteer/core/page/Request.java index b015a7b6..41f1d721 100644 --- a/src/main/java/com/ruiyun/jvppeteer/core/page/Request.java +++ b/src/main/java/com/ruiyun/jvppeteer/core/page/Request.java @@ -197,24 +197,32 @@ public String failure() { * @param headers 请求头 * @return Future */ - public Future continueRequest(String url, String method, String postData, Map headers) { - return Helper.commonExecutor().submit(() -> { - // Request interception is not supported for data: urls. - if (url().startsWith("data:")) - return null; - ValidateUtil.assertArg(isAllowInterception(), "Request Interception is not enabled!"); - ValidateUtil.assertArg(!isInterceptionHandled(), "Request is already handled!"); - setInterceptionHandled(true); - Map params = new HashMap<>(); - params.put("requestId", interceptionId()); + public JsonNode continueRequest(String url, String method, String postData, Map headers) { + // Request interception is not supported for data: urls. + if (url().startsWith("data:")) + return null; + + ValidateUtil.assertArg(isAllowInterception(), "Request Interception is not enabled!"); + ValidateUtil.assertArg(!isInterceptionHandled(), "Request is already handled!"); + + setInterceptionHandled(true); + Map params = new HashMap<>(); + params.put("requestId", interceptionId()); + + if(StringUtil.isNotEmpty(url)) { params.put("url", url); + } + if(StringUtil.isNotEmpty(method)) { params.put("method", method); - params.put("postData", postData); - if (headers != null && headers.size() > 0) { - params.put("headers", headersArray(headers)); - } - return client.send("Fetch.continueRequest", params, true); - }); + } + if(StringUtil.isNotEmpty(postData)){ + params.put("postData", new String(Base64.getEncoder().encode(postData.getBytes()),StandardCharsets.UTF_8)); + } + + if (headers != null && headers.size() > 0) { + params.put("headers", headersArray(headers)); + } + return client.send("Fetch.continueRequest", params, true); } /** @@ -232,40 +240,43 @@ public void continueRequest() { * @param body 响应体 * @return Future */ - public Future respond(int status, Map headers, String contentType, String body) { - return Helper.commonExecutor().submit(() -> { - // Mocking responses for dataURL requests is not currently supported. - if (url().startsWith("data:")) - return null; - ValidateUtil.assertArg(allowInterception, "Request Interception is not enabled!"); - ValidateUtil.assertArg(!interceptionHandled, "Request is already handled!"); - setInterceptionHandled(true); - byte[] responseBody = null; - if (StringUtil.isNotEmpty(body)) { - responseBody = body.getBytes(StandardCharsets.UTF_8); - } - Map responseHeaders = new HashMap<>(); - if (headers != null && headers.size() > 0) { - for (Map.Entry entry : headers.entrySet()) - responseHeaders.put(entry.getKey().toLowerCase(), entry.getValue()); - } - if (StringUtil.isNotEmpty(contentType)){ - responseHeaders.put("content-type", contentType); - } + public JsonNode respond(int status, Map headers, String contentType, String body) { + // Mocking responses for dataURL requests is not currently supported. + if (url().startsWith("data:")) + return null; + + ValidateUtil.assertArg(allowInterception, "Request Interception is not enabled!"); + ValidateUtil.assertArg(!interceptionHandled, "Request is already handled!"); + + setInterceptionHandled(true); + byte[] responseBody = null; + if (StringUtil.isNotEmpty(body)) { + responseBody = body.getBytes(StandardCharsets.UTF_8); + } + Map responseHeaders = new HashMap<>(); - if (responseBody != null && !responseHeaders.containsKey("content-length")){ - responseHeaders.put("content-length", String.valueOf(responseBody.length)); - } - Map params = new HashMap<>(); - params.put("requestId", interceptionId); - params.put("responseCode", status); - params.put("responsePhrase", STATUS_TEXTS.get(status)); - params.put("responseHeaders", headersArray(responseHeaders)); - if (responseBody != null) { - params.put("body", Base64.getDecoder().decode(responseBody)); - } - return client.send("Fetch.fulfillRequest", params, true); - }); + if (headers != null && headers.size() > 0) { + for (Map.Entry entry : headers.entrySet()) + responseHeaders.put(entry.getKey().toLowerCase(), entry.getValue()); + } + + if (StringUtil.isNotEmpty(contentType)){ + responseHeaders.put("content-type", contentType); + } + + if (responseBody != null && !responseHeaders.containsKey("content-length")){ + responseHeaders.put("content-length", String.valueOf(responseBody.length)); + } + + Map params = new HashMap<>(); + params.put("requestId", interceptionId); + params.put("responseCode", status); + params.put("responsePhrase", STATUS_TEXTS.get(status)); + params.put("responseHeaders", headersArray(responseHeaders)); + if (responseBody != null) { + params.put("body", Base64.getDecoder().decode(responseBody)); + } + return client.send("Fetch.fulfillRequest", params, true); } /** @@ -274,20 +285,21 @@ public Future respond(int status, Map headers, String * @param errorCode errorCode错误码 * @return Future */ - public Future abort(ErrorCode errorCode) { - return Helper.commonExecutor().submit(() -> { - // Request interception is not supported for data: urls. - if (url().startsWith("data:")) - return null; - String errorReason = errorCode.getName(); - ValidateUtil.assertArg(allowInterception, "Request Interception is not enabled!"); - ValidateUtil.assertArg(!interceptionHandled, "Request is already handled!"); - setInterceptionHandled(true); - Map params = new HashMap<>(); - params.put("requestId", interceptionId); - params.put("errorReason", errorReason); - return client.send("Fetch.failRequest", params, true); - }); + public JsonNode abort(ErrorCode errorCode) { + // Request interception is not supported for data: urls. + if (url().startsWith("data:")) + return null; + + String errorReason = errorCode.getName(); + ValidateUtil.assertArg(allowInterception, "Request Interception is not enabled!"); + ValidateUtil.assertArg(!interceptionHandled, "Request is already handled!"); + + setInterceptionHandled(true); + Map params = new HashMap<>(); + params.put("requestId", interceptionId); + params.put("errorReason", errorReason); + return client.send("Fetch.failRequest", params, true); + } private List headersArray(Map headers) { @@ -301,6 +313,9 @@ private List headersArray(Map headers) { return result; } + /** + * 截断请求 + */ public void abort() { this.abort(ErrorCode.FAILED); }