From df4326a48c8b313b427389f6c7aa1daba8ea49cb Mon Sep 17 00:00:00 2001 From: naibo Date: Fri, 22 Dec 2023 01:07:34 +0800 Subject: [PATCH] =?UTF-8?q?=E6=88=AA=E5=9B=BE=E5=A4=A7=E5=B0=8F=E6=81=A2?= =?UTF-8?q?=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ElectronJS/tasks/303.json | 1 + ExecuteStage/.vscode/launch.json | 2 +- ExecuteStage/easyspider_executestage.py | 13 +++++++++---- 3 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 ElectronJS/tasks/303.json diff --git a/ElectronJS/tasks/303.json b/ElectronJS/tasks/303.json new file mode 100644 index 0000000..95f680a --- /dev/null +++ b/ElectronJS/tasks/303.json @@ -0,0 +1 @@ +{"id":303,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"12/22/2023, 12:59:12 AM","update_time":"12/22/2023, 1:06:11 AM","version":"0.6.0","saveThreshold":10,"quitWaitTime":60,"environment":0,"maximizeWindow":1,"maxViewLength":15,"recordLog":1,"outputFormat":"xlsx","saveName":"current_time","dataWriteMode":1,"inputExcel":"","startFromExit":0,"pauseKey":"p","containJudge":false,"browser":"chrome","removeDuplicate":0,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"text","exampleValue":"https://www.jd.com"}],"outputParameters":[{"id":0,"name":"参数1_文本","desc":"","type":"text","recordASField":1,"exampleValue":"/手机/数码"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"cookies":""}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环采集数据","sequence":[3],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","code":"","waitTime":0,"exitCount":0,"exitElement":"//body","historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"skipCount":0,"allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]","//div[contains(., '/手机/数码')]","//DIV[@class='LeftSide_menu_item__SBMWC LeftSide_text_space__2UhbG ']","/html/body/div[last()-5]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]/div[last()-12]"]}},{"id":3,"index":3,"parentId":2,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"waitElement":"","waitElementTime":10,"waitElementIframeIndex":0,"clear":0,"newLine":1,"params":[{"nodeType":0,"contentType":8,"relative":true,"name":"参数1_文本","desc":"","extractType":0,"relativeXPath":"","allXPaths":"","exampleValues":[{"num":0,"value":"/手机/数码"}],"unique_index":"gdj5cmlqcevlqfg6f30","iframe":false,"default":"","paraType":"text","recordASField":1,"beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0,"splitLine":0}]}}]} \ No newline at end of file diff --git a/ExecuteStage/.vscode/launch.json b/ExecuteStage/.vscode/launch.json index f8e9fc2..e764396 100644 --- a/ExecuteStage/.vscode/launch.json +++ b/ExecuteStage/.vscode/launch.json @@ -12,7 +12,7 @@ "justMyCode": false, // "args": ["--ids", "[7]", "--read_type", "remote", "--headless", "0"] // "args": ["--ids", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"] - "args": ["--ids", "[22]", "--headless", "0", "--user_data", "0", "--keyboard", "0", + "args": ["--ids", "[45]", "--headless", "0", "--user_data", "0", "--keyboard", "0", "--read_type", "remote"] // "args": "--ids '[97]' --user_data 1 --server_address http://localhost:8074 --config_folder '/Users/naibo/Documents/EasySpider/ElectronJS/' --headless 0 --read_type remote --config_file_name config.json --saved_file_name" } diff --git a/ExecuteStage/easyspider_executestage.py b/ExecuteStage/easyspider_executestage.py index d62ced9..b65f540 100644 --- a/ExecuteStage/easyspider_executestage.py +++ b/ExecuteStage/easyspider_executestage.py @@ -1835,16 +1835,20 @@ class BrowserThread(Thread): content = self.browser.title elif p["contentType"] == 7: # 获取整个网页的高度和宽度 - height = self.browser.execute_script( - "return document.body.scrollHeight") - width = self.browser.execute_script( - "return document.body.scrollWidth") + size = self.browser.get_window_size() + width = size["width"] + height = size["height"] # 调整浏览器窗口的大小 self.browser.set_window_size(width, height) element.screenshot("Data/Task_" + str(self.id) + "/" + self.saveName + "/" + str(time.time()) + ".png") + # 截图完成后，将浏览器的窗口大小设置为原来的大小 + self.browser.set_window_size(width, height) elif p["contentType"] == 8: try: + size = self.browser.get_window_size() + width = size["width"] + height = size["height"] screenshot = element.screenshot_as_png screenshot_stream = io.BytesIO(screenshot) # 使用Pillow库打开截图，并转换为灰度图像 @@ -1858,6 +1862,7 @@ class BrowserThread(Thread): image_bytes = f.read() content = ocr.classification(image_bytes) os.remove(location) + self.browser.set_window_size(width, height) # 使用Tesseract OCR引擎识别图像中的文本 # content = pytesseract.image_to_string(image, lang='chi_sim+eng') except Exception as e: