mirror of
https://github.com/NaiboWang/EasySpider.git
synced 2026-08-31 01:40:53 +08:00
Optimize execute stage procedure
This commit is contained in:
naibo
Vendored
+1
-1
@@ -12,7 +12,7 @@
|
||||
"console": "integratedTerminal",
|
||||
"justMyCode": true,
|
||||
// "args": ["--id", "38", "--read_type", "local", "--headless", "1"]
|
||||
"args": ["--id", "15", "--headless", "0"]
|
||||
"args": ["--id", "14", "--headless", "0"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -641,6 +641,84 @@ def clickElement(para, loopElement=None, clickPath="", index=0):
|
||||
recordLog(str(e))
|
||||
rt.end()
|
||||
|
||||
def get_content(p, element):
|
||||
global saveName
|
||||
content = ""
|
||||
# 先处理特殊节点类型
|
||||
if p["nodeType"] == 2:
|
||||
if element.get_attribute("href") != None:
|
||||
content = element.get_attribute("href")
|
||||
else:
|
||||
content = ""
|
||||
elif p["nodeType"] == 3:
|
||||
if element.get_attribute("value") != None:
|
||||
content = element.get_attribute("value")
|
||||
else:
|
||||
content = ""
|
||||
elif p["nodeType"] == 4: # 图片
|
||||
if element.get_attribute("src") != None:
|
||||
content = element.get_attribute("src")
|
||||
else:
|
||||
content = ""
|
||||
try:
|
||||
downloadPic = p["downloadPic"]
|
||||
except:
|
||||
downloadPic = 0
|
||||
if downloadPic == 1:
|
||||
download_image(content, "Data/" +saveName + "/")
|
||||
else: # 普通节点
|
||||
if p["contentType"] == 0:
|
||||
content = element.text
|
||||
elif p["contentType"] == 1: # 只采集当期元素下的文本，不包括子元素
|
||||
command = 'var arr = [];\
|
||||
var content = arguments[0];\
|
||||
for(var i = 0, len = content.childNodes.length; i < len; i++) {\
|
||||
if(content.childNodes[i].nodeType === 3){ \
|
||||
arr.push(content.childNodes[i].nodeValue);\
|
||||
}\
|
||||
}\
|
||||
var str = arr.join(" "); \
|
||||
return str;'
|
||||
content = browser.execute_script(command, element).replace(
|
||||
"\n", "").replace("\\s+", " ")
|
||||
elif p["contentType"] == 2:
|
||||
content = element.get_attribute('innerHTML')
|
||||
elif p["contentType"] == 3:
|
||||
content = element.get_attribute('outerHTML')
|
||||
elif p["contentType"] == 4:
|
||||
# 获取元素的背景图片地址
|
||||
bg_url = element.value_of_css_property('background-image')
|
||||
# 清除背景图片地址中的多余字符
|
||||
bg_url = bg_url.replace('url("', '').replace('")', '')
|
||||
content = bg_url
|
||||
elif p["contentType"] == 5:
|
||||
content = browser.current_url
|
||||
elif p["contentType"] == 6:
|
||||
content = browser.title
|
||||
elif p["contentType"] == 7:
|
||||
# 获取整个网页的高度和宽度
|
||||
height = browser.execute_script("return document.body.scrollHeight");
|
||||
width = browser.execute_script("return document.body.scrollWidth");
|
||||
# 调整浏览器窗口的大小
|
||||
browser.set_window_size(width, height)
|
||||
element.screenshot("Data/" +saveName + "/"+ str(time.time()) + ".png")
|
||||
elif p["contentType"] == 8:
|
||||
try:
|
||||
screenshot = element.screenshot_as_png
|
||||
screenshot_stream = io.BytesIO(screenshot)
|
||||
# 使用Pillow库打开截图，并转换为灰度图像
|
||||
image = Image.open(screenshot_stream).convert('L')
|
||||
# 使用Tesseract OCR引擎识别图像中的文本
|
||||
text = pytesseract.image_to_string(image, lang='chi_sim+eng')
|
||||
content = text
|
||||
except Exception as e:
|
||||
content = "OCR Error"
|
||||
print("To use OCR, You need to install Tesseract-OCR and add it to the environment variable PATH: https://tesseract-ocr.github.io/tessdoc/Installation.html")
|
||||
print("要使用OCR识别功能，你需要安装Tesseract-OCR并将其添加到环境变量PATH中：https://blog.csdn.net/u010454030/article/details/80515501")
|
||||
elif p["contentType"] == 9:
|
||||
content = execute_code(2, p["JS"], p["JSWaitTime"], element)
|
||||
return content
|
||||
|
||||
|
||||
# 提取数据事件
|
||||
def getData(para, loopElement, isInLoop=True, parentPath="", index=0):
|
||||
@@ -693,79 +771,7 @@ def getData(para, loopElement, isInLoop=True, parentPath="", index=0):
|
||||
element = browser.find_element(By.XPATH, "//body")
|
||||
try:
|
||||
execute_code(2, p["beforeJS"], p["beforeJSWaitTime"], element) # 执行前置js
|
||||
# 先处理特殊节点类型
|
||||
if p["nodeType"] == 2:
|
||||
if element.get_attribute("href") != None:
|
||||
content = element.get_attribute("href")
|
||||
else:
|
||||
content = ""
|
||||
elif p["nodeType"] == 3:
|
||||
if element.get_attribute("value") != None:
|
||||
content = element.get_attribute("value")
|
||||
else:
|
||||
content = ""
|
||||
elif p["nodeType"] == 4: # 图片
|
||||
if element.get_attribute("src") != None:
|
||||
content = element.get_attribute("src")
|
||||
else:
|
||||
content = ""
|
||||
try:
|
||||
downloadPic = p["downloadPic"]
|
||||
except:
|
||||
downloadPic = 0
|
||||
if downloadPic == 1:
|
||||
download_image(content, "Data/" +saveName + "/")
|
||||
else: # 普通节点
|
||||
if p["contentType"] == 0:
|
||||
content = element.text
|
||||
elif p["contentType"] == 1: # 只采集当期元素下的文本，不包括子元素
|
||||
command = 'var arr = [];\
|
||||
var content = arguments[0];\
|
||||
for(var i = 0, len = content.childNodes.length; i < len; i++) {\
|
||||
if(content.childNodes[i].nodeType === 3){ \
|
||||
arr.push(content.childNodes[i].nodeValue);\
|
||||
}\
|
||||
}\
|
||||
var str = arr.join(" "); \
|
||||
return str;'
|
||||
content = browser.execute_script(command, element).replace(
|
||||
"\n", "").replace("\\s+", " ")
|
||||
elif p["contentType"] == 2:
|
||||
content = element.get_attribute('innerHTML')
|
||||
elif p["contentType"] == 3:
|
||||
content = element.get_attribute('outerHTML')
|
||||
elif p["contentType"] == 4:
|
||||
# 获取元素的背景图片地址
|
||||
bg_url = element.value_of_css_property('background-image')
|
||||
# 清除背景图片地址中的多余字符
|
||||
bg_url = bg_url.replace('url("', '').replace('")', '')
|
||||
content = bg_url
|
||||
elif p["contentType"] == 5:
|
||||
content = browser.current_url
|
||||
elif p["contentType"] == 6:
|
||||
content = browser.title
|
||||
elif p["contentType"] == 7:
|
||||
# 获取整个网页的高度和宽度
|
||||
height = browser.execute_script("return document.body.scrollHeight");
|
||||
width = browser.execute_script("return document.body.scrollWidth");
|
||||
# 调整浏览器窗口的大小
|
||||
browser.set_window_size(width, height)
|
||||
element.screenshot("Data/" +saveName + "/"+ str(time.time()) + ".png")
|
||||
elif p["contentType"] == 8:
|
||||
try:
|
||||
screenshot = element.screenshot_as_png
|
||||
screenshot_stream = io.BytesIO(screenshot)
|
||||
# 使用Pillow库打开截图，并转换为灰度图像
|
||||
image = Image.open(screenshot_stream).convert('L')
|
||||
# 使用Tesseract OCR引擎识别图像中的文本
|
||||
text = pytesseract.image_to_string(image, lang='chi_sim+eng')
|
||||
content = text
|
||||
except Exception as e:
|
||||
content = "OCR Error"
|
||||
print("To use OCR, You need to install Tesseract-OCR and add it to the environment variable PATH: https://tesseract-ocr.github.io/tessdoc/Installation.html")
|
||||
print("要使用OCR识别功能，你需要安装Tesseract-OCR并将其添加到环境变量PATH中：https://blog.csdn.net/u010454030/article/details/80515501")
|
||||
elif p["contentType"] == 9:
|
||||
content = execute_code(2, p["JS"], p["JSWaitTime"], element)
|
||||
content = get_content(p, element)
|
||||
except StaleElementReferenceException: # 发生找不到元素的异常后，等待几秒重新查找
|
||||
recordLog('StaleElementReferenceException：'+p["relativeXPath"])
|
||||
time.sleep(3)
|
||||
@@ -783,79 +789,7 @@ def getData(para, loopElement, isInLoop=True, parentPath="", index=0):
|
||||
element = browser.find_element(
|
||||
By.XPATH, p["relativeXPath"])
|
||||
recordLog('StaleElementReferenceException：relativeXPath')
|
||||
# 先处理特殊节点类型
|
||||
if p["nodeType"] == 2:
|
||||
if element.get_attribute("href") != None:
|
||||
content = element.get_attribute("href")
|
||||
else:
|
||||
content = ""
|
||||
elif p["nodeType"] == 3:
|
||||
if element.get_attribute("value") != None:
|
||||
content = element.get_attribute("value")
|
||||
else:
|
||||
content = ""
|
||||
elif p["nodeType"] == 4: # 图片
|
||||
if element.get_attribute("src") != None:
|
||||
content = element.get_attribute("src")
|
||||
else:
|
||||
content = ""
|
||||
try:
|
||||
downloadPic = p["downloadPic"]
|
||||
except:
|
||||
downloadPic = 0
|
||||
if downloadPic == 1:
|
||||
download_image(content, "Data/" +saveName + "/")
|
||||
else: # 普通节点
|
||||
if p["contentType"] == 0:
|
||||
content = element.text
|
||||
elif p["contentType"] == 1: # 只采集当期元素下的文本，不包括子元素
|
||||
command = 'var arr = [];\
|
||||
var content = arguments[0];\
|
||||
for(var i = 0, len = content.childNodes.length; i < len; i++) {\
|
||||
if(content.childNodes[i].nodeType === 3){ \
|
||||
arr.push(content.childNodes[i].nodeValue);\
|
||||
}\
|
||||
}\
|
||||
var str = arr.join(" "); \
|
||||
return str;'
|
||||
content = browser.execute_script(command, element).replace(
|
||||
"\n", "").replace("\\s+", " ")
|
||||
elif p["contentType"] == 2:
|
||||
content = element.get_attribute('innerHTML')
|
||||
elif p["contentType"] == 3:
|
||||
content = element.get_attribute('outerHTML')
|
||||
elif p["contentType"] == 4:
|
||||
# 获取元素的背景图片地址
|
||||
bg_url = element.value_of_css_property('background-image')
|
||||
# 清除背景图片地址中的多余字符
|
||||
bg_url = bg_url.replace('url("', '').replace('")', '')
|
||||
content = bg_url
|
||||
elif p["contentType"] == 5:
|
||||
content = browser.current_url
|
||||
elif p["contentType"] == 6:
|
||||
content = browser.title
|
||||
elif p["contentType"] == 7:
|
||||
# 获取整个网页的高度和宽度
|
||||
height = browser.execute_script("return document.body.scrollHeight");
|
||||
width = browser.execute_script("return document.body.scrollWidth");
|
||||
# 调整浏览器窗口的大小
|
||||
browser.set_window_size(width, height)
|
||||
element.screenshot("Data/" +saveName + "/"+ str(time.time()) + ".png")
|
||||
elif p["contentType"] == 8:
|
||||
try:
|
||||
screenshot = element.screenshot_as_png
|
||||
screenshot_stream = io.BytesIO(screenshot)
|
||||
# 使用Pillow库打开截图，并转换为灰度图像
|
||||
image = Image.open(screenshot_stream).convert('L')
|
||||
# 使用Tesseract OCR引擎识别图像中的文本
|
||||
text = pytesseract.image_to_string(image, lang='chi_sim+eng')
|
||||
content = text
|
||||
except Exception as e:
|
||||
content = "OCR Error"
|
||||
print("To use OCR, You need to install Tesseract-OCR and add it to the environment variable PATH: https://tesseract-ocr.github.io/tessdoc/Installation.html")
|
||||
print("要使用OCR识别功能，你需要安装Tesseract-OCR并将其添加到环境变量PATH中：https://blog.csdn.net/u010454030/article/details/80515501")
|
||||
elif p["contentType"] == 9:
|
||||
content = execute_code(2, p["JS"], p["JSWaitTime"], element)
|
||||
content = get_content(p, element)
|
||||
except StaleElementReferenceException:
|
||||
recordLog('StaleElementReferenceException：'+p["relativeXPath"])
|
||||
continue # 再出现类似问题直接跳过
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
中国大陆境内下载地址，速度会快一些（或者建议直接科学上网从Github Release Page下载，速度会提升很多）：
|
||||
|
||||
1. 云服务器：http://easyspider.naibo.wang
|
||||
|
||||
2. 浙江大学Onedrive分享：https://zjueducn-my.sharepoint.com/:f:/g/personal/wangnaibo_zju_edu_cn/EuAZBiI49p9Jipyz7fcV4T4B6lGxHNvcPqIlO3ohLOae1w?e=Zl5FBG
|
||||
Reference in New Issue
Block a user