mirror of
https://github.com/NaiboWang/EasySpider.git
synced 2026-08-30 17:33:48 +08:00
执行阶段可自动读取设置好的Excel文件作为输入字段参数值
This commit is contained in:
naibo
@@ -126,7 +126,6 @@
|
||||
</tbody>
|
||||
</table>
|
||||
<label>{{"The 'Invoke Name' is shown at the main table of this page, such as 'urlList_0' is the invoke name of the open page operation.~“调用名称”在本页面的主表格中显示，例如“urlList_0”是“打开网页“操作的调用名称。" | lang}}</label>
|
||||
<label>{{`对于循环输入文字的参数（loopText）需要配置索引值的情况，即输入文字操作用了相对循环内的索引值，您可以在Excel文件中写同一个参数名写多列，程序将自动合并。 例如，如果想要设置'loopText_1'参数值为"A~B~C"，则Excel文件中可以在第一次循环时的值为'1'，第二次循环时的值为'2'，则在Excel文件中，'loopText'参数的第一列写'1'，第二列写'2'。~For the case that the index value of the loopText parameter needs to be configured, that is, the input text operation uses the index value relative to the loop, you can write multiple columns with the same parameter name in the Excel file, and the program will automatically merge them. For example, the value of the 'loopText' parameter is '1' in the first loop, and the value of the second loop is '2', then in the Excel file, the first column of the 'loopText' parameter is written '1', and the second column is written '2'.` | lang}}</label>
|
||||
<label>{{"You can just put part of the arguments in the Excel file, and the values of the rest of the arguments will be set to default. Example table for this task is:~您可以只在Excel文件中放入部分参数，其余参数值将被设置为默认值。一个针对此任务的表格示例为：" | lang}}</label>
|
||||
<table class="table table-bordered excel">
|
||||
<thead>
|
||||
@@ -156,7 +155,29 @@
|
||||
<tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<label v-if='lang == "zh"' style="width: 95%">对于循环输入文字的参数（loopText）需要配置索引值的情况，即输入文字操作用了相对循环内的索引值，您可以在Excel文件中写同一个参数名写多列，程序将自动合并。 例如，想要设置'loopText_1'参数值两行，分别为"A~B~C"和"D~E~F"，则Excel文件可以这样设置：</label>
|
||||
<label v-else style="width: 95%"> For parameters that need to configure the index value of the loop text (loopText), that is, the input text operation uses the index value relative to the loop, you can write multiple columns with the same parameter name in the Excel file, and the program will automatically merge. For example, if you want to set the parameter value of 'loopText_1' to two rows, which are "A~B~C" and "D~E~F", the Excel file can be set like this:</label>
|
||||
<table class="table table-bordered excel" style="text-align: center; font-size: 13px">
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col">inputText_1</th>
|
||||
<th scope="col">inputText_1</th>
|
||||
<th scope="col">inputText_1</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>A</td>
|
||||
<td>B</td>
|
||||
<td>C</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>D</td>
|
||||
<td>E</td>
|
||||
<td>F</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.modal-content -->
|
||||
@@ -264,6 +285,7 @@
|
||||
task: {},
|
||||
show: false, //是否渲染
|
||||
ID: "",
|
||||
lang: getUrlParam("lang"),
|
||||
tip: "The parameter values in the Excel file have been successfully imported into the corresponding field text box~Excel文件中的参数值已成功导入到对应字段文本框中",
|
||||
file:null,
|
||||
user_data_folder:"",
|
||||
@@ -343,7 +365,19 @@
|
||||
let inputParameters = app.$data.task.inputParameters;
|
||||
inputParameters.forEach(function (item, index) {
|
||||
if(Object.keys(response).includes(item.name)){
|
||||
item.value = response[item.name].join("\r\n");
|
||||
let temp = "";
|
||||
let tempArray = [];
|
||||
for (let i = 0; i < response[item.name].length; i++) {
|
||||
for(let key of Object.keys(response)){
|
||||
if(key.includes(item.name)){
|
||||
temp += response[key][i] == undefined? "": response[key][i] + "~";
|
||||
}
|
||||
}
|
||||
temp = temp.substring(0, temp.length - 1); //去掉最后一个~
|
||||
tempArray.push(temp);
|
||||
temp = "";
|
||||
}
|
||||
item.value = tempArray.join("\r\n");
|
||||
}
|
||||
});
|
||||
$("#tipCustom").slideDown(); //提示框
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Vendored
+1
-1
@@ -12,7 +12,7 @@
|
||||
"justMyCode": false,
|
||||
// "args": ["--id", "[7]", "--read_type", "remote", "--headless", "0"]
|
||||
// "args": ["--id", "[9]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"]
|
||||
"args": ["--id", "[71]", "--headless", "0", "--user_data", "1"]
|
||||
"args": ["--id", "[76]", "--headless", "0", "--user_data", "1"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -162,6 +162,7 @@ class BrowserThread(Thread):
|
||||
print("MySQL Mode")
|
||||
self.containJudge = service["containJudge"] # 是否含有判断语句
|
||||
self.outputParameters = {}
|
||||
self.service = service
|
||||
self.outputParametersTypes = []
|
||||
self.outputParametersRecord = [] # 字段是否被记录
|
||||
self.dataNotFoundKeys = {} # 记录没有找到数据的key
|
||||
@@ -186,6 +187,11 @@ class BrowserThread(Thread):
|
||||
self.OUTPUT[0].append(para["name"])
|
||||
self.urlId = 0 # 全局记录变量
|
||||
self.preprocess() # 预处理，优化提取数据流程
|
||||
try:
|
||||
self.inputExcel = service["inputExcel"] # 输入Excel
|
||||
except:
|
||||
self.inputExcel = ""
|
||||
self.readFromExcel() # 读取Excel获得参数值
|
||||
|
||||
# 检测如果没有复杂的操作，优化提取数据流程
|
||||
def preprocess(self):
|
||||
@@ -252,6 +258,61 @@ class BrowserThread(Thread):
|
||||
node["parameters"]["xpath"] = "" # 0.3.5及以下版本的EasySpider下的循环点击不支持相对XPath
|
||||
print("您的任务版本号为" + self.task_version + "，循环点击不支持相对XPath写法，已自动切换为纯循环的XPath")
|
||||
|
||||
def readFromExcel(self):
|
||||
if self.inputExcel == "":
|
||||
return 0
|
||||
try:
|
||||
workbook = load_workbook(self.inputExcel)
|
||||
except:
|
||||
print("读取Excel失败，将会使用默认参数执行任务，请检查文件路径是否正确：", os.path.abspath(self.inputExcel))
|
||||
print("Failed to read Excel, will execute the task with default parameters, please check if the file path is correct: ", os.path.abspath(self.inputExcel))
|
||||
time.sleep(5)
|
||||
return 0
|
||||
|
||||
sheet_name_list = workbook.sheetnames
|
||||
sheet = workbook[sheet_name_list[0]]
|
||||
data = []
|
||||
|
||||
for row in sheet.iter_rows(values_only=True):
|
||||
data.append(list(row))
|
||||
|
||||
result = list(zip(*data))
|
||||
result_dict = {}
|
||||
for row in result:
|
||||
key = row[0]
|
||||
values = [str(val) for val in row[1:] if val is not None]
|
||||
result_dict.setdefault(key, []).extend([values])
|
||||
|
||||
data = {}
|
||||
for key, arr in result_dict.items():
|
||||
result = []
|
||||
for cols in zip(*arr):
|
||||
result.append("~".join(cols))
|
||||
data[key] = result
|
||||
|
||||
try:
|
||||
if "urlList_0" in data.keys():
|
||||
self.links = data["urlList_0"]
|
||||
except:
|
||||
pass
|
||||
task = self.service
|
||||
for key, value in data.items():
|
||||
for i in range(len(task["inputParameters"])):
|
||||
if key == task["inputParameters"][i]["name"]:
|
||||
nodeId = int(task["inputParameters"][i]["nodeId"])
|
||||
node = task["graph"][nodeId]
|
||||
value = "\r\n".join(value)
|
||||
if node["option"] == 1:
|
||||
node["parameters"]["links"] = value
|
||||
elif node["option"] == 4:
|
||||
node["parameters"]["value"] = value
|
||||
elif node["option"] == 8 and node["parameters"]["loopType"] == 0:
|
||||
node["parameters"]["exitCount"] = int(value)
|
||||
elif node["option"] == 8:
|
||||
node["parameters"]["textList"] = value
|
||||
break
|
||||
print("已从Excel读取输入参数，覆盖了原有输入参数。")
|
||||
print("Alread read input parameters from Excel and overwrite the original input parameters.")
|
||||
|
||||
def run(self):
|
||||
# 挨个执行程序
|
||||
|
||||
Binary file not shown.
Reference in New Issue
Block a user