From e54e25172ac2c802366df4ab33d3689d26c94686 Mon Sep 17 00:00:00 2001 From: NaiboWang-Alienware Date: Tue, 4 Jul 2023 19:19:46 +0800 Subject: [PATCH] New videos --- .../EasySpider_Linux_x64/软件使用说明.txt | 2 +- .../软件使用说明.txt | 2 +- .../EasySpider_windows_x64/软件使用说明.txt | 2 +- .../EasySpider_windows_x86/软件使用说明.txt | 2 +- ElectronJS/tasks/115.json | 1234 +++++++++++++++-- ElectronJS/tasks/116.json | 1 + ElectronJS/tasks/117.json | 1 + ElectronJS/tasks/118.json | 379 +++++ ElectronJS/tasks/119.json | 1 + ElectronJS/tasks/120.json | 1 + ElectronJS/tasks/121.json | 1 + ElectronJS/tasks/122.json | 1 + ElectronJS/tasks/123.json | 1 + ExecuteStage/.vscode/launch.json | 2 +- ExecuteStage/easyspider_executestage.py | 97 +- ExecuteStage/myChrome.py | 75 + Readme.md | 4 +- 17 files changed, 1622 insertions(+), 184 deletions(-) create mode 100644 ElectronJS/tasks/116.json create mode 100644 ElectronJS/tasks/117.json create mode 100644 ElectronJS/tasks/118.json create mode 100644 ElectronJS/tasks/119.json create mode 100644 ElectronJS/tasks/120.json create mode 100644 ElectronJS/tasks/121.json create mode 100644 ElectronJS/tasks/122.json create mode 100644 ElectronJS/tasks/123.json create mode 100644 ExecuteStage/myChrome.py diff --git a/.temp_to_pub/EasySpider_Linux_x64/软件使用说明.txt b/.temp_to_pub/EasySpider_Linux_x64/软件使用说明.txt index 152bae7..47cc599 100644 --- a/.temp_to_pub/EasySpider_Linux_x64/软件使用说明.txt +++ b/.temp_to_pub/EasySpider_Linux_x64/软件使用说明.txt @@ -8,7 +8,7 @@ 支持Ubuntu 20.04, Debian, Deepin x64及以上版本。 -视频教程：https://www.bilibili.com/video/BV1Fk4y1L7xX/ +视频教程：https://www.bilibili.com/video/BV1th411A7ey/ 可以从其他机器导入任务，只需要把其他机器的tasks文件夹里的.json文件放入此目录的tasks文件夹里即可。同理执行号文件可以通过复制execution_instances文件夹中的.json文件来导入。注意，两个文件夹里的.json文件只支持命名为大于0的数字。 diff --git a/.temp_to_pub/EasySpider_MacOS_all_arch/软件使用说明.txt b/.temp_to_pub/EasySpider_MacOS_all_arch/软件使用说明.txt index 535d71b..80f987c 100644 --- a/.temp_to_pub/EasySpider_MacOS_all_arch/软件使用说明.txt +++ b/.temp_to_pub/EasySpider_MacOS_all_arch/软件使用说明.txt @@ -6,7 +6,7 @@ 10.x版本MacOS请下载v0.2.0版本使用。 -视频教程：https://www.bilibili.com/video/BV1Fk4y1L7xX/ +视频教程：https://www.bilibili.com/video/BV1th411A7ey/ 可以从其他机器导入任务，只需要把其他机器的tasks文件夹里的.json文件放入/Users/你的用户名/Library/Application Support/EasySpider/tasks文件夹里即可。同理执行号文件可以通过复制execution_instances文件夹中的.json文件来导入。注意，两个文件夹里的.json文件只支持命名为大于0的数字。 diff --git a/.temp_to_pub/EasySpider_windows_x64/软件使用说明.txt b/.temp_to_pub/EasySpider_windows_x64/软件使用说明.txt index 3c21b0c..bcad525 100644 --- a/.temp_to_pub/EasySpider_windows_x64/软件使用说明.txt +++ b/.temp_to_pub/EasySpider_windows_x64/软件使用说明.txt @@ -6,7 +6,7 @@ Windows 7此版本无直接可用版本（因为Chrome 109是最后一个支持Windows 7的Chrome版本），但v0.2.0的32位版本可用，且可以通过自行编译软件来运行，因此如想使用Windows 7采集数据，请下载v0.2.0的32位版本或自行下载代码并编译：https://github.com/NaiboWang/EasySpider/releases/tag/v0.2.0 -视频教程：https://www.bilibili.com/video/BV1Fk4y1L7xX/ +视频教程：https://www.bilibili.com/video/BV1th411A7ey/ 这个软件绝对不是特洛伊木马/病毒！如果被像Windows Defender这样的杀毒软件误认为是病毒，请进行恢复，或者打开“EasySpider.bat”来运行我们的软件。 diff --git a/.temp_to_pub/EasySpider_windows_x86/软件使用说明.txt b/.temp_to_pub/EasySpider_windows_x86/软件使用说明.txt index bce0841..e8da56f 100644 --- a/.temp_to_pub/EasySpider_windows_x86/软件使用说明.txt +++ b/.temp_to_pub/EasySpider_windows_x86/软件使用说明.txt @@ -6,7 +6,7 @@ Windows 7此版本无直接可用版本（因为Chrome 109是最后一个支持Windows 7的Chrome版本），但v0.2.0的32位版本可用，且可以通过自行编译软件来运行，因此如想使用Windows 7采集数据，请下载v0.2.0的32位版本或自行下载代码并编译：https://github.com/NaiboWang/EasySpider/releases/tag/v0.2.0 -视频教程：https://www.bilibili.com/video/BV1Fk4y1L7xX/ +视频教程：https://www.bilibili.com/video/BV1th411A7ey/ 这个软件绝对不是特洛伊木马/病毒！如果被像Windows Defender这样的杀毒软件误认为是病毒，请进行恢复，或者打开“EasySpider.bat”来运行我们的软件。 diff --git a/ElectronJS/tasks/115.json b/ElectronJS/tasks/115.json index cce2ec6..8ebd12b 100644 --- a/ElectronJS/tasks/115.json +++ b/ElectronJS/tasks/115.json @@ -3,7 +3,7 @@ "name": "网易云音乐", "url": "https://music.163.com/#/discover/toplist?id=71384707", "links": "https://music.163.com/#/discover/toplist?id=71384707", - "create_time": "7/4/2023, 5:06:27 AM", + "create_time": "7/4/2023, 7:14:50 AM", "version": "0.3.5", "saveThreshold": 10, "cloudflare": 0, @@ -20,308 +20,239 @@ "desc": "要采集的网址列表，多行以\\n分开", "type": "string", "exampleValue": "https://music.163.com/#/discover/toplist?id=71384707" - }, - { - "id": 1, - "name": "loopTimes_循环_1", - "nodeId": 3, - "nodeName": "循环", - "desc": "循环循环执行的次数（0代表无限循环）", - "type": "int", - "exampleValue": 0, - "value": 0 - }, - { - "id": 2, - "name": "inputText_2", - "nodeName": "输入文字", - "nodeId": 7, - "desc": "要输入的文本，如京东搜索框输入：电脑", - "type": "string", - "exampleValue": "123", - "value": "123" } ], "outputParameters": [ { "id": 0, - "name": "参数73_文本", + "name": "参数1_文本", "desc": "", "type": "string", - "exampleValue": "1" + "exampleValue": "Nocturne No. 2 in E Flat Major, Op. 9, No. 2" }, { "id": 1, - "name": "参数74_文本", + "name": "参数2_文本", "desc": "", "type": "string", - "exampleValue": "1" + "exampleValue": "作曲 : Frédéric François Chopin纯音乐，请欣赏" }, { "id": 2, - "name": "参数75_链接文本", + "name": "参数3_链接文本", "desc": "", "type": "string", "exampleValue": "" }, { "id": 3, - "name": "参数76_链接地址", + "name": "参数4_链接地址", "desc": "", "type": "string", - "exampleValue": "/song?id=1060914" + "exampleValue": "/user/home?id=6622403" }, { "id": 4, - "name": "参数77_图片地址", + "name": "参数5_图片地址", "desc": "", "type": "string", - "exampleValue": "http://p1.music.126.net/Dvhon4mH7qimTtE7q3omTw==/1419469524716144.jpg?param=50y50&quality=100" + "exampleValue": "http://p2.music.126.net/XM9AGb0vN4OSKK4WeZM7aA==/109951165633941106.jpg?param=50y50" }, { "id": 5, - "name": "参数78_链接文本", + "name": "参数6_文本", "desc": "", "type": "string", - "exampleValue": "NocturEJpne No. 2 in E Flat Major, Op. 9, No. 2" + "exampleValue": "：建议这种纯音乐，小编在歌词中做简介" }, { "id": 6, - "name": "参数79_链接地址", + "name": "参数7_链接文本", "desc": "", "type": "string", - "exampleValue": "/song?id=1060914" + "exampleValue": "新鑫曼巴" }, { "id": 7, - "name": "参数80_文本", + "name": "参数8_链接地址", "desc": "", "type": "string", - "exampleValue": "NocturneNo.2inEFlatMajor,Op.9,No.2" + "exampleValue": "/user/home?id=6622403" }, { "id": 8, - "name": "参数81_文本", + "name": "参数9_图片地址", "desc": "", "type": "string", - "exampleValue": "EJp" + "exampleValue": "http://p5.music.126.net/obj/wovDmcKXw6PCn2_CmsOk/17941162599/305d/0dee/968a/22f0de72becf0485ecac1d6a5898825a.png" }, { "id": 9, - "name": "参数82_文本", + "name": "参数10_文本", "desc": "", "type": "string", - "exampleValue": "-(降E大调第2号夜曲，作品9)" + "exampleValue": "2016年4月18日" }, { "id": 10, - "name": "参数83_文本", + "name": "参数11_链接文本", "desc": "", "type": "string", - "exampleValue": "函潖奪" + "exampleValue": "删除" }, { "id": 11, - "name": "参数84_文本", + "name": "参数12_链接地址", "desc": "", "type": "string", - "exampleValue": "04:26" + "exampleValue": "javascript:void(0)" }, { "id": 12, - "name": "参数85_链接文本", + "name": "参数13_文本", "desc": "", "type": "string", - "exampleValue": "" + "exampleValue": "|" }, { "id": 13, - "name": "参数86_链接地址", + "name": "参数14_链接文本", "desc": "", "type": "string", - "exampleValue": "javascript:;" + "exampleValue": " (29844)" }, { "id": 14, - "name": "参数87_文本", + "name": "参数15_链接地址", "desc": "", "type": "string", - "exampleValue": "分享" + "exampleValue": "javascript:void(0)" }, { "id": 15, - "name": "参数88_链接文本", + "name": "参数16_文本", "desc": "", "type": "string", - "exampleValue": "Arthur Rubiwf8knstein" + "exampleValue": "|" }, { "id": 16, - "name": "参数89_链接地址", + "name": "参数17_链接文本", "desc": "", "type": "string", - "exampleValue": "/artist?id=12541250" + "exampleValue": "回复" }, { "id": 17, - "name": "参数90_文本", + "name": "参数18_链接地址", "desc": "", "type": "string", - "exampleValue": "wf8k" + "exampleValue": "javascript:void(0)" }, { "id": 18, - "name": "参数91_文本", + "name": "参数19_文本", "desc": "", "type": "string", - "exampleValue": "NobleMusicProject" + "exampleValue": "：有次我在琴房练琴，练的是一首通俗歌曲改编的钢琴曲。正当我沉醉于自己的琴声时，隔壁忽然传来这首夜曲，那一刻我突然觉得自己练的曲子简直是没法听，于是后半段就没练，坐在琴房听隔壁弹了一个小时夜曲。" }, { "id": 19, - "name": "参数92_文本", + "name": "参数20_文本", "desc": "", "type": "string", - "exampleValue": "fXuJ" + "exampleValue": "◆" }, { "id": 20, - "name": "参数93_文本", + "name": "参数21_文本", "desc": "", "type": "string", - "exampleValue": "/" + "exampleValue": "◆" }, { "id": 21, - "name": "参数94_链接文本", + "name": "参数22_文本", "desc": "", "type": "string", - "exampleValue": "Chris USnelling" + "exampleValue": "2016年4月24日" }, { "id": 22, - "name": "参数95_链接地址", + "name": "参数23_链接文本", "desc": "", "type": "string", - "exampleValue": "/artist?id=13065095" + "exampleValue": "删除" }, { "id": 23, - "name": "参数96_文本", + "name": "参数24_链接地址", "desc": "", "type": "string", - "exampleValue": "U" + "exampleValue": "javascript:void(0)" }, { "id": 24, - "name": "参数97_链接文本", + "name": "参数25_文本", "desc": "", "type": "string", - "exampleValue": "Eberhard aFJFinke" + "exampleValue": "|" }, { "id": 25, - "name": "参数98_链接地址", + "name": "参数26_链接文本", "desc": "", "type": "string", - "exampleValue": "/artist?id=265005" + "exampleValue": " (7809)" }, { "id": 26, - "name": "参数99_文本", + "name": "参数27_链接地址", "desc": "", "type": "string", - "exampleValue": "aFJ" + "exampleValue": "javascript:void(0)" }, { "id": 27, - "name": "参数100_链接文本", + "name": "参数28_文本", "desc": "", "type": "string", - "exampleValue": "Mstip4Uslav Rostropovich" + "exampleValue": "|" }, { "id": 28, - "name": "参数101_链接地址", + "name": "参数29_链接文本", "desc": "", "type": "string", - "exampleValue": "/artist?id=155815" + "exampleValue": "回复" }, { "id": 29, - "name": "参数102_文本", + "name": "参数30_链接地址", "desc": "", "type": "string", - "exampleValue": "p4U" + "exampleValue": "javascript:void(0)" }, { "id": 30, - "name": "参数103_文本", + "name": "参数31_图片地址", "desc": "", "type": "string", - "exampleValue": "MV" + "exampleValue": "http://s1.music.126.net/style/web2/emt/emoji_13.png" }, { "id": 31, - "name": "参数104_链接文本", + "name": "参数32_图片地址", "desc": "", "type": "string", - "exampleValue": "AdIUBrpad Gérecz" + "exampleValue": "https://p5.music.126.net/obj/wonDlsKUwrLClGjCm8Kx/21892874340/50e6/e81d/5292/2d693cbb98954e28d4d9fdb11be99a8b.png" }, { "id": 32, - "name": "参数105_链接地址", + "name": "参数33_链接文本", "desc": "", "type": "string", - "exampleValue": "/artist?id=213459" - }, - { - "id": 33, - "name": "参数106_文本", - "desc": "", - "type": "string", - "exampleValue": "dIUB" - }, - { - "id": 34, - "name": "参数107_链接文本", - "desc": "", - "type": "string", - "exampleValue": "New Philharmoniali Orchestra" - }, - { - "id": 35, - "name": "参数108_链接地址", - "desc": "", - "type": "string", - "exampleValue": "/artist?id=129486" - }, - { - "id": 36, - "name": "参数109_文本", - "desc": "", - "type": "string", - "exampleValue": "li" - }, - { - "id": 37, - "name": "参数110_链接文本", - "desc": "", - "type": "string", - "exampleValue": "Edo de WafczUart" - }, - { - "id": 38, - "name": "参数111_链接地址", - "desc": "", - "type": "string", - "exampleValue": "/artist?id=144817" - }, - { - "id": 39, - "name": "参数112_文本", - "desc": "", - "type": "string", - "exampleValue": "fczU" + "exampleValue": "网易云音乐" } ], "graph": [ @@ -332,7 +263,7 @@ "type": -1, "option": 0, "title": "root", - "sequence": [1, 2, 3], + "sequence": [1, 10, 11, 12], "parameters": { "history": 1, "tabIndex": 0, @@ -363,14 +294,14 @@ "afterJSWaitTime": 0, "url": "https://music.163.com/#/discover/toplist?id=71384707", "links": "https://music.163.com/#/discover/toplist?id=71384707", - "maxWaitTime": 10, + "maxWaitTime": 100, "scrollType": 0, "scrollCount": 1, "scrollWaitTime": 1 } }, { - "id": 2, + "id": -1, "index": 2, "parentId": 0, "type": 0, @@ -406,7 +337,7 @@ } }, { - "id": 3, + "id": -1, "index": 3, "parentId": 0, "type": 1, @@ -449,7 +380,7 @@ } }, { - "id": 5, + "id": -1, "index": 4, "parentId": 3, "type": 0, @@ -486,7 +417,7 @@ } }, { - "id": 4, + "id": -1, "index": 5, "parentId": 3, "type": 1, @@ -524,7 +455,7 @@ } }, { - "id": 6, + "id": -1, "index": 6, "parentId": 4, "type": 0, @@ -556,7 +487,7 @@ } }, { - "id": 7, + "id": -1, "index": 7, "parentId": 4, "type": 0, @@ -588,7 +519,7 @@ } }, { - "id": 8, + "id": -1, "index": 8, "parentId": 4, "type": 1, @@ -632,7 +563,7 @@ } }, { - "id": 9, + "id": -1, "index": 9, "parentId": 8, "type": 0, @@ -1653,6 +1584,1019 @@ } ] } + }, + { + "id": 2, + "index": 10, + "parentId": 0, + "type": 0, + "option": 2, + "title": "点击元素", + "sequence": [], + "isInLoop": false, + "position": 1, + "parameters": { + "history": 3, + "tabIndex": -1, + "useLoop": false, + "xpath": "//*[@id=\"10609141688426033070\"]/td[2]/div[1]/div[1]/div[1]/span[1]/a[1]/b[1]", + "iframe": true, + "wait": 2, + "waitType": 0, + "beforeJS": "", + "beforeJSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "scrollType": 0, + "scrollCount": 1, + "scrollWaitTime": 1, + "clickWay": 0, + "maxWaitTime": 10, + "paras": [], + "allXPaths": [ + "/html/body/div[3]/div[2]/div[1]/div[2]/div[2]/div[1]/div[1]/table[1]/tbody[1]/tr[1]/td[2]/div[1]/div[1]/div[1]/span[1]/a[1]/b[1]", + "//b[contains(., 'Nocturne N')]", + "/html/body/div[last()-10]/div/div/div/div[last()-1]/div/div[last()-1]/table/tbody/tr[last()-99]/td[last()-2]/div/div/div/span/a/b" + ] + } + }, + { + "id": 3, + "index": 11, + "parentId": 0, + "type": 0, + "option": 3, + "title": "提取数据", + "sequence": [], + "isInLoop": false, + "position": 2, + "parameters": { + "history": 4, + "tabIndex": -1, + "useLoop": false, + "xpath": "", + "iframe": true, + "wait": 0, + "waitType": 0, + "beforeJS": "", + "beforeJSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "paras": [ + { + "nodeType": 0, + "contentType": 0, + "relative": false, + "name": "参数1_文本", + "desc": "", + "relativeXPath": "/html/body/div[3]/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div[1]/div[1]/em[1]", + "allXPaths": [ + "/html/body/div[3]/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div[1]/div[1]/em[1]", + "//em[contains(., 'Nocturne N')]", + "//EM[@class='f-ff2']", + "/html/body/div[last()-8]/div[last()-1]/div/div/div[last()-1]/div[last()-1]/div/div[last()-2]/div/em" + ], + "exampleValues": [ + { + "num": 0, + "value": "Nocturne No. 2 in E Flat Major, Op. 9, No. 2" + } + ], + "unique_index": "8l265os1r85ljnha0rx", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 0, + "contentType": 0, + "relative": false, + "name": "参数2_文本", + "desc": "", + "relativeXPath": "/html/body/div[3]/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div[3]", + "allXPaths": [ + "/html/body/div[3]/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div[3]", + "//div[contains(., '作曲 : Frédé')]", + "id(\"lyric-content\")", + "//DIV[@class='bd bd-open f-brk f-ib']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div[last()-1]/div[last()-1]/div/div" + ], + "exampleValues": [ + { + "num": 0, + "value": "作曲 : Frédéric François Chopin纯音乐，请欣赏" + } + ], + "unique_index": "wo55mnquaokljnha0rx", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + } + ] + } + }, + { + "id": 4, + "index": 12, + "parentId": 0, + "type": 1, + "option": 8, + "title": "循环", + "sequence": [13, 14], + "isInLoop": false, + "position": 3, + "parameters": { + "history": 4, + "tabIndex": -1, + "useLoop": false, + "xpath": "/html/body/div[3]/div[1]/div[1]/div[1]/div[2]/div[1]/div[2]/div[2]/div", + "iframe": true, + "wait": 0, + "waitType": 0, + "beforeJS": "", + "beforeJSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "scrollType": 0, + "scrollCount": 1, + "scrollWaitTime": 1, + "loopType": 1, + "pathList": "", + "textList": "", + "code": "", + "waitTime": 0, + "exitCount": 0, + "historyWait": 2, + "breakMode": 0, + "breakCode": "", + "breakCodeWaitTime": 0, + "allXPaths": [ + "/html/body/div[3]/div[1]/div[1]/div[1]/div[2]/div[1]/div[2]/div[2]/div[1]", + "//div[contains(., '新鑫曼巴：建议这种纯')]", + "id(\"1430545231688426048755\")", + "//DIV[@class='itm']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]" + ] + } + }, + { + "id": 5, + "index": 13, + "parentId": 4, + "type": 0, + "option": 3, + "title": "提取数据", + "sequence": [], + "isInLoop": true, + "position": 0, + "parameters": { + "history": 4, + "tabIndex": -1, + "useLoop": false, + "xpath": "", + "iframe": true, + "wait": 0, + "waitType": 0, + "beforeJS": "", + "beforeJSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "paras": [ + { + "nodeType": 1, + "contentType": 0, + "relative": true, + "name": "参数3_链接文本", + "desc": "", + "relativeXPath": "/div[1]/a[1]", + "allXPaths": [ + "/div[1]/a[1]", + "//a[contains(., '')]", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div[last()-1]/a" + ], + "exampleValues": [{ "num": 0, "value": "" }], + "unique_index": "/div[1]/a[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 2, + "contentType": 0, + "relative": true, + "name": "参数4_链接地址", + "desc": "", + "relativeXPath": "/div[1]/a[1]", + "allXPaths": [ + "/div[1]/a[1]", + "//a[contains(., '')]", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div[last()-1]/a" + ], + "exampleValues": [{ "num": 0, "value": "/user/home?id=6622403" }], + "unique_index": "/div[1]/a[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 4, + "contentType": 1, + "relative": true, + "name": "参数5_图片地址", + "desc": "", + "relativeXPath": "/div[1]/a[1]/img[1]", + "allXPaths": [ + "/div[1]/a[1]/img[1]", + "//img[contains(., '')]", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div[last()-1]/a/img" + ], + "exampleValues": [ + { + "num": 0, + "value": "http://p2.music.126.net/XM9AGb0vN4OSKK4WeZM7aA==/109951165633941106.jpg?param=50y50" + } + ], + "unique_index": "/div[1]/a[1]/img[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 0, + "contentType": 1, + "relative": true, + "name": "参数6_文本", + "desc": "", + "relativeXPath": "/div[2]/div[1]/div[1]", + "allXPaths": [ + "/div[2]/div[1]/div[1]", + "//div[contains(., '新鑫曼巴：建议这种纯')]", + "//DIV[@class='cnt f-brk']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div/div[last()-1]/div" + ], + "exampleValues": [ + { "num": 0, "value": "：建议这种纯音乐，小编在歌词中做简介" } + ], + "unique_index": "/div[2]/div[1]/div[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 1, + "contentType": 0, + "relative": true, + "name": "参数7_链接文本", + "desc": "", + "relativeXPath": "/div[2]/div[1]/div[1]/a[1]", + "allXPaths": [ + "/div[2]/div[1]/div[1]/a[1]", + "//a[contains(., '新鑫曼巴')]", + "//A[@class='s-fc7']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div/div[last()-1]/div/a" + ], + "exampleValues": [{ "num": 0, "value": "新鑫曼巴" }], + "unique_index": "/div[2]/div[1]/div[1]/a[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 2, + "contentType": 0, + "relative": true, + "name": "参数8_链接地址", + "desc": "", + "relativeXPath": "/div[2]/div[1]/div[1]/a[1]", + "allXPaths": [ + "/div[2]/div[1]/div[1]/a[1]", + "//a[contains(., '新鑫曼巴')]", + "//A[@class='s-fc7']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div/div[last()-1]/div/a" + ], + "exampleValues": [{ "num": 0, "value": "/user/home?id=6622403" }], + "unique_index": "/div[2]/div[1]/div[1]/a[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 4, + "contentType": 1, + "relative": true, + "name": "参数9_图片地址", + "desc": "", + "relativeXPath": "/div[2]/div[1]/div[1]/img[1]", + "allXPaths": [ + "/div[2]/div[1]/div[1]/img[1]", + "//img[contains(., '')]", + "//IMG[@class='brand-tag brand-vip']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div/div[last()-1]/div/img" + ], + "exampleValues": [ + { + "num": 0, + "value": "http://p5.music.126.net/obj/wovDmcKXw6PCn2_CmsOk/17941162599/305d/0dee/968a/22f0de72becf0485ecac1d6a5898825a.png" + } + ], + "unique_index": "/div[2]/div[1]/div[1]/img[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 0, + "contentType": 1, + "relative": true, + "name": "参数10_文本", + "desc": "", + "relativeXPath": "/div[2]/div[2]/div[1]", + "allXPaths": [ + "/div[2]/div[2]/div[1]", + "//div[contains(., '2016年4月18日')]", + "//DIV[@class='time s-fc4']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div/div/div" + ], + "exampleValues": [{ "num": 0, "value": "2016年4月18日" }], + "unique_index": "/div[2]/div[2]/div[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 1, + "contentType": 0, + "relative": true, + "name": "参数11_链接文本", + "desc": "", + "relativeXPath": "/div[2]/div[2]/span[1]/span[1]/a[1]", + "allXPaths": [ + "/div[2]/div[2]/span[1]/span[1]/a[1]", + "//a[contains(., '删除')]", + "//A[@class='s-fc3']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div/div/span[last()-1]/span/a" + ], + "exampleValues": [{ "num": 0, "value": "删除" }], + "unique_index": "/div[2]/div[2]/span[1]/span[1]/a[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 2, + "contentType": 0, + "relative": true, + "name": "参数12_链接地址", + "desc": "", + "relativeXPath": "/div[2]/div[2]/span[1]/span[1]/a[1]", + "allXPaths": [ + "/div[2]/div[2]/span[1]/span[1]/a[1]", + "//a[contains(., '删除')]", + "//A[@class='s-fc3']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div/div/span[last()-1]/span/a" + ], + "exampleValues": [{ "num": 0, "value": "javascript:void(0)" }], + "unique_index": "/div[2]/div[2]/span[1]/span[1]/a[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 0, + "contentType": 1, + "relative": true, + "name": "参数13_文本", + "desc": "", + "relativeXPath": "/div[2]/div[2]/span[1]/span[1]/span[1]", + "allXPaths": [ + "/div[2]/div[2]/span[1]/span[1]/span[1]", + "//span[contains(., '|')]", + "//SPAN[@class='sep']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div/div/span[last()-1]/span/span" + ], + "exampleValues": [{ "num": 0, "value": "|" }], + "unique_index": "/div[2]/div[2]/span[1]/span[1]/span[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 1, + "contentType": 0, + "relative": true, + "name": "参数14_链接文本", + "desc": "", + "relativeXPath": "/div[2]/div[2]/a[1]", + "allXPaths": [ + "/div[2]/div[2]/a[1]", + "//a[contains(., '(29844)')]", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div/div/a[last()-1]" + ], + "exampleValues": [{ "num": 0, "value": " (29844)" }], + "unique_index": "/div[2]/div[2]/a[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 2, + "contentType": 0, + "relative": true, + "name": "参数15_链接地址", + "desc": "", + "relativeXPath": "/div[2]/div[2]/a[1]", + "allXPaths": [ + "/div[2]/div[2]/a[1]", + "//a[contains(., '(29844)')]", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div/div/a[last()-1]" + ], + "exampleValues": [{ "num": 0, "value": "javascript:void(0)" }], + "unique_index": "/div[2]/div[2]/a[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 0, + "contentType": 1, + "relative": true, + "name": "参数16_文本", + "desc": "", + "relativeXPath": "/div[2]/div[2]/span[2]", + "allXPaths": [ + "/div[2]/div[2]/span[2]", + "//span[contains(., '|')]", + "//SPAN[@class='sep']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div/div/span" + ], + "exampleValues": [{ "num": 0, "value": "|" }], + "unique_index": "/div[2]/div[2]/span[2]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 1, + "contentType": 0, + "relative": true, + "name": "参数17_链接文本", + "desc": "", + "relativeXPath": "/div[2]/div[2]/a[2]", + "allXPaths": [ + "/div[2]/div[2]/a[2]", + "//a[contains(., '回复')]", + "//A[@class='s-fc3']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div/div/a" + ], + "exampleValues": [{ "num": 0, "value": "回复" }], + "unique_index": "/div[2]/div[2]/a[2]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 2, + "contentType": 0, + "relative": true, + "name": "参数18_链接地址", + "desc": "", + "relativeXPath": "/div[2]/div[2]/a[2]", + "allXPaths": [ + "/div[2]/div[2]/a[2]", + "//a[contains(., '回复')]", + "//A[@class='s-fc3']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-34]/div/div/a" + ], + "exampleValues": [{ "num": 0, "value": "javascript:void(0)" }], + "unique_index": "/div[2]/div[2]/a[2]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 0, + "contentType": 1, + "relative": true, + "name": "参数19_文本", + "desc": "", + "relativeXPath": "/div[2]/div[2]", + "allXPaths": [ + "/div[2]/div[2]", + "//div[contains(., '◆◆_Being_：')]", + "//DIV[@class='que f-brk f-pr s-fc3']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-28]/div/div[last()-1]" + ], + "exampleValues": [ + { + "num": 6, + "value": "：有次我在琴房练琴，练的是一首通俗歌曲改编的钢琴曲。正当我沉醉于自己的琴声时，隔壁忽然传来这首夜曲，那一刻我突然觉得自己练的曲子简直是没法听，于是后半段就没练，坐在琴房听隔壁弹了一个小时夜曲。" + } + ], + "unique_index": "/div[2]/div[2]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 0, + "contentType": 1, + "relative": true, + "name": "参数20_文本", + "desc": "", + "relativeXPath": "/div[2]/div[2]/span[1]/i[1]", + "allXPaths": [ + "/div[2]/div[2]/span[1]/i[1]", + "//i[contains(., '◆')]", + "//I[@class='bd']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-28]/div/div[last()-1]/span/i[last()-1]" + ], + "exampleValues": [{ "num": 6, "value": "◆" }], + "unique_index": "/div[2]/div[2]/span[1]/i[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 0, + "contentType": 1, + "relative": true, + "name": "参数21_文本", + "desc": "", + "relativeXPath": "/div[2]/div[2]/span[1]/i[2]", + "allXPaths": [ + "/div[2]/div[2]/span[1]/i[2]", + "//i[contains(., '◆')]", + "//I[@class='bg']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-28]/div/div[last()-1]/span/i" + ], + "exampleValues": [{ "num": 6, "value": "◆" }], + "unique_index": "/div[2]/div[2]/span[1]/i[2]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 0, + "contentType": 1, + "relative": true, + "name": "参数22_文本", + "desc": "", + "relativeXPath": "/div[2]/div[3]/div[1]", + "allXPaths": [ + "/div[2]/div[3]/div[1]", + "//div[contains(., '2016年4月24日')]", + "//DIV[@class='time s-fc4']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-28]/div/div/div" + ], + "exampleValues": [{ "num": 6, "value": "2016年4月24日" }], + "unique_index": "/div[2]/div[3]/div[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 1, + "contentType": 0, + "relative": true, + "name": "参数23_链接文本", + "desc": "", + "relativeXPath": "/div[2]/div[3]/span[1]/span[1]/a[1]", + "allXPaths": [ + "/div[2]/div[3]/span[1]/span[1]/a[1]", + "//a[contains(., '删除')]", + "//A[@class='s-fc3']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-28]/div/div/span[last()-1]/span/a" + ], + "exampleValues": [{ "num": 6, "value": "删除" }], + "unique_index": "/div[2]/div[3]/span[1]/span[1]/a[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 2, + "contentType": 0, + "relative": true, + "name": "参数24_链接地址", + "desc": "", + "relativeXPath": "/div[2]/div[3]/span[1]/span[1]/a[1]", + "allXPaths": [ + "/div[2]/div[3]/span[1]/span[1]/a[1]", + "//a[contains(., '删除')]", + "//A[@class='s-fc3']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-28]/div/div/span[last()-1]/span/a" + ], + "exampleValues": [{ "num": 6, "value": "javascript:void(0)" }], + "unique_index": "/div[2]/div[3]/span[1]/span[1]/a[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 0, + "contentType": 1, + "relative": true, + "name": "参数25_文本", + "desc": "", + "relativeXPath": "/div[2]/div[3]/span[1]/span[1]/span[1]", + "allXPaths": [ + "/div[2]/div[3]/span[1]/span[1]/span[1]", + "//span[contains(., '|')]", + "//SPAN[@class='sep']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-28]/div/div/span[last()-1]/span/span" + ], + "exampleValues": [{ "num": 6, "value": "|" }], + "unique_index": "/div[2]/div[3]/span[1]/span[1]/span[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 1, + "contentType": 0, + "relative": true, + "name": "参数26_链接文本", + "desc": "", + "relativeXPath": "/div[2]/div[3]/a[1]", + "allXPaths": [ + "/div[2]/div[3]/a[1]", + "//a[contains(., '(7809)')]", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-28]/div/div/a[last()-1]" + ], + "exampleValues": [{ "num": 6, "value": " (7809)" }], + "unique_index": "/div[2]/div[3]/a[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 2, + "contentType": 0, + "relative": true, + "name": "参数27_链接地址", + "desc": "", + "relativeXPath": "/div[2]/div[3]/a[1]", + "allXPaths": [ + "/div[2]/div[3]/a[1]", + "//a[contains(., '(7809)')]", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-28]/div/div/a[last()-1]" + ], + "exampleValues": [{ "num": 6, "value": "javascript:void(0)" }], + "unique_index": "/div[2]/div[3]/a[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 0, + "contentType": 1, + "relative": true, + "name": "参数28_文本", + "desc": "", + "relativeXPath": "/div[2]/div[3]/span[2]", + "allXPaths": [ + "/div[2]/div[3]/span[2]", + "//span[contains(., '|')]", + "//SPAN[@class='sep']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-28]/div/div/span" + ], + "exampleValues": [{ "num": 6, "value": "|" }], + "unique_index": "/div[2]/div[3]/span[2]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 1, + "contentType": 0, + "relative": true, + "name": "参数29_链接文本", + "desc": "", + "relativeXPath": "/div[2]/div[3]/a[2]", + "allXPaths": [ + "/div[2]/div[3]/a[2]", + "//a[contains(., '回复')]", + "//A[@class='s-fc3']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-28]/div/div/a" + ], + "exampleValues": [{ "num": 6, "value": "回复" }], + "unique_index": "/div[2]/div[3]/a[2]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 2, + "contentType": 0, + "relative": true, + "name": "参数30_链接地址", + "desc": "", + "relativeXPath": "/div[2]/div[3]/a[2]", + "allXPaths": [ + "/div[2]/div[3]/a[2]", + "//a[contains(., '回复')]", + "//A[@class='s-fc3']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-28]/div/div/a" + ], + "exampleValues": [{ "num": 6, "value": "javascript:void(0)" }], + "unique_index": "/div[2]/div[3]/a[2]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 4, + "contentType": 1, + "relative": true, + "name": "参数31_图片地址", + "desc": "", + "relativeXPath": "/div[2]/div[2]/img[1]", + "allXPaths": [ + "/div[2]/div[2]/img[1]", + "//img[contains(., '')]", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-19]/div/div[last()-1]/img" + ], + "exampleValues": [ + { + "num": 15, + "value": "http://s1.music.126.net/style/web2/emt/emoji_13.png" + } + ], + "unique_index": "/div[2]/div[2]/img[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 4, + "contentType": 1, + "relative": true, + "name": "参数32_图片地址", + "desc": "", + "relativeXPath": "/div[2]/div[1]/div[1]/img[2]", + "allXPaths": [ + "/div[2]/div[1]/div[1]/img[2]", + "//img[contains(., '')]", + "//IMG[@class='brand-tag brand-package']", + "/html/body/div[last()-10]/div[last()-1]/div/div/div/div/div/div[last()-1]/div[last()-3]/div/div[last()-1]/div/img" + ], + "exampleValues": [ + { + "num": 31, + "value": "https://p5.music.126.net/obj/wonDlsKUwrLClGjCm8Kx/21892874340/50e6/e81d/5292/2d693cbb98954e28d4d9fdb11be99a8b.png" + } + ], + "unique_index": "/div[2]/div[1]/div[1]/img[2]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + } + ], + "loopType": 1 + } + }, + { + "id": 6, + "index": 14, + "parentId": 4, + "type": 0, + "option": 3, + "title": "提取数据", + "sequence": [], + "isInLoop": true, + "position": 1, + "parameters": { + "history": 4, + "tabIndex": -1, + "useLoop": false, + "xpath": "", + "iframe": false, + "wait": 0, + "waitType": 0, + "beforeJS": "", + "beforeJSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "paras": [ + { + "nodeType": 1, + "contentType": 0, + "relative": false, + "name": "参数33_链接文本", + "desc": "", + "extractType": 0, + "relativeXPath": "/html/body/div[1]/div[1]/div[1]/h1[1]/a[1]", + "allXPaths": [ + "/html/body/div[1]/div[1]/div[1]/h1[1]/a[1]", + "//a[contains(., '网易云音乐')]", + "/html/body/div[last()-7]/div[last()-2]/div/h1/a" + ], + "exampleValues": [{ "num": 0, "value": "网易云音乐" }], + "unique_index": "3mjhotxlh4mljnhapl0", + "iframe": false, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + } + ] + } } ] } diff --git a/ElectronJS/tasks/116.json b/ElectronJS/tasks/116.json new file mode 100644 index 0000000..8cfa817 --- /dev/null +++ b/ElectronJS/tasks/116.json @@ -0,0 +1 @@ +{"id":116,"name":"iP地址查询--手机号码查询归属地 | 邮政编码查询 | iP地址归属地查询 | 身份证号码验证在线查询网","url":"https://www.ip138.com","links":"https://www.ip138.com","create_time":"7/4/2023, 8:21:10 AM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"containJudge":false,"desc":"https://www.ip138.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.ip138.com","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://www.ip138.com"}],"outputParameters":[{"id":0,"name":"参数1_文本","desc":"","type":"string","exampleValue":"502BadGateway"},{"id":1,"name":"参数2_文本","desc":"","type":"string","exampleValue":"nginx"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2,3],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.ip138.com","links":"https://www.ip138.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1}},{"id":2,"index":2,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/p[1]/a[1]","iframe":true,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/p[1]/a[1]","//a[contains(., '220.255.29')]","/html/body/p[last()-2]/a[last()-1]"]}},{"id":3,"index":3,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[4],"isInLoop":false,"position":2,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/center","iframe":true,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/center[1]","//center[contains(., '502 Bad Ga')]","/html/body/center[last()-1]"]}},{"id":4,"index":4,"parentId":3,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":true,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"paras":[{"nodeType":0,"contentType":1,"relative":true,"name":"参数1_文本","desc":"","relativeXPath":"/h1[1]","allXPaths":["/h1[1]","//h1[contains(., '502 Bad Ga')]","/html/body/center[last()-1]/h1"],"exampleValues":[{"num":0,"value":"502BadGateway"}],"unique_index":"/h1[1]","iframe":true,"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"参数2_文本","desc":"","relativeXPath":"","allXPaths":["","//center[contains(., 'nginx')]","/html/body/center"],"exampleValues":[{"num":1,"value":"nginx"}],"unique_index":"","iframe":true,"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}}]} \ No newline at end of file diff --git a/ElectronJS/tasks/117.json b/ElectronJS/tasks/117.json new file mode 100644 index 0000000..99c074d --- /dev/null +++ b/ElectronJS/tasks/117.json @@ -0,0 +1 @@ +{"id":117,"name":"iP地址查询--手机号码查询归属地 | 邮政编码查询 | iP地址归属地查询 | 身份证号码验证在线查询网","url":"https://www.ip138.com","links":"https://www.ip138.com","create_time":"7/4/2023, 8:33:55 AM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"containJudge":false,"desc":"https://www.ip138.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.ip138.com","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://www.ip138.com"}],"outputParameters":[{"id":0,"name":"参数1_文本","desc":"","type":"string","exampleValue":"您的iP地址是：[]来自：新加坡Singtel"},{"id":1,"name":"参数2_链接文本","desc":"","type":"string","exampleValue":"220.255.29.208"},{"id":2,"name":"参数3_链接地址","desc":"","type":"string","exampleValue":"https://www.ip138.com/iplookup.php?ip=220.255.29.208&action=2"},{"id":3,"name":"参数4_链接文本","desc":"","type":"string","exampleValue":""},{"id":4,"name":"参数5_链接地址","desc":"","type":"string","exampleValue":"https://www.ipshudi.com/"},{"id":5,"name":"参数6_图片地址","desc":"","type":"string","exampleValue":"https://6.ipchaxun.net/220.255.29.208.gif"},{"id":6,"name":"参数7_文本","desc":"","type":"string","exampleValue":"ip查询api接口"}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.ip138.com","links":"https://www.ip138.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1}},{"id":2,"index":2,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[3],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"/html/body/p","iframe":true,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":1,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0,"allXPaths":["/html/body/p[1]","//p[contains(., '您的iP地址是：[')]","/html/body/p[last()-2]"]}},{"id":3,"index":3,"parentId":2,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":true,"position":0,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":true,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"paras":[{"nodeType":0,"contentType":1,"relative":true,"name":"参数1_文本","desc":"","relativeXPath":"","allXPaths":["","//p[contains(., '您的iP地址是：[')]","/html/body/p[last()-2]"],"exampleValues":[{"num":0,"value":"您的iP地址是：[]来自：新加坡Singtel"}],"unique_index":"","iframe":true,"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":1,"contentType":0,"relative":true,"name":"参数2_链接文本","desc":"","relativeXPath":"/a[1]","allXPaths":["/a[1]","//a[contains(., '220.255.29')]","/html/body/p[last()-2]/a[last()-1]"],"exampleValues":[{"num":0,"value":"220.255.29.208"}],"unique_index":"/a[1]","iframe":true,"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":2,"contentType":0,"relative":true,"name":"参数3_链接地址","desc":"","relativeXPath":"/a[1]","allXPaths":["/a[1]","//a[contains(., '220.255.29')]","/html/body/p[last()-2]/a[last()-1]"],"exampleValues":[{"num":0,"value":"https://www.ip138.com/iplookup.php?ip=220.255.29.208&action=2"}],"unique_index":"/a[1]","iframe":true,"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":1,"contentType":0,"relative":true,"name":"参数4_链接文本","desc":"","relativeXPath":"/a[2]","allXPaths":["/a[2]","//a[contains(., '')]","/html/body/p[last()-2]/a"],"exampleValues":[{"num":0,"value":""}],"unique_index":"/a[2]","iframe":true,"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":2,"contentType":0,"relative":true,"name":"参数5_链接地址","desc":"","relativeXPath":"/a[2]","allXPaths":["/a[2]","//a[contains(., '')]","/html/body/p[last()-2]/a"],"exampleValues":[{"num":0,"value":"https://www.ipshudi.com/"}],"unique_index":"/a[2]","iframe":true,"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":4,"contentType":1,"relative":true,"name":"参数6_图片地址","desc":"","relativeXPath":"/a[2]/img[1]","allXPaths":["/a[2]/img[1]","//img[contains(., '')]","/html/body/p[last()-2]/a/img"],"exampleValues":[{"num":0,"value":"https://6.ipchaxun.net/220.255.29.208.gif"}],"unique_index":"/a[2]/img[1]","iframe":true,"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0},{"nodeType":0,"contentType":1,"relative":true,"name":"参数7_文本","desc":"","relativeXPath":"/a[1]/font[1]","allXPaths":["/a[1]/font[1]","//font[contains(., 'ip查询api接口')]","/html/body/p[last()-1]/a/font"],"exampleValues":[{"num":1,"value":"ip查询api接口"}],"unique_index":"/a[1]/font[1]","iframe":true,"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}],"loopType":1}}]} \ No newline at end of file diff --git a/ElectronJS/tasks/118.json b/ElectronJS/tasks/118.json new file mode 100644 index 0000000..3041d68 --- /dev/null +++ b/ElectronJS/tasks/118.json @@ -0,0 +1,379 @@ +{ + "id": 118, + "name": "iP地址查询--手机号码查询归属地 | 邮政编码查询 | iP地址归属地查询 | 身份证号码验证在线查询网", + "url": "https://www.ip138.com", + "links": "https://www.ip138.com", + "create_time": "7/4/2023, 8:43:31 AM", + "version": "0.3.5", + "saveThreshold": 10, + "cloudflare": 0, + "environment": 0, + "containJudge": false, + "desc": "https://www.ip138.com", + "inputParameters": [ + { + "id": 0, + "name": "urlList_0", + "nodeId": 1, + "nodeName": "打开网页", + "value": "https://www.ip138.com", + "desc": "要采集的网址列表，多行以\\n分开", + "type": "string", + "exampleValue": "https://www.ip138.com" + } + ], + "outputParameters": [ + { + "id": 0, + "name": "参数1_文本", + "desc": "", + "type": "string", + "exampleValue": "502 Bad Gateway" + } + ], + "graph": [ + { + "index": 0, + "id": 0, + "parentId": 0, + "type": -1, + "option": 0, + "title": "root", + "sequence": [1, 4], + "parameters": { + "history": 1, + "tabIndex": 0, + "useLoop": false, + "xpath": "", + "wait": 0 + }, + "isInLoop": false + }, + { + "id": 1, + "index": 1, + "parentId": 0, + "type": 0, + "option": 1, + "title": "打开网页", + "sequence": [], + "isInLoop": false, + "position": 0, + "parameters": { + "useLoop": false, + "xpath": "", + "wait": 0, + "waitType": 0, + "beforeJS": "", + "beforeJSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "url": "https://www.ip138.com", + "links": "https://www.ip138.com", + "maxWaitTime": 10, + "scrollType": 0, + "scrollCount": 1, + "scrollWaitTime": 1 + } + }, + { + "id": -1, + "index": 2, + "parentId": 0, + "type": 1, + "option": 8, + "title": "循环", + "sequence": [3], + "isInLoop": false, + "position": 1, + "parameters": { + "history": 4, + "tabIndex": -1, + "useLoop": false, + "xpath": "/html/body/p", + "iframe": true, + "wait": 0, + "waitType": 0, + "beforeJS": "", + "beforeJSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "scrollType": 0, + "scrollCount": 1, + "scrollWaitTime": 1, + "loopType": 1, + "pathList": "", + "textList": "", + "code": "", + "waitTime": 0, + "exitCount": 0, + "historyWait": 2, + "breakMode": 0, + "breakCode": "", + "breakCodeWaitTime": 0, + "allXPaths": [ + "/html/body/p[1]", + "//p[contains(., '您的iP地址是：[')]", + "/html/body/p[last()-2]" + ] + } + }, + { + "id": -1, + "index": 3, + "parentId": 2, + "type": 0, + "option": 3, + "title": "提取数据", + "sequence": [], + "isInLoop": true, + "position": 0, + "parameters": { + "history": 4, + "tabIndex": -1, + "useLoop": false, + "xpath": "", + "iframe": true, + "wait": 0, + "waitType": 0, + "beforeJS": "", + "beforeJSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "paras": [ + { + "nodeType": 0, + "contentType": 1, + "relative": true, + "name": "参数1_文本", + "desc": "", + "relativeXPath": "", + "allXPaths": [ + "", + "//p[contains(., '您的iP地址是：[')]", + "/html/body/p[last()-2]" + ], + "exampleValues": [ + { "num": 0, "value": "您的iP地址是：[]来自：新加坡Singtel" } + ], + "unique_index": "", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 1, + "contentType": 0, + "relative": true, + "name": "参数2_链接文本", + "desc": "", + "relativeXPath": "/a[1]", + "allXPaths": [ + "/a[1]", + "//a[contains(., '220.255.29')]", + "/html/body/p[last()-2]/a[last()-1]" + ], + "exampleValues": [{ "num": 0, "value": "220.255.29.208" }], + "unique_index": "/a[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 2, + "contentType": 0, + "relative": true, + "name": "参数3_链接地址", + "desc": "", + "relativeXPath": "/a[1]", + "allXPaths": [ + "/a[1]", + "//a[contains(., '220.255.29')]", + "/html/body/p[last()-2]/a[last()-1]" + ], + "exampleValues": [ + { + "num": 0, + "value": "https://www.ip138.com/iplookup.php?ip=220.255.29.208&action=2" + } + ], + "unique_index": "/a[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 1, + "contentType": 0, + "relative": true, + "name": "参数4_链接文本", + "desc": "", + "relativeXPath": "/a[2]", + "allXPaths": [ + "/a[2]", + "//a[contains(., '')]", + "/html/body/p[last()-2]/a" + ], + "exampleValues": [{ "num": 0, "value": "" }], + "unique_index": "/a[2]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 2, + "contentType": 0, + "relative": true, + "name": "参数5_链接地址", + "desc": "", + "relativeXPath": "/a[2]", + "allXPaths": [ + "/a[2]", + "//a[contains(., '')]", + "/html/body/p[last()-2]/a" + ], + "exampleValues": [ + { "num": 0, "value": "https://www.ipshudi.com/" } + ], + "unique_index": "/a[2]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 4, + "contentType": 1, + "relative": true, + "name": "参数6_图片地址", + "desc": "", + "relativeXPath": "/a[2]/img[1]", + "allXPaths": [ + "/a[2]/img[1]", + "//img[contains(., '')]", + "/html/body/p[last()-2]/a/img" + ], + "exampleValues": [ + { "num": 0, "value": "https://6.ipchaxun.net/220.255.29.208.gif" } + ], + "unique_index": "/a[2]/img[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + }, + { + "nodeType": 0, + "contentType": 1, + "relative": true, + "name": "参数7_文本", + "desc": "", + "relativeXPath": "/a[1]/font[1]", + "allXPaths": [ + "/a[1]/font[1]", + "//font[contains(., 'ip查询api接口')]", + "/html/body/p[last()-1]/a/font" + ], + "exampleValues": [{ "num": 1, "value": "ip查询api接口" }], + "unique_index": "/a[1]/font[1]", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + } + ], + "loopType": 1 + } + }, + { + "id": 2, + "index": 4, + "parentId": 0, + "type": 0, + "option": 3, + "title": "提取数据", + "sequence": [], + "isInLoop": false, + "position": 1, + "parameters": { + "history": 3, + "tabIndex": -1, + "useLoop": false, + "xpath": "", + "iframe": true, + "wait": 0, + "waitType": 0, + "beforeJS": "", + "beforeJSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "paras": [ + { + "nodeType": 0, + "contentType": 0, + "relative": false, + "name": "参数1_文本", + "desc": "", + "extractType": 0, + "relativeXPath": "/html/body/center[1]/h1[1]", + "allXPaths": [ + "/html/body/center[1]/h1[1]", + "//h1[contains(., '502 Bad Ga')]", + "/html/body/center[last()-1]/h1" + ], + "exampleValues": [{ "num": 0, "value": "502 Bad Gateway" }], + "unique_index": "l37gwwpsg29ljnkgn7r", + "iframe": true, + "default": "", + "beforeJS": "", + "beforeJSWaitTime": 0, + "JS": "", + "JSWaitTime": 0, + "afterJS": "", + "afterJSWaitTime": 0, + "downloadPic": 0 + } + ] + } + } + ] +} diff --git a/ElectronJS/tasks/119.json b/ElectronJS/tasks/119.json new file mode 100644 index 0000000..e339d03 --- /dev/null +++ b/ElectronJS/tasks/119.json @@ -0,0 +1 @@ +{"id":119,"name":"","url":"https://lihkg.com/","links":"https://lihkg.com/thread/3433502/page/1","create_time":"7/4/2023, 5:09:13 PM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"containJudge":false,"desc":"https://lihkg.com/","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://lihkg.com/thread/3433502/page/1","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://lihkg.com/thread/3433502/page/1"}],"outputParameters":[],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,6],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://lihkg.com/","links":"https://lihkg.com/thread/3433502/page/1","maxWaitTime":10,"scrollType":"2","scrollCount":3,"scrollWaitTime":1}},{"id":-1,"index":2,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"//*[contains(@class, \"P3e8vKaXmUeXC9dJgjnsu\")]/div[1]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/div[2]/div[1]/div[1]","//div[contains(., 'LIHKG 討論區使')]","/html/body/div[last()-4]/div[last()-2]/div/div/div"]}},{"id":-1,"index":3,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"//*[contains(@class, \"P3e8vKaXmUeXC9dJgjnsu\")]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/div[2]/div[1]","//div[contains(., 'LIHKG 討論區使')]","//DIV[@class='P3e8vKaXmUeXC9dJgjnsu']","/html/body/div[last()-4]/div[last()-2]/div/div"]}},{"id":-1,"index":4,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"//*[contains(@class, \"_1PdImYJBCsN8lH0MB4tnqV\")]/a[1]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":"2","scrollCount":5,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/div[1]/div[1]/ul[1]/li[1]/a[1]","//a[contains(., '最新')]","/html/body/div[last()-4]/div[last()-2]/div[last()-1]/div[last()-1]/ul/li[last()-1]/a"]}},{"id":-1,"index":5,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":3,"tabIndex":-1,"useLoop":false,"xpath":"//*[contains(@class, \"P3e8vKaXmUeXC9dJgjnsu\")]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/div[2]/div[1]","//div[contains(., 'LIHKG 討論區使')]","//DIV[@class='P3e8vKaXmUeXC9dJgjnsu']","/html/body/div[last()-4]/div[last()-2]/div/div"]}},{"id":2,"index":6,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":5,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"1\"]/div[1]/small[1]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":"2","scrollCount":4,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/div[2]/div[1]/div[2]/div[2]/div[2]/div[1]/div[1]/small[1]","//small[contains(., '#1李芯悅•1 小時')]","//SMALL[@class='_1VcuFUmnOEK51TsshmrnJM']","/html/body/div[last()-5]/div[last()-2]/div/div/div[last()-2]/div/div[last()-24]/div[last()-1]/div/small"]}}]} \ No newline at end of file diff --git a/ElectronJS/tasks/120.json b/ElectronJS/tasks/120.json new file mode 100644 index 0000000..092ccd8 --- /dev/null +++ b/ElectronJS/tasks/120.json @@ -0,0 +1 @@ +{"id":120,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"7/4/2023, 5:36:05 PM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://www.jd.com"},{"id":1,"name":"inputText_1","nodeName":"输入文字","nodeId":3,"desc":"要输入的文本，如京东搜索框输入：电脑","type":"string","exampleValue":"123Field[\"自定义操作\"]456","value":"123Field[\"自定义操作\"]456"}],"outputParameters":[{"id":0,"name":"自定义操作","desc":"自定义操作返回的数据","type":"string","exampleValue":""}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2,3],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1}},{"id":2,"index":2,"parentId":0,"type":0,"option":5,"title":"自定义操作","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"codeMode":"1","code":"python D:/test.py","waitTime":0,"recordASField":"1"}},{"id":3,"index":3,"parentId":0,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":false,"position":2,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"key\"]","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"value":"123Field[\"自定义操作\"]456","allXPaths":["/html/body/div[4]/div[1]/div[2]/div[1]/input[1]","//input[contains(., '')]","id(\"key\")","//INPUT[@class='text defcolor']","/html/body/div[last()-6]/div/div[last()-2]/div/input"]}}]} \ No newline at end of file diff --git a/ElectronJS/tasks/121.json b/ElectronJS/tasks/121.json new file mode 100644 index 0000000..40c3df1 --- /dev/null +++ b/ElectronJS/tasks/121.json @@ -0,0 +1 @@ +{"id":121,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"7/4/2023, 5:55:11 PM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://www.jd.com"}],"outputParameters":[],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":15,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":"2","scrollCount":3,"scrollWaitTime":1}},{"id":-1,"index":2,"parentId":0,"type":1,"option":8,"title":"循环","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"loopType":0,"pathList":"","textList":"","code":"","waitTime":0,"exitCount":0,"historyWait":2,"breakMode":0,"breakCode":"","breakCodeWaitTime":0}},{"id":-1,"index":3,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":0,"scrollCount":1,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[]}}]} \ No newline at end of file diff --git a/ElectronJS/tasks/122.json b/ElectronJS/tasks/122.json new file mode 100644 index 0000000..f5f409c --- /dev/null +++ b/ElectronJS/tasks/122.json @@ -0,0 +1 @@ +{"id":122,"name":"bbs","url":"https://lihkg.com/thread/3429557/page/1","links":"https://lihkg.com/thread/3429557/page/1","create_time":"7/4/2023, 5:57:56 PM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"containJudge":false,"desc":"https://lihkg.com/thread/3429557/page/1","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://lihkg.com/thread/3429557/page/1","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://lihkg.com/thread/3429557/page/1"}],"outputParameters":[],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://lihkg.com/thread/3429557/page/1","links":"https://lihkg.com/thread/3429557/page/1","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1}},{"id":2,"index":2,"parentId":0,"type":0,"option":2,"title":"点击元素","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"//*[contains(@class, \"_1PdImYJBCsN8lH0MB4tnqV\")]/a[1]","iframe":false,"wait":2,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"scrollType":"2","scrollCount":3,"scrollWaitTime":1,"clickWay":0,"maxWaitTime":10,"paras":[],"allXPaths":["/html/body/div[1]/div[2]/div[1]/div[1]/ul[1]/li[1]/a[1]","//a[contains(., '最新')]","/html/body/div[last()-5]/div[last()-2]/div[last()-1]/div[last()-1]/ul/li[last()-1]/a"]}}]} \ No newline at end of file diff --git a/ElectronJS/tasks/123.json b/ElectronJS/tasks/123.json new file mode 100644 index 0000000..b71c8b6 --- /dev/null +++ b/ElectronJS/tasks/123.json @@ -0,0 +1 @@ +{"id":123,"name":"京东全球版-专业的综合网上购物商城","url":"https://www.jd.com","links":"https://www.jd.com","create_time":"7/4/2023, 6:21:37 PM","version":"0.3.5","saveThreshold":10,"cloudflare":0,"environment":0,"containJudge":false,"desc":"https://www.jd.com","inputParameters":[{"id":0,"name":"urlList_0","nodeId":1,"nodeName":"打开网页","value":"https://www.jd.com","desc":"要采集的网址列表，多行以\\n分开","type":"string","exampleValue":"https://www.jd.com"},{"id":1,"name":"inputText_1","nodeName":"输入文字","nodeId":2,"desc":"要输入的文本，如京东搜索框输入：电脑","type":"string","exampleValue":"123Field[\"文本1\"]","value":"123Field[\"文本1\"]"},{"id":2,"name":"inputText_2","nodeName":"输入文字","nodeId":6,"desc":"要输入的文本，如京东搜索框输入：电脑","type":"string","exampleValue":"123Field[\"文本1\"]","value":"123Field[\"文本1\"]"}],"outputParameters":[{"id":0,"name":"文本1","desc":"自定义操作返回的数据","type":"string","exampleValue":""}],"graph":[{"index":0,"id":0,"parentId":0,"type":-1,"option":0,"title":"root","sequence":[1,4,6,5,2],"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","wait":0},"isInLoop":false},{"id":1,"index":1,"parentId":0,"type":0,"option":1,"title":"打开网页","sequence":[],"isInLoop":false,"position":0,"parameters":{"useLoop":false,"xpath":"","wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"url":"https://www.jd.com","links":"https://www.jd.com","maxWaitTime":10,"scrollType":0,"scrollCount":1,"scrollWaitTime":1}},{"id":5,"index":2,"parentId":0,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":false,"position":4,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"key\"]","iframe":false,"wait":5,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"value":"123Field[\"文本1\"]","allXPaths":["/html/body/div[4]/div[1]/div[2]/div[1]/input[1]","//input[contains(., '')]","id(\"key\")","//INPUT[@class='text defcolor']","/html/body/div[last()-6]/div/div[last()-2]/div/input"]}},{"id":-1,"index":3,"parentId":0,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"paras":[{"nodeType":0,"contentType":0,"relative":false,"name":"参数1_文本","desc":"","extractType":0,"relativeXPath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[3]","allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[3]","//div[contains(., '/电脑/办公')]","//DIV[@class='LeftSide_menu_item__SBMWC LeftSide_text_space__2UhbG LeftSide_menu_hover__OCHiO']","/html/body/div[last()-5]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]/div[last()-10]"],"exampleValues":[{"num":0,"value":"/电脑/办公"}],"unique_index":"w50zsf5f2mljo4y3pm","iframe":false,"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}]}},{"id":2,"index":4,"parentId":0,"type":0,"option":5,"title":"文本1","sequence":[],"isInLoop":false,"position":1,"parameters":{"history":1,"tabIndex":0,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"codeMode":"1","code":"python D:/test.py","waitTime":0,"recordASField":"1"}},{"id":4,"index":5,"parentId":0,"type":0,"option":3,"title":"提取数据","sequence":[],"isInLoop":false,"position":3,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"","iframe":false,"wait":0,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"paras":[{"nodeType":1,"contentType":0,"relative":false,"name":"文本1","desc":"","extractType":0,"relativeXPath":"/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/a[1]","allXPaths":["/html/body/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/a[1]","//a[contains(., '家用电器')]","/html/body/div[last()-5]/div/div[last()-4]/div/div[last()-2]/div/div/div/div[last()-1]/div[last()-11]/a"],"exampleValues":[{"num":0,"value":"家用电器"}],"unique_index":"lzzj3pipdu9ljo4zt5g","iframe":false,"default":"","beforeJS":"","beforeJSWaitTime":0,"JS":"","JSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"downloadPic":0}]}},{"id":3,"index":6,"parentId":0,"type":0,"option":4,"title":"输入文字","sequence":[],"isInLoop":false,"position":2,"parameters":{"history":4,"tabIndex":-1,"useLoop":false,"xpath":"//*[@id=\"key\"]","iframe":false,"wait":5,"waitType":0,"beforeJS":"","beforeJSWaitTime":0,"afterJS":"","afterJSWaitTime":0,"value":"123Field[\"文本1\"]","allXPaths":["/html/body/div[4]/div[1]/div[2]/div[1]/input[1]","//input[contains(., '')]","id(\"key\")","//INPUT[@class='text defcolor']","/html/body/div[last()-6]/div/div[last()-2]/div/input"]}}]} \ No newline at end of file diff --git a/ExecuteStage/.vscode/launch.json b/ExecuteStage/.vscode/launch.json index 8972abb..292941f 100644 --- a/ExecuteStage/.vscode/launch.json +++ b/ExecuteStage/.vscode/launch.json @@ -10,7 +10,7 @@ "program": "${file}", "console": "integratedTerminal", "justMyCode": true, - "args": ["--id", "[5]", "--read_type", "remote", "--headless", "0"] + "args": ["--id", "[18]", "--read_type", "remote", "--headless", "0"] // "args": ["--id", "[2]", "--read_type", "remote", "--headless", "0", "--saved_file_name", "YOUTUBE"] // "args": ["--id", "[44]", "--headless", "0", "--user_data", "1"] } diff --git a/ExecuteStage/easyspider_executestage.py b/ExecuteStage/easyspider_executestage.py index 4d77ec6..43061e9 100644 --- a/ExecuteStage/easyspider_executestage.py +++ b/ExecuteStage/easyspider_executestage.py @@ -12,7 +12,7 @@ import sys # import base64 # import hashlib import time -import keyboard +# import keyboard import requests from lxml import etree from selenium.webdriver.chrome.options import Options @@ -28,18 +28,18 @@ from selenium.common.exceptions import StaleElementReferenceException, InvalidSe from selenium.webdriver.common.desired_capabilities import DesiredCapabilities from selenium.webdriver.support.ui import Select from selenium.webdriver import ActionChains +from selenium.webdriver.common.by import By import undetected_chromedriver as uc import random # import numpy import csv import os -from selenium.webdriver.common.by import By from commandline_config import Config import pytesseract from PIL import Image -import uuid +# import uuid from threading import Thread, Event - +from myChrome import MyChrome from utils import check_pause, download_image, get_output_code, isnull desired_capabilities = DesiredCapabilities.CHROME desired_capabilities["pageLoadStrategy"] = "none" @@ -111,11 +111,23 @@ class BrowserThread(Thread): # 检测如果没有复杂的操作，优化提取数据流程 def preprocess(self): for node in self.procedure: + try: + iframe = node["parameters"]["iframe"] + except: + node["parameters"]["iframe"] = False if node["option"] == 3: # 提取数据操作 paras = node["parameters"]["paras"] for para in paras: + try: + iframe = para["iframe"] + except: + para["iframe"] = False if para["beforeJS"] == "" and para["afterJS"] == "" and para["contentType"] <= 1 and para["nodeType"] <= 2: - para["optimizable"] = True + # iframe中提取数据的绝对寻址操作不可优化 + if para["relative"] == False and para["iframe"] == True: + para["optimizable"] = False + else: + para["optimizable"] = True else: para["optimizable"] = False @@ -170,7 +182,8 @@ class BrowserThread(Thread): if scrollType != 0 and para["scrollCount"] > 0: # 控制屏幕向下滚动 for i in range(para["scrollCount"]): self.Log("Wait for set second after screen scrolling") - body = self.browser.find_element(By.CSS_SELECTOR, "body") + body = self.browser.find_element( + By.CSS_SELECTOR, "body", iframe=para["iframe"]) if scrollType == 1: body.send_keys(Keys.PAGE_DOWN) elif scrollType == 2: @@ -183,7 +196,8 @@ class BrowserThread(Thread): if scrollType != 0 and para["scrollCount"] > 0: # 控制屏幕向下滚动 for i in range(para["scrollCount"]): self.Log("Wait for set second after screen scrolling") - body = self.browser.find_element(By.CSS_SELECTOR, "body") + body = self.browser.find_element( + By.CSS_SELECTOR, "body", iframe=para["iframe"]) if scrollType == 1: body.send_keys(Keys.PGDN) elif scrollType == 2: @@ -253,7 +267,8 @@ class BrowserThread(Thread): max_wait_time = int(paras["waitTime"]) if codeMode == 2: # 使用循环的情况下，传入的clickPath就是实际的xpath try: - elements = self.browser.find_elements(By.XPATH, loopPath) + elements = self.browser.find_elements( + By.XPATH, loopPath, iframe=paras["iframe"]) element = elements[index] output = self.execute_code( codeMode, code, max_wait_time, element) @@ -277,7 +292,7 @@ class BrowserThread(Thread): optionValue = para["optionValue"] try: dropdown = Select(self.browser.find_element( - By.XPATH, para["xpath"])) + By.XPATH, para["xpath"], iframe=para["iframe"])) try: if optionMode == 0: # 获取当前选中的选项索引 @@ -310,7 +325,8 @@ class BrowserThread(Thread): index = 0 path = para["xpath"] # 不然使用元素定义的xpath try: - elements = self.browser.find_elements(By.XPATH, path) + elements = self.browser.find_elements( + By.XPATH, path, iframe=para["iframe"]) element = elements[index] try: ActionChains(self.browser).move_to_element(element).perform() @@ -396,7 +412,7 @@ class BrowserThread(Thread): continue elif tType == 2: # 当前页面包含元素 try: - if self.browser.find_element(By.XPATH, cnode["parameters"]["value"]): + if self.browser.find_element(By.XPATH, cnode["parameters"]["value"], iframe=cnode["parameters"]["iframe"]): executeBranchId = i break except: # 找不到元素或者xpath写错了，下一个条件 @@ -410,7 +426,7 @@ class BrowserThread(Thread): continue elif tType == 4: # 当前循环元素包括元素 try: - if loopElement.find_element(By.XPATH, cnode["parameters"]["value"][1:]): + if loopElement.find_element(By.XPATH, cnode["parameters"]["value"][1:], iframe=cnode["parameters"]["iframe"]): executeBranchId = i break except: # 找不到元素或者xpath写错了，下一个条件 @@ -449,7 +465,8 @@ class BrowserThread(Thread): 'return history.length') # 记录本次循环内的history的length self.history["index"] = thisHistoryLength self.history["handle"] = thisHandle - + if node["parameters"]["iframe"]: + self.browser.switch_to.default_content() # 循环前切换到主文档 if int(node["parameters"]["loopType"]) == 0: # 单个元素循环 # 无跳转标签页操作 count = 0 # 执行次数 @@ -457,7 +474,7 @@ class BrowserThread(Thread): try: finished = False element = self.browser.find_element( - By.XPATH, node["parameters"]["xpath"]) + By.XPATH, node["parameters"]["xpath"], iframe=node["parameters"]["iframe"]) for i in node["sequence"]: # 挨个执行操作 self.executeNode( i, element, node["parameters"]["xpath"], 0) @@ -504,7 +521,7 @@ class BrowserThread(Thread): elif int(node["parameters"]["loopType"]) == 1: # 不固定元素列表 try: elements = self.browser.find_elements(By.XPATH, - node["parameters"]["xpath"]) + node["parameters"]["xpath"], iframe=node["parameters"]["iframe"]) if len(elements) == 0: print("Loop element not found: ", node["parameters"]["xpath"]) @@ -552,7 +569,8 @@ class BrowserThread(Thread): # 千万不要忘了分割！！ for path in node["parameters"]["pathList"].split("\n"): try: - element = self.browser.find_element(By.XPATH, path) + element = self.browser.find_element( + By.XPATH, path, iframe=node["parameters"]["iframe"]) for i in node["sequence"]: # 挨个执行操作 self.executeNode(i, element, path, 0) if self.browser.current_window_handle != thisHandle: # 如果执行完一次循环之后标签页的位置发生了变化 @@ -633,10 +651,11 @@ class BrowserThread(Thread): self.executeNode(i, code, node["parameters"]["xpath"], 0) self.history["index"] = thisHistoryLength self.history["handle"] = self.browser.current_window_handle + if node["parameters"]["iframe"]: + self.browser.switch_to.default_content() self.scrollDown(node["parameters"]) # 打开网页事件 - def openPage(self, para, loopValue): time.sleep(1) # 打开网页后强行等待至少1秒 if len(self.browser.window_handles) > 1: @@ -677,19 +696,25 @@ class BrowserThread(Thread): self.Log('Time out after set seconds when loading page: ' + url) self.recordLog( 'Time out after set seconds when loading page: ' + url) - self.browser.execute_script('window.stop()') + try: + self.browser.execute_script('window.stop()') + except: + pass try: self.history["index"] = self.browser.execute_script( "return history.length") except TimeoutException: - self.browser.execute_script('window.stop()') - self.history["index"] = self.browser.execute_script( - "return history.length") + try: + self.browser.execute_script('window.stop()') + self.history["index"] = self.browser.execute_script( + "return history.length") + except: + self.history["index"] = 0 self.scrollDown(para) # 控制屏幕向下滚动 if self.containJudge: try: self.bodyText = self.browser.find_element( - By.CSS_SELECTOR, "body").text + By.CSS_SELECTOR, "body", iframe=False).text self.Log('URL Page: ' + url) self.recordLog('URL Page: ' + url) except TimeoutException: @@ -702,7 +727,7 @@ class BrowserThread(Thread): self.Log("Need to wait 1 second to get body text") # 再执行一遍 self.bodyText = self.browser.find_element( - By.CSS_SELECTOR, "body").text + By.CSS_SELECTOR, "body", iframe=False).text except Exception as e: self.Log(e) self.recordLog(str(e)) @@ -713,7 +738,8 @@ class BrowserThread(Thread): time.sleep(0.1) # 输入之前等待0.1秒 self.Log("Wait 0.1 second before input") try: - textbox = self.browser.find_element(By.XPATH, para["xpath"]) + textbox = self.browser.find_element( + By.XPATH, para["xpath"], iframe=para["iframe"]) # textbox.send_keys(Keys.CONTROL, 'a') # textbox.send_keys(Keys.BACKSPACE) self.execute_code( @@ -770,7 +796,8 @@ class BrowserThread(Thread): self.browser.set_script_timeout(maxWaitTime) # 点击前对该元素执行一段JavaScript代码 try: - element = self.browser.find_element(By.XPATH, path) + element = self.browser.find_element( + By.XPATH, path, iframe=para["iframe"]) if para["beforeJS"] != "": self.execute_code(2, para["beforeJS"], para["beforeJSWaitTime"], element) @@ -786,7 +813,7 @@ class BrowserThread(Thread): except: click_way = 0 try: - if click_way == 0: # 用selenium的点击方法 + if click_way == 0 or para["iframe"]: # 用selenium的点击方法 actions = ActionChains(self.browser) # 实例化一个action对象 actions.click(element).perform() elif click_way == 1: # 用js的点击方法 @@ -804,7 +831,8 @@ class BrowserThread(Thread): # 点击前对该元素执行一段JavaScript代码 try: if para["afterJS"] != "": - element = self.browser.find_element(By.XPATH, path) + element = self.browser.find_element( + By.XPATH, path, iframe=para["iframe"]) self.execute_code(2, para["afterJS"], para["afterJSWaitTime"], element) except: @@ -812,6 +840,8 @@ class BrowserThread(Thread): self.recordLog("Cannot find element:" + path + ", please try to set the wait time before executing this operation") print("找不到要点击的元素:" + path + "，请尝试在执行此操作前设置等待时间") + if para["iframe"]: + self.browser.switch_to.default_content() waitTime = float(para["wait"]) + 0.01 # 点击之后等待 try: waitType = int(para["waitType"]) @@ -1071,13 +1101,13 @@ class BrowserThread(Thread): p["relativeXPath"] + ")" + \ "[" + str(index + 1) + "]" element = self.browser.find_element( - By.XPATH, full_path) + By.XPATH, full_path, iframe=p["iframe"]) else: element = loopElement.find_element(By.XPATH, p["relativeXPath"][1:]) else: element = self.browser.find_element( - By.XPATH, p["relativeXPath"]) + By.XPATH, p["relativeXPath"], iframe=p["iframe"]) except (NoSuchElementException, InvalidSelectorException, StaleElementReferenceException): # 找不到元素的时候，使用默认值 # print(p) try: @@ -1110,10 +1140,11 @@ class BrowserThread(Thread): p["relativeXPath"][1:]) else: element = self.browser.find_element( - By.XPATH, p["relativeXPath"]) + By.XPATH, p["relativeXPath"], iframe=p["iframe"]) # rt.end() else: - element = self.browser.find_element(By.XPATH, "//body") + element = self.browser.find_element( + By.XPATH, "//body", iframe=p["iframe"]) try: self.execute_code( 2, p["beforeJS"], p["beforeJSWaitTime"], element) # 执行前置js @@ -1135,7 +1166,7 @@ class BrowserThread(Thread): 'StaleElementReferenceException: loopElement+relativeXPath') else: element = self.browser.find_element( - By.XPATH, p["relativeXPath"]) + By.XPATH, p["relativeXPath"], iframe=p["iframe"]) self.recordLog( 'StaleElementReferenceException: relativeXPath') content = self.get_content(p, element) @@ -1327,7 +1358,7 @@ if __name__ == '__main__': 'mobileEmulation', {'deviceName': 'iPhone X'}) # 模拟iPhone X浏览 except: pass - browser_t = webdriver.Chrome( + browser_t = MyChrome( options=options, chrome_options=option, executable_path=driver_path) elif cloudflare == 1: browser_t = uc.Chrome( diff --git a/ExecuteStage/myChrome.py b/ExecuteStage/myChrome.py new file mode 100644 index 0000000..6fd6fe7 --- /dev/null +++ b/ExecuteStage/myChrome.py @@ -0,0 +1,75 @@ + + +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.common.action_chains import ActionChains +from selenium import webdriver +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.common.by import By +from selenium.common.exceptions import NoSuchElementException +from selenium.common.exceptions import TimeoutException +from selenium.common.exceptions import StaleElementReferenceException, InvalidSelectorException +from selenium.webdriver.common.desired_capabilities import DesiredCapabilities +from selenium.webdriver.support.ui import Select +from selenium.webdriver import ActionChains +from selenium.webdriver.common.by import By +desired_capabilities = DesiredCapabilities.CHROME +desired_capabilities["pageLoadStrategy"] = "none" + + +class MyChrome(webdriver.Chrome): + def find_element(self, by=By.ID, value=None, iframe=False): + # 在这里改变查找元素的行为 + if iframe: + # 获取所有的 iframe + try: + iframes = super().find_elements(By.XPATH, "//iframe") + except Exception as e: + print(e) + find_element = False + # 遍历所有的 iframe 并点击里面的元素 + for iframe in iframes: + # 切换到 iframe + super().switch_to.frame(iframe) + try: + # 在 iframe 中查找并点击元素 + # 在这个例子中，我们查找 XPath 为 '//div[1]' 的元素 + element = super().find_element(by=by, value=value) + find_element = True + except NoSuchElementException: + print("No such element found in the iframe") + # 完成操作后切回主文档 + # super().switch_to.default_content() + if find_element: + return element + if not find_element: + raise NoSuchElementException + else: + return super().find_element(by=by, value=value) + + def find_elements(self, by=By.ID, value=None, iframe=False): + # 在这里改变查找元素的行为 + if iframe: + # 获取所有的 iframe + iframes = iframes = super().find_elements(By.CSS_SELECTOR, "iframe") + find_element = False + # 遍历所有的 iframe 并点击里面的元素 + for iframe in iframes: + # 切换到 iframe + super().switch_to.frame(iframe) + try: + # 在 iframe 中查找并点击元素 + # 在这个例子中，我们查找 XPath 为 '//div[1]' 的元素 + elements = super().find_elements(by=by, value=value) + find_element = True + except NoSuchElementException: + print("No such element found in the iframe") + # 完成操作后切回主文档 + # super().switch_to.default_content() + if find_element: + return elements + if not find_element: + raise NoSuchElementException + else: + return super().find_elements(by=by, value=value) diff --git a/Readme.md b/Readme.md index 47e2830..c0134ac 100644 --- a/Readme.md +++ b/Readme.md @@ -42,7 +42,7 @@ Bilibili/B站视频教程: [EasySpider介绍 - 中国地震台网采集案例](https://www.bilibili.com/video/BV1th411A7ey/) -[自动/手动同类型元素匹配功能说明](https://www.bilibili.com/video/BV1pu411a7pK/) +[设置页面向下滚动](https://www.bilibili.com/video/BV1G14y1o7Qa/) [如何无代码可视化的爬取需要登录才能爬的网站 - 知乎网站案例](https://www.bilibili.com/video/BV1HV4y1r7v8) @@ -70,6 +70,8 @@ Bilibili/B站视频教程: [如何同时执行多个任务（并行多开）](https://www.bilibili.com/video/BV13c411G7LE/) +[Python代码运算后的结果作为文本框的输入](https://www.bilibili.com/video/BV1kF411R7VJ/) + [实例 - 反人类网站文章采集和代码调试](https://www.bilibili.com/video/BV11W4y1D71t/) Refer to [Youtube Playlist](https://youtube.com/playlist?list=PL0kEFEkWrT7mt9MUlEBV2DTo1QsaanUTp) to see the video tutorials of EasySpider.