mirror of
https://github.com/NaiboWang/EasySpider.git
synced 2026-09-01 15:22:26 +08:00
beta 0.1.0
This commit is contained in:
NaiboWang-Alienware
Binary file not shown.
@@ -0,0 +1,151 @@
|
||||
# _*_coding:utf-8_*_
|
||||
from hashlib import new
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from multiprocessing import Process
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
import os
|
||||
import pickle
|
||||
import calendar
|
||||
import re
|
||||
from copy import deepcopy
|
||||
import requests
|
||||
import csv
|
||||
from commandline_config import Config
|
||||
from service_invoke import invokeService
|
||||
|
||||
|
||||
class TimeUtil(object):
|
||||
@classmethod
|
||||
def parse_timezone(cls, timezone):
|
||||
"""
|
||||
解析时区表示
|
||||
:param timezone: str eg: +8
|
||||
:return: dict{symbol, offset}
|
||||
"""
|
||||
result = re.match(r'(?P<symbol>[+-])(?P<offset>\d+)', timezone)
|
||||
symbol = result.groupdict()['symbol']
|
||||
offset = int(result.groupdict()['offset'])
|
||||
|
||||
return {
|
||||
'symbol': symbol,
|
||||
'offset': offset
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def convert_timezone(cls, dt, timezone="+0"):
|
||||
"""默认是utc时间，需要"""
|
||||
result = cls.parse_timezone(timezone)
|
||||
symbol = result['symbol']
|
||||
|
||||
offset = result['offset']
|
||||
|
||||
if symbol == '+':
|
||||
return dt + timedelta(hours=offset)
|
||||
elif symbol == '-':
|
||||
return dt - timedelta(hours=offset)
|
||||
else:
|
||||
raise Exception('dont parse timezone format')
|
||||
|
||||
|
||||
def generate_timestamp():
|
||||
current_GMT = time.gmtime()
|
||||
# ts stores timestamp
|
||||
ts = calendar.timegm(current_GMT)
|
||||
|
||||
current_time = datetime.utcnow()
|
||||
convert_now = TimeUtil.convert_timezone(current_time, '+8')
|
||||
print("current_time: " + str(convert_now))
|
||||
return str(convert_now)
|
||||
|
||||
|
||||
def main():
|
||||
# result = os.popen('python ServiceWrapper_ExecuteStage.py 38')
|
||||
# res = result.read()
|
||||
# for line in res.splitlines():
|
||||
# print("\n\n\n\nfinename:\n\n\n\n\n", line)
|
||||
config = {
|
||||
"pages": 5,
|
||||
"test": False,
|
||||
"test_pages": 3,
|
||||
}
|
||||
c = Config(config)
|
||||
print(c)
|
||||
csv_reader = csv.reader(open("./关键词.csv", encoding='utf-8'))
|
||||
keywords = []
|
||||
i = 0
|
||||
for line in csv_reader:
|
||||
if i < c.test_pages:
|
||||
print(line)
|
||||
i += 1
|
||||
keywords.append(line[0])
|
||||
urlList = ""
|
||||
i = 0
|
||||
for keyword in keywords:
|
||||
url = "https://so.toutiao.com/search?dvpf=pc&source=pagination&filter_vendor=site&keyword=%s&pd=synthesis&filter_vendor=site&action_type=pagination&page_num=0\r\n" % keyword
|
||||
# print(url)
|
||||
urlList += url
|
||||
i += 1
|
||||
if c.test and i > c.test_pages:
|
||||
break
|
||||
print(urlList)
|
||||
|
||||
# result = requests.post(
|
||||
# "http://servicewrapper.naibo.wang/backEnd/invokeService",
|
||||
# data={"id": 6, # serviceID
|
||||
# "paras": json.dumps({"loopTimes_Loop_Click_1": c.pages,
|
||||
# "urlList_0": urlList,
|
||||
# }),
|
||||
# })
|
||||
# authorTaskID = int(result.text)
|
||||
authorTaskID = invokeService(
|
||||
0, {"loopTimes_Loop_Click_1": c.pages, "urlList_0": urlList})
|
||||
print("authorTaskID: " + str(authorTaskID))
|
||||
# exit(0)
|
||||
filename = generate_timestamp().replace(" ", "").replace(":", "-")
|
||||
print("filename:", filename)
|
||||
|
||||
command = 'python ServiceWrapper_ExecuteStage_local.py ' + \
|
||||
str(authorTaskID) + ' ' + filename
|
||||
result = os.system(command)
|
||||
|
||||
# authorTaskID = 53
|
||||
file_name = "task_" + str(authorTaskID) + "_" + filename + ".csv"
|
||||
# file_name = "task_53_2022-10-1723-35-40.881448.csv"
|
||||
print("file_name:", file_name)
|
||||
csv_reader = csv.reader(
|
||||
open("./Data/"+file_name, encoding='utf-8')) # taskID
|
||||
new_author_list = []
|
||||
i = 0
|
||||
for line in csv_reader:
|
||||
# print(line)
|
||||
if i > 0:
|
||||
new_author_list.append(line[0])
|
||||
i += 1
|
||||
# print(new_author_list)
|
||||
new_author_list = list(set(new_author_list)) # 去重
|
||||
|
||||
csv_reader = csv.reader(open("./author_list.csv", encoding='utf-8'))
|
||||
author_list = []
|
||||
for line in csv_reader:
|
||||
author_list.append(line[0])
|
||||
author_list = list(set(author_list)) # 去重
|
||||
|
||||
print("author_list:", author_list)
|
||||
print("new_author_list:", new_author_list)
|
||||
|
||||
real_new_author_list = list(
|
||||
set(new_author_list).difference(set(author_list)))
|
||||
print("real_new_author_list:", real_new_author_list)
|
||||
with open("author_list.csv", "a", encoding='utf-8', newline='') as csvfile:
|
||||
writer = csv.writer(csvfile)
|
||||
for row in real_new_author_list:
|
||||
writer.writerow([row])
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,202 @@
|
||||
# _*_coding:utf-8_*_
|
||||
from hashlib import new
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from multiprocessing import Process
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
import os
|
||||
import pickle
|
||||
import calendar
|
||||
import re
|
||||
from copy import deepcopy
|
||||
import requests
|
||||
import csv
|
||||
from commandline_config import Config
|
||||
from service_invoke import invokeService
|
||||
|
||||
|
||||
class TimeUtil(object):
|
||||
@classmethod
|
||||
def parse_timezone(cls, timezone):
|
||||
"""
|
||||
解析时区表示
|
||||
:param timezone: str eg: +8
|
||||
:return: dict{symbol, offset}
|
||||
"""
|
||||
result = re.match(r'(?P<symbol>[+-])(?P<offset>\d+)', timezone)
|
||||
symbol = result.groupdict()['symbol']
|
||||
offset = int(result.groupdict()['offset'])
|
||||
|
||||
return {
|
||||
'symbol': symbol,
|
||||
'offset': offset
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def convert_timezone(cls, dt, timezone="+0"):
|
||||
"""默认是utc时间，需要"""
|
||||
result = cls.parse_timezone(timezone)
|
||||
symbol = result['symbol']
|
||||
|
||||
offset = result['offset']
|
||||
|
||||
if symbol == '+':
|
||||
return dt + timedelta(hours=offset)
|
||||
elif symbol == '-':
|
||||
return dt - timedelta(hours=offset)
|
||||
else:
|
||||
raise Exception('dont parse timezone format')
|
||||
|
||||
|
||||
def generate_timestamp():
|
||||
current_GMT = time.gmtime()
|
||||
# ts stores timestamp
|
||||
ts = calendar.timegm(current_GMT)
|
||||
|
||||
current_time = datetime.utcnow()
|
||||
convert_now = TimeUtil.convert_timezone(current_time, '+8')
|
||||
print("current_time: " + str(convert_now))
|
||||
return str(convert_now)
|
||||
|
||||
|
||||
def main():
|
||||
# result = os.popen('python ServiceWrapper_ExecuteStage.py 38')
|
||||
# res = result.read()
|
||||
# for line in res.splitlines():
|
||||
# print("\n\n\n\nfinename:\n\n\n\n\n", line)
|
||||
config = {
|
||||
"pages": 5,
|
||||
"test": False,
|
||||
"test_pages": 3,
|
||||
}
|
||||
c = Config(config)
|
||||
print(c)
|
||||
csv_reader = csv.reader(open("./raw_data.csv", encoding='utf-8'))
|
||||
author_list = []
|
||||
for line in csv_reader:
|
||||
author_list.append(line[4])
|
||||
|
||||
csv_reader = csv.reader(open("./author_list.csv", encoding='utf-8'))
|
||||
keywords = []
|
||||
i = 0
|
||||
for line in csv_reader:
|
||||
if line[0] not in author_list:
|
||||
keywords.append(line[0])
|
||||
else:
|
||||
print("Will not append keyword %s", line[0])
|
||||
i += 1
|
||||
if c.test and i > c.test_pages * 100:
|
||||
break
|
||||
# print("author_list:", author_list)
|
||||
# exit(0)
|
||||
|
||||
urlList = ""
|
||||
i = 0
|
||||
|
||||
for keyword in keywords:
|
||||
url = "https://so.toutiao.com/search?dvpf=pc&source=input&keyword=%s&pd=user&action_type=search_subtab_switch&page_num=0&from=media&cur_tab_title=media\r\n" % keyword
|
||||
# print(url)
|
||||
urlList += url
|
||||
i += 1
|
||||
if c.test and i > c.test_pages:
|
||||
break
|
||||
print(urlList)
|
||||
# exit(0)
|
||||
# result = requests.post(
|
||||
# "http://servicewrapper.naibo.wang/backEnd/invokeService",
|
||||
# data={"id": 7, # serviceID
|
||||
# "paras": json.dumps({"urlList_0": urlList,
|
||||
# }),
|
||||
# })
|
||||
# descTaskID = int(result.text)
|
||||
descTaskID = invokeService(
|
||||
1, {"urlList_0": urlList})
|
||||
print("descTaskID: " + str(descTaskID))
|
||||
# exit(0)
|
||||
filename = generate_timestamp().replace(" ", "").replace(":", "-")
|
||||
print("filename:", filename)
|
||||
|
||||
command = 'python ServiceWrapper_ExecuteStage_local.py ' + \
|
||||
str(descTaskID) + ' ' + filename
|
||||
result = os.system(command)
|
||||
|
||||
# authorTaskID = 53
|
||||
file_name = "task_" + str(descTaskID) + "_" + filename + ".csv"
|
||||
# file_name = "task_53_2022-10-1723-35-40.881448.csv"
|
||||
print("file_name:", file_name)
|
||||
csv_reader = csv.reader(
|
||||
open("./Data/"+file_name, encoding='utf-8')) # taskID
|
||||
new_descTaskID = []
|
||||
i = 0
|
||||
for line in csv_reader:
|
||||
# print(line)
|
||||
if i > 0:
|
||||
new_descTaskID.append(line)
|
||||
i += 1
|
||||
# print(new_author_list)
|
||||
# new_descTaskID = list(set([tuple(t) for t in new_descTaskID]))
|
||||
# new_descTaskID = list(set(new_descTaskID)) # 去重
|
||||
|
||||
after_remove_duplicate = []
|
||||
for i in range(len(new_descTaskID)):
|
||||
try:
|
||||
if i > 0:
|
||||
if new_descTaskID[i][2] == new_descTaskID[i-1][2]:
|
||||
continue
|
||||
if new_descTaskID[i][2] != "":
|
||||
zan = new_descTaskID[i][1].split("获赞")[0]
|
||||
fans = new_descTaskID[i][1].split("粉丝")[0].split("获赞")[1]
|
||||
follow = new_descTaskID[i][1].split("关注")[0].split("粉丝")[1]
|
||||
after_remove_duplicate.append(
|
||||
[new_descTaskID[i][0], zan, fans, follow, new_descTaskID[i][2], new_descTaskID[i][3]])
|
||||
except:
|
||||
pass
|
||||
|
||||
print("after_remove_duplicate", after_remove_duplicate)
|
||||
|
||||
all_collected = []
|
||||
for author in after_remove_duplicate:
|
||||
all_collected.append(author[4])
|
||||
print("all_collected:", all_collected)
|
||||
|
||||
for keyword in keywords:
|
||||
if keyword not in all_collected:
|
||||
print("keyword not collected:", keyword)
|
||||
after_remove_duplicate.append(['', '', '', '', keyword, ''])
|
||||
|
||||
new_descTaskID = after_remove_duplicate
|
||||
|
||||
print("new_descTaskID:", new_descTaskID)
|
||||
|
||||
# for i in range(len(keywords)):
|
||||
# author_list[i] = [keywords[i]].extend(new_descTaskID[i])
|
||||
# for row in author_list:
|
||||
# print(row)
|
||||
|
||||
with open("raw_data.csv", "a", encoding='utf-8', newline='') as csvfile:
|
||||
writer = csv.writer(csvfile)
|
||||
for row in new_descTaskID:
|
||||
writer.writerow(row)
|
||||
|
||||
import xlwt
|
||||
|
||||
csv_reader = csv.reader(open("./raw_data.csv", encoding='utf-8'))
|
||||
all_data = []
|
||||
for line in csv_reader:
|
||||
all_data.append(line)
|
||||
|
||||
workbook = xlwt.Workbook()
|
||||
sheet = workbook.add_sheet("Sheet")
|
||||
|
||||
for i in range(len(all_data)):
|
||||
for j in range(len(all_data[i])):
|
||||
sheet.write(i, j, all_data[i][j])
|
||||
|
||||
workbook.save("all_data.xls")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
@@ -0,0 +1,7 @@
|
||||
import author_crawl
|
||||
import desc_crawl
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
author_crawl.main()
|
||||
desc_crawl.main()
|
||||
@@ -0,0 +1,6 @@
|
||||
import pandas as pd
|
||||
|
||||
if __name__ == "__main__":
|
||||
df = pd.read_excel('list.xlsx', "sheet2")
|
||||
data = df.values
|
||||
print(len(data), data[0])
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
电商实战运营
|
||||
|
Reference in New Issue
Block a user