Merge branch 'Geniusay:master' into master

.gitignore

@@ -1,13 +1,15 @@
 # Project exclude paths
-/console/target/
-/common/target/
-/HotModule/target/
-/FileModule/target/
-/BarrageModule/target/
-/SectionModule/target/
-/SectionWorkModule/target/
-/PublishModule/target/
-/LiveRecordModule/target/
+/chopperbot-console/target/
+/chopperbot-account/target/
+/chopperbot-creeper/target/
+/chopperbot-common/target/
+/chopperbot-hot/target/
+/chopperbot-file/target/
+/chopperbot-barrage/target/
+/chopperbot-section/target/
+/chopperbot-section-work/target/
+/chopperbot-publish/target/
+/chopperbot-live/target/
 /doc/docs/.vuepress/dist/
 /CreeperModule/target/
 /config/

chopperbot-common/pom.xml

@@ -79,5 +79,13 @@
             <groupId>org.xerial</groupId>
             <artifactId>sqlite-jdbc</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpcore</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+        </dependency>
     </dependencies>
 </project>

chopperbot-common/src/main/java/org/example/bean/live/DouyinLive.java

@@ -0,0 +1,23 @@
+package org.example.bean.live;
+
+import org.example.bean.Live;
+import org.example.constpool.ConstPool;
+
+/**
+ * @author dhx
+ * @date 2024/5/26 11:04
+ */
+public class DouyinLive  extends Live {
+
+
+
+    private String uid;
+
+
+    public DouyinLive(int watcherNum, String liveId, String liveName, String liver, String description,
+                     String roomCoverPic, String uid, String moduleId,String moduleName) {
+        super(watcherNum, liveId, liveName, liver, description, ConstPool.PLATFORM.DOUYIN.getName(),moduleId,moduleName);
+        setRoomPic(roomCoverPic);
+        this.uid = uid;
+    }
+}

chopperbot-common/src/main/java/org/example/constpool/ConstPool.java

@@ -34,7 +34,7 @@ public class ConstPool {
         DOUYU("douyu"),
         HUYA("huya"),
         BILIBILI("bilibili"),
-        DOUYING("douyin"),
+        DOUYIN("douyin"),
         TIKTOK("tiktok"),
         TWITCH("twitch");
         private final String name;
@@ -48,7 +48,7 @@ public class ConstPool {
 
     }
     public static final String DOUYU = "douyu";
-
+    public static final String DOUYIN = "douyin";
     public static final String HUYA = "huya";
     public static final String BILIBILI = "bilibili";
 

chopperbot-common/src/main/java/org/example/util/ByteDanceUtil.java

@@ -0,0 +1,32 @@
+package org.example.util;
+
+import org.springframework.http.HttpMethod;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author dhx
+ * @date 2024/5/26 13:16
+ */
+public class ByteDanceUtil {
+    public static String getTtwid( ) {
+        Map<String,String> douyinHeader = new HashMap<>();
+        douyinHeader.put("UserAgent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36");
+        douyinHeader.put("Origin","https://live.douyin.com");
+        douyinHeader.put("Referer","https://live.douyin.com");
+        douyinHeader.put("Upgrade-Insecure-Requests","1");
+        douyinHeader.put("Accept","*/*");
+        douyinHeader.put("Host","live.douyin.com");
+        douyinHeader.put("Connection","keep-alive");
+        Map<String,String> respHeaders = HttpClientUtil.getResponseHeaders("https://live.douyin.com/", HttpMethod.GET,"",douyinHeader);
+        String ttwid = RegexUtil.match(respHeaders.get("Set-Cookie"),"ttwid=([^;]+)")[1];
+        if(ttwid!=null){
+            return ttwid;
+        }
+        String ac_nonce = RegexUtil.match(respHeaders.get("Set-Cookie"),"__ac_nonce=([a-zA-Z0-9]+)")[1];
+        douyinHeader.put("Cookie",String.format("__ac_nonce=%s",ac_nonce));
+        respHeaders = HttpClientUtil.getResponseHeaders("https://live.douyin.com/", HttpMethod.GET,"",douyinHeader);
+        return RegexUtil.match(respHeaders.get("Set-Cookie"),"ttwid=([^;]+)")[0];
+    }
+}

chopperbot-common/src/main/java/org/example/util/HttpClientUtil.java

@@ -0,0 +1,178 @@
+package org.example.util;
+
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.*;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.springframework.http.HttpMethod;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * 简单的get,post请求工具类
+ * @author 燧枫
+ * @date 2023/5/16 19:24
+*/
+public class HttpClientUtil {
+    private static final CloseableHttpClient httpClient = HttpClients.createDefault();
+    private static final ResponseHandler responseHandler = new DefaultResponseHandler();
+    private static final HttpClientExecutor executor = new HttpClientExecutor(httpClient, responseHandler);
+
+    public static String get(String url) {
+        HttpUriRequest request = HttpRequestFactory.createRequest(HttpMethod.GET, url, null, null);
+        return executor.execute(request);
+    }
+
+    public static String get(String url, Map<String, String> headers) {
+        HttpUriRequest request = HttpRequestFactory.createRequest(HttpMethod.GET, url, null, headers);
+        return executor.execute(request);
+    }
+
+    public static String post(String url) {
+        HttpUriRequest request = HttpRequestFactory.createRequest(HttpMethod.POST, url, null, null);
+        return executor.execute(request);
+    }
+
+    public static String post(String url, String json) {
+        HttpUriRequest request = HttpRequestFactory.createRequest(HttpMethod.POST, url, json, null);
+        return executor.execute(request);
+    }
+
+    public static String post(String url, String json, Map<String, String> headers) {
+        HttpUriRequest request = HttpRequestFactory.createRequest(HttpMethod.POST, url, json, headers);
+        return executor.execute(request);
+    }
+
+    public static String put(String url) {
+        HttpUriRequest request = HttpRequestFactory.createRequest(HttpMethod.PUT, url, null, null);
+        return executor.execute(request);
+    }
+
+    public static String put(String url, String json) {
+        HttpUriRequest request = HttpRequestFactory.createRequest(HttpMethod.PUT, url, json, null);
+        return executor.execute(request);
+    }
+
+    public static String delete(String url) {
+        HttpUriRequest request = HttpRequestFactory.createRequest(HttpMethod.DELETE, url, null, null);
+        return executor.execute(request);
+    }
+
+    public static String delete(String url, Map<String, String> headers) {
+        HttpUriRequest request = HttpRequestFactory.createRequest(HttpMethod.DELETE, url, null, headers);
+        return executor.execute(request);
+    }
+
+    public static Map<String, String> getResponseHeaders(String url, HttpMethod method, String json, Map<String, String> headers) {
+        HttpUriRequest request = HttpRequestFactory.createRequest(method, url, json, headers);
+        return executor.executeForHeaders(request);
+    }
+}
+class HttpClientExecutor {
+    private final CloseableHttpClient httpClient;
+    private final ResponseHandler responseHandler;
+
+    public HttpClientExecutor(CloseableHttpClient httpClient, ResponseHandler responseHandler) {
+        this.httpClient = httpClient;
+        this.responseHandler = responseHandler;
+    }
+
+    public String execute(HttpUriRequest request) {
+        try (CloseableHttpResponse response = httpClient.execute(request)) {
+            return responseHandler.handleResponse(response);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+    public Map<String, String> executeForHeaders(HttpUriRequest request) {
+        try (CloseableHttpResponse response = httpClient.execute(request)) {
+            return responseHandler.handleResponseHeaders(response);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
+interface ResponseHandler {
+    String handleResponse(HttpResponse response) throws IOException;
+    Map<String, String> handleResponseHeaders(HttpResponse response) throws IOException;
+}
+
+class DefaultResponseHandler implements ResponseHandler {
+    @Override
+    public String handleResponse(HttpResponse response) throws IOException {
+        HttpEntity entity = response.getEntity();
+        if (entity != null) {
+            return EntityUtils.toString(entity, StandardCharsets.UTF_8);
+        } else {
+            throw new RuntimeException("Response entity is null");
+        }
+    }
+    public Map<String, String> handleResponseHeaders(HttpResponse response) {
+        Header[] headers = response.getAllHeaders();
+        Map<String, String> headersMap = new HashMap<>();
+        for (Header header : headers) {
+            headersMap.put(header.getName(), header.getValue());
+        }
+        return headersMap;
+    }
+}
+
+class HttpRequestFactory {
+    public static HttpUriRequest createRequest(HttpMethod method, String url, String json, Map<String, String> headers) {
+        switch (method) {
+            case GET:
+                return createGetRequest(url, headers);
+            case POST:
+                return createPostRequest(url, json, headers);
+            case PUT:
+                return createPutRequest(url, json, headers);
+            case DELETE:
+                return createDeleteRequest(url, headers);
+            default:
+                throw new IllegalArgumentException("Unsupported HTTP method: " + method);
+        }
+    }
+
+    private static HttpGet createGetRequest(String url, Map<String, String> headers) {
+        HttpGet request = new HttpGet(url);
+        addHeaders(request, headers);
+        return request;
+    }
+
+    private static HttpPost createPostRequest(String url, String json, Map<String, String> headers) {
+        HttpPost request = new HttpPost(url);
+        if(json!=null)request.setEntity(new StringEntity(json, ContentType.APPLICATION_JSON));
+        addHeaders(request, headers);
+        return request;
+    }
+
+    private static HttpPut createPutRequest(String url, String json, Map<String, String> headers) {
+        HttpPut request = new HttpPut(url);
+        if(json!=null)request.setEntity(new StringEntity(json, ContentType.APPLICATION_JSON));
+        addHeaders(request, headers);
+        return request;
+    }
+
+    private static HttpDelete createDeleteRequest(String url, Map<String, String> headers) {
+        HttpDelete request = new HttpDelete(url);
+        addHeaders(request, headers);
+        return request;
+    }
+
+    private static void addHeaders(HttpRequest request, Map<String, String> headers) {
+        if (headers != null && !headers.isEmpty()) {
+            for (Map.Entry<String, String> entry : headers.entrySet()) {
+                request.addHeader(entry.getKey(), entry.getValue());
+            }
+        }
+    }
+}

chopperbot-common/src/main/java/org/example/util/RegexUtil.java

@@ -0,0 +1,24 @@
+package org.example.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author dhx
+ * @date 2023/8/4 19:36
+ */
+public class RegexUtil {
+    public static String[] match(String input, String regex) {
+        List<String> matches = new ArrayList<>();
+        Matcher matcher = Pattern.compile(regex).matcher(input);
+        while (matcher.find()) {
+            matches.add(matcher.group());
+            for (int i = 1; i <= matcher.groupCount(); i++) {
+                matches.add(matcher.group(i));
+            }
+        }
+        return matches.toArray(new String[0]);
+    }
+}

chopperbot-creeper/src/main/java/org/example/config/CreeperConfigFile.java

@@ -25,7 +25,8 @@ public class CreeperConfigFile extends ConfigFile<Map<String,Object>> {
                 Map.of("taskCenter",new TaskCenterConfig(10,50,1000),
                         "spiderConfig",Map.of(
                                 ConstPool.PLATFORM.DOUYU.getName(),new SpiderConfig(),
-                                ConstPool.PLATFORM.BILIBILI.getName(),new SpiderConfig()
+                                ConstPool.PLATFORM.BILIBILI.getName(),new SpiderConfig(),
+                                ConstPool.PLATFORM.DOUYIN.getName(),new SpiderConfig()
                         )
                 ));
     }

chopperbot-hot/src/main/java/org/example/core/creeper/builder/DouyinLiverCheckerBuilder.java

@@ -0,0 +1,24 @@
+package org.example.core.creeper.builder;
+
+import org.example.bean.FocusLiver;
+import org.example.bean.live.DouyinLive;
+import org.example.core.creeper.loadconfig.DouyinLiverCheckerConfig;
+import org.example.core.manager.CommonLoadConfigBuilder;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author dhx
+ * @date 2024/5/26 11:07
+ */
+@Component
+public class DouyinLiverCheckerBuilder extends CommonLoadConfigBuilder<DouyinLiverCheckerConfig> {
+    @Override
+    public DouyinLiverCheckerConfig build(Object obj) {
+        if(obj instanceof FocusLiver){
+            return new DouyinLiverCheckerConfig(((FocusLiver) obj).getRoomId());
+        }else if(obj instanceof DouyinLive){
+            return new DouyinLiverCheckerConfig(((DouyinLive) obj).getLiveId());
+        }
+        return null;
+    }
+}

chopperbot-hot/src/main/java/org/example/core/creeper/loadconfig/DouyinLiverCheckerConfig.java

@@ -0,0 +1,48 @@
+package org.example.core.creeper.loadconfig;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import lombok.Data;
+import org.example.constpool.ConstGroup;
+import org.example.constpool.ConstPool;
+import org.example.core.creeper.loadtask.DouyinLiverCheckerLoadTask;
+import org.example.core.loadconfig.LoadConfig;
+import org.example.core.manager.Creeper;
+import org.example.util.ByteDanceUtil;
+import org.example.util.HttpClientUtil;
+import org.example.util.RegexUtil;
+import org.springframework.http.HttpMethod;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author dhx
+ * @date 2024/5/26 10:56
+ */
+@Data
+@Creeper(creeperName = "抖音直播检测爬虫",
+        loadTask = DouyinLiverCheckerLoadTask.class,
+        creeperDescription = "用于检测抖音主播是否开播，并且获取直播详细信息",
+        priority = 10,
+        group = ConstGroup.LIVER_CHECKER,
+        platform = ConstPool.DOUYIN
+)
+public class DouyinLiverCheckerConfig extends LoadConfig {
+    private String roomId;
+
+    public DouyinLiverCheckerConfig(String roomId) {
+        setRoomId(roomId);
+        Map<String,String> dyHeader = new HashMap<>();
+        dyHeader.put("Cookie",String.format("ttwid=%s;", ByteDanceUtil.getTtwid()));
+        dyHeader.put("rid",roomId);
+        setHeader(dyHeader);
+        setUrl("https://live.douyin.com/webcast/room/web/enter/?aid=6383&app_name=douyin_web&live_id=1&device_platform=web&language=zh-CN&enter_from=web_live&cookie_enabled=true&screen_width=1728&screen_height=1117&browser_language=zh-CN&browser_platform=MacIntel&browser_name=Chrome&browser_version=116.0.0.0&web_rid="+roomId);
+    }
+
+
+    @Override
+    public String getTaskId() {
+        return super.getTaskId()+"_"+roomId;
+    }
+}

chopperbot-hot/src/main/java/org/example/core/creeper/loadtask/DouyinLiverCheckerLoadTask.java

@@ -0,0 +1,45 @@
+package org.example.core.creeper.loadtask;
+
+import org.example.bean.live.DouyinLive;
+import org.example.constpool.ConstPool;
+import org.example.core.creeper.loadconfig.DouyinLiverCheckerConfig;
+import org.example.core.creeper.processor.DouyinLiverCheckerProcessor;
+import org.example.core.factory.SpiderFactory;
+import org.example.core.loadtask.WebMagicLoadTask;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.Request;
+
+/**
+ * @author dhx
+ * @date 2024/5/26 11:08
+ */
+public class DouyinLiverCheckerLoadTask extends WebMagicLoadTask<DouyinLive> {
+
+    public DouyinLiverCheckerLoadTask(DouyinLiverCheckerConfig loadConfig) {
+        super(loadConfig);
+    }
+
+    @Override
+    public DouyinLive start() {
+        DouyinLive live = null;
+        Request request = new Request(loadConfig.getUrl());
+        request.addHeader("rid",loadConfig.getHeader().get("rid"));
+        request.addHeader("Cookie",loadConfig.getHeader().get("Cookie"));
+        Spider spider = SpiderFactory.buildSpider(
+                ConstPool.PLATFORM.DOUYIN.getName(),
+                new DouyinLiverCheckerProcessor(),
+                request
+        );
+        try {
+            live = getData(spider,loadConfig.getUrl());
+        }catch (Exception e){
+            return null;
+        }
+        return live;
+    }
+
+    @Override
+    public void end() {
+
+    }
+}

chopperbot-hot/src/main/java/org/example/core/creeper/processor/DouyinLiverCheckerProcessor.java

@@ -0,0 +1,47 @@
+package org.example.core.creeper.processor;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import org.example.bean.HotModule;
+import org.example.bean.live.DouyinLive;
+import org.example.constpool.ConstPool;
+import org.example.core.HotModuleDataCenter;
+import org.example.core.processor.AbstractProcessor;
+import org.example.util.TimeUtil;
+import us.codecraft.webmagic.Page;
+
+/**
+ * @author dhx
+ * @date 2024/5/26 11:40
+ */
+public class DouyinLiverCheckerProcessor extends AbstractProcessor {
+
+    @Override
+    public void process(Page page) {
+        System.out.println(page.getHeaders());
+        JSONObject live = JSON.parseObject(page.getRawText());
+        DouyinLive douyinLive = null;
+        if (live!=null) {
+            JSONObject liveInfo = live.getJSONObject("data").getJSONArray("data").getJSONObject(0);
+            JSONObject userInfo = live.getJSONObject("data").getJSONObject("user");
+            String room_name = liveInfo.getString("title");
+            String room_id = page.getRequest().getHeaders().get("rid");
+            String pic_url = liveInfo.getJSONObject("cover").getJSONArray("url_list").getString(0);
+            String owner_uid = userInfo.getString("sec_uid");
+            String nickname = userInfo.getString("nickname");
+            Long show_time = live.getJSONObject("extra").getLong("now");
+            String description = "";
+            Integer moduleId = Integer.valueOf(live.getJSONObject("data").getJSONObject("partition_road_map").getJSONObject("partition").getString("id_str"));
+            String moduleName = live.getJSONObject("data").getJSONObject("partition_road_map").getJSONObject("partition").getString("title");
+            try {
+                HotModule module = HotModuleDataCenter.DataCenter().getModuleById(ConstPool.DOUYIN,String.valueOf(moduleId));
+                moduleName = module==null?"未知模块":module.getTagName();
+            }catch (Exception e){
+                throw new RuntimeException(e);
+            }
+            douyinLive = new DouyinLive(0,room_id,room_name,nickname,description,pic_url,owner_uid,String.valueOf(moduleId),moduleName);
+            douyinLive.setShowTime(TimeUtil.getFormatDate(show_time));
+        }
+        page.putField("data",douyinLive);
+    }
+}

chopperbot-live/src/main/java/org/example/core/component/SimpleRequest.java

@@ -6,7 +6,7 @@ import com.alibaba.fastjson.JSONObject;
 import org.example.pojo.record.RecordDayEntry;
 import org.example.pojo.record.RecordEntry;
 import org.example.pojo.record.RecordList;
-import org.example.utils.HttpClientUtil;
+import org.example.util.HttpClientUtil;
 
 import java.util.ArrayList;
 import java.util.List;

chopperbot-live/src/main/java/org/example/core/creeper/builder/DouyinLiveLoadConfigBuilder.java

@@ -0,0 +1,35 @@
+package org.example.core.creeper.builder;
+
+import org.example.bean.live.DouyinLive;
+import org.example.constpool.ConstPool;
+import org.example.constpool.FileNameBuilder;
+import org.example.core.creeper.loadconfig.DouyinLiveOnlineConfig;
+import org.example.core.manager.CommonLoadConfigBuilder;
+import org.example.pool.LiveModuleConstPool;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author dhx
+ * @date 2024/5/26 11:03
+ */
+@Component
+public class DouyinLiveLoadConfigBuilder  extends CommonLoadConfigBuilder<DouyinLiveOnlineConfig> {
+
+    @Override
+    public DouyinLiveOnlineConfig build(Object obj) {
+        if(obj instanceof DouyinLive){
+            String liveId = ((DouyinLive) obj).getLiveId();
+            String liver = ((DouyinLive) obj).getLiveName();
+            String path = LiveModuleConstPool.getPlatformLiveSavePath(ConstPool.PLATFORM.DOUYIN);
+            String showTime = ((DouyinLive) obj).getShowTime();
+            DouyinLiveOnlineConfig douyinLiveOnlineConfig = new DouyinLiveOnlineConfig(liveId, path, null, false);
+            douyinLiveOnlineConfig.setShowTime(showTime);
+            douyinLiveOnlineConfig.setVideoName(FileNameBuilder.buildVideoFileNameNoSuffix(liver,douyinLiveOnlineConfig.getStartTime()));
+            douyinLiveOnlineConfig.setShowDownloadTable(true);
+            douyinLiveOnlineConfig.setLiverName(liver);
+            douyinLiveOnlineConfig.setRoomName(((DouyinLive) obj).getLiveName());
+            return douyinLiveOnlineConfig;
+        }
+        return null;
+    }
+}

chopperbot-live/src/main/java/org/example/core/creeper/loadconfig/DouyinLiveOnlineConfig.java

@@ -0,0 +1,55 @@
+package org.example.core.creeper.loadconfig;
+
+import lombok.Data;
+import org.example.constpool.ConstGroup;
+import org.example.constpool.ConstPool;
+import org.example.core.creeper.loadtask.DouyinLiveOnlineLoadTask;
+import org.example.core.manager.Creeper;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author dhx
+ * @date 2024/5/19 15:33
+ */
+@Data
+@Creeper(creeperName = "抖音直播爬虫",
+        loadTask = DouyinLiveOnlineLoadTask.class,
+        creeperDescription = "抖音直播爬取(包含监控器)",
+        priority = 10,
+        group = ConstGroup.LIVE_ONLINE,
+        platform = ConstPool.DOUYIN
+)
+public class DouyinLiveOnlineConfig extends LoadLiveConfig {
+    public DouyinLiveOnlineConfig(String roomId, String videoPath, String videoName,int clarity) {
+        super(roomId, videoPath, videoName, false);
+        this.platform = ConstPool.PLATFORM.DOUYIN.getName();
+        setHeader();
+    }
+
+    public DouyinLiveOnlineConfig(String roomId, String videoPath, String videoName,boolean convertToMp4,int clarity) {
+        super(roomId, videoPath, videoName, convertToMp4);
+        this.platform = ConstPool.PLATFORM.DOUYIN.getName();
+        setHeader();
+    }
+
+    public DouyinLiveOnlineConfig(String roomId, String videoPath, String videoName,boolean convertToMp4) {
+        super(roomId, videoPath, videoName, convertToMp4);
+        this.platform = ConstPool.PLATFORM.DOUYIN.getName();
+        setHeader();
+    }
+
+    private void setHeader(){
+        this.url = "https://live.douyin.com/webcast/room/web/enter/?aid=6383&app_name=douyin_web&live_id=1&device_platform=web&language=zh-CN&enter_from=web_live&cookie_enabled=true&screen_width=1728&screen_height=1117&browser_language=zh-CN&browser_platform=MacIntel&browser_name=Chrome&browser_version=116.0.0.0&web_rid=";
+        this.UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36";
+        this.Origin = "https://live.douyin.com";
+        this.Referer = "https://live.douyin.com";
+        Map<String,String> dyHeader = new HashMap<>();
+        dyHeader.put("Upgrade-Insecure-Requests","1");
+        dyHeader.put("Accept","*/*");
+        dyHeader.put("Host","live.douyin.com");
+        dyHeader.put("Connection","keep-alive");
+        this.header = dyHeader;
+    }
+}

chopperbot-live/src/main/java/org/example/core/creeper/loadtask/DouyinLiveOnlineLoadTask.java

@@ -0,0 +1,26 @@
+package org.example.core.creeper.loadtask;
+
+import org.example.core.VideoTaskMonitor;
+import org.example.core.creeper.loadconfig.DouyinLiveOnlineConfig;
+import org.example.core.taskmonitor.Monitor;
+
+/**
+ * @author dhx
+ * @date 2024/5/26 10:45
+ */
+@Monitor(clazz = VideoTaskMonitor.class)
+public class DouyinLiveOnlineLoadTask extends LiveOnlineLoadTask{
+    public DouyinLiveOnlineLoadTask(DouyinLiveOnlineConfig douyinLiveOnlineConfig) {
+        super(douyinLiveOnlineConfig);
+    }
+
+    @Override
+    public String start() {
+        return this.start(logger,(DouyinLiveOnlineConfig)loadConfig);
+    }
+
+    @Override
+    public void end() {
+
+    }
+}

chopperbot-live/src/main/java/org/example/core/creeper/loadtask/DouyuRecordListLoadTask.java

@@ -10,7 +10,7 @@ import org.example.core.loadtask.LoadTask;
 import org.example.pojo.record.RecordDayEntry;
 import org.example.pojo.record.RecordEntry;
 import org.example.pojo.record.RecordList;
-import org.example.utils.HttpClientUtil;
+import org.example.util.HttpClientUtil;
 
 import java.time.LocalDate;
 import java.util.ArrayList;

chopperbot-live/src/main/java/org/example/core/factory/LiveTaskFactory.java

@@ -1,12 +1,10 @@
 package org.example.core.factory;
 
-import org.example.core.creeper.loadconfig.BilibiliLiveOnlineConfig;
-import org.example.core.creeper.loadconfig.DouyuLiveOnlineConfig;
-import org.example.core.creeper.loadconfig.HuyaLiveOnlineConfig;
-import org.example.core.creeper.loadconfig.LoadLiveConfig;
+import org.example.core.creeper.loadconfig.*;
 import org.example.core.parser.PlatformVideoUrlParser;
 import org.example.core.parser.impl.BilibiliFlvUrlParser;
 import org.example.core.component.LiveStreamTask;
+import org.example.core.parser.impl.DouyinFlvUrlParser;
 import org.example.core.parser.impl.DouyuFlvUrlParser;
 import org.example.core.parser.impl.HuyaFlvUrlParser;
 
@@ -23,7 +21,8 @@ public class LiveTaskFactory {
     private final Map<Class<? extends LoadLiveConfig>, PlatformVideoUrlParser> parserMap = Map.of(
             BilibiliLiveOnlineConfig.class,new BilibiliFlvUrlParser(),
             DouyuLiveOnlineConfig.class, new DouyuFlvUrlParser(),
-            HuyaLiveOnlineConfig.class, new HuyaFlvUrlParser()
+            HuyaLiveOnlineConfig.class, new HuyaFlvUrlParser(),
+            DouyinLiveOnlineConfig.class, new DouyinFlvUrlParser()
     );
 
     public LiveTaskFactory() {

chopperbot-live/src/main/java/org/example/core/parser/impl/BilibiliFlvUrlParser.java

@@ -2,7 +2,7 @@ package org.example.core.parser.impl;
 
 import org.example.core.creeper.loadconfig.BilibiliLiveOnlineConfig;
 import org.example.core.parser.PlatformVideoUrlParser;
-import org.example.utils.HttpClientUtil;
+import org.example.util.HttpClientUtil;
 import org.json.JSONArray;
 import org.json.JSONObject;
 

chopperbot-live/src/main/java/org/example/core/parser/impl/DouyinFlvUrlParser.java

@@ -0,0 +1,34 @@
+package org.example.core.parser.impl;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import org.example.core.creeper.loadconfig.DouyinLiveOnlineConfig;
+import org.example.core.parser.PlatformVideoUrlParser;
+import org.example.util.HttpClientUtil;
+import org.example.util.ByteDanceUtil;
+
+import java.util.Map;
+
+/**
+ * @author dhx
+ * @date 2024/5/19 15:32
+ */
+public class DouyinFlvUrlParser implements PlatformVideoUrlParser<DouyinLiveOnlineConfig> {
+
+    @Override
+    public String getUrl(DouyinLiveOnlineConfig loadConfig) throws Exception {
+        String url = loadConfig.getUrl();
+        String roomId = loadConfig.getRoomId();
+        Map<String,String> header = loadConfig.getHeader();
+        try {
+            header.put("Cookie",String.format("ttwid=%s;",ByteDanceUtil.getTtwid()));
+            String resp = HttpClientUtil.get(url+roomId,header);
+            JSONObject jsonObject = JSON.parseObject(resp);
+            if(jsonObject==null||jsonObject.getJSONObject("data")==null)return null;
+            return (String) jsonObject.getJSONObject("data").getJSONArray("data").getJSONObject(0).getJSONObject("stream_url").getJSONObject("flv_pull_url").get("FULL_HD1");
+        } catch (Exception e){
+            e.printStackTrace();
+            return null;
+        }
+    }
+}

chopperbot-live/src/main/java/org/example/core/parser/impl/DouyuFlvUrlParser.java

@@ -3,8 +3,8 @@ package org.example.core.parser.impl;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.example.core.creeper.loadconfig.DouyuLiveOnlineConfig;
 import org.example.core.parser.PlatformVideoUrlParser;
-import org.example.utils.HttpClientUtil;
-import org.example.utils.RegexUtil;
+import org.example.util.RegexUtil;
+import org.example.util.HttpClientUtil;
 import org.json.JSONObject;
 
 import javax.script.Invocable;

chopperbot-live/src/main/java/org/example/core/parser/impl/HuyaFlvUrlParser.java

@@ -10,8 +10,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.example.core.creeper.loadconfig.HuyaLiveOnlineConfig;
 import org.example.core.parser.PlatformVideoUrlParser;
-import org.example.utils.HttpClientUtil;
-import org.example.utils.RegexUtil;
+import org.example.util.HttpClientUtil;
 import org.json.JSONArray;
 import org.json.JSONObject;