Merge remote-tracking branch 'origin/master'

# Conflicts:
#	database.db
This commit is contained in:
welsir
2023-10-18 09:44:27 +08:00
13 changed files with 417 additions and 132 deletions
@@ -0,0 +1,40 @@
package org.example.core.analysis;
import lombok.Data;
import org.example.bean.Barrage;
import org.example.pojo.AnalysisScheme;
import org.jdom2.output.EscapeStrategy;
import java.util.List;
/**
* @Date 2023/10/16
* @Author xiaochun
*/
@Data
public class AnalysisSchemeBuilder {
private AnalysisScheme scheme;
private List<Barrage> barrages;
public AnalysisSchemeBuilder labels(String msg){
scheme.setLabels(msg);
return this;
}
public AnalysisSchemeBuilder system(String msg){
scheme.setSystem(msg);
return this;
}
public AnalysisSchemeBuilder comment(String msg){
scheme.setComment(msg);
return this;
}
public AnalysisSchemeBuilder barrages(List<Barrage> barrages){
this.barrages = barrages;
return this;
}
}
@@ -0,0 +1,87 @@
package org.example.core.analysis;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import lombok.Data;
import org.example.bean.Barrage;
import org.example.core.gpt.ChatGPTMsgBuilder;
import org.example.core.gpt.ChatGPTPlugin;
import org.example.mapper.AnalysisSchemeMapper;
import org.example.plugin.SpringBootPlugin;
import org.example.pojo.AnalysisScheme;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* @Date 2023/10/16
* @Author xiaochun
*/
@Data
@Component
public class EmotionAnalysisPlugin extends SpringBootPlugin {
@Resource
AnalysisSchemeMapper mapper;
@Resource
ChatGPTPlugin chatGPTPlugin;
String barrage;
AnalysisScheme scheme;
@Override
public boolean init(){
try {
chooseScheme();
} catch (Exception e){
throw new RuntimeException("init analysis failed!pleas check or try again!");
}
return super.init();
}
private void chooseScheme(){
List<AnalysisScheme> schemes = mapper.selectList(new QueryWrapper<>());
if(schemes == null || schemes.isEmpty()) throw new RuntimeException("invaild anlysis scheme!please set scheme!");
scheme = schemes.get(0);
}
public String analysis(AnalysisSchemeBuilder analysisSchemeBuilder){
return analysis(analysisSchemeBuilder.getBarrages(), analysisSchemeBuilder.getScheme());
}
public String analysis(List<Barrage> barrages, AnalysisScheme scheme){
if(scheme != null) this.scheme = scheme;
return analysis(barrages);
}
public String analysis(List<Barrage> barrages){
try {
barrage = List.of(barrages.stream().map(Barrage::getContent).collect(Collectors.toList())).toString();
ChatGPTMsgBuilder builder = new ChatGPTMsgBuilder().model(chatGPTPlugin.getKey().getModel())
.system(this.scheme.getSystem())
.user("弹幕:" + barrage)
.stream(false);
JSONObject object = chatGPTPlugin.reqGPT(builder);
// System.out.println(object.getJSONArray("choices").getJSONObject(0).getJSONObject("message").getString("content"));
Pattern pattern = Pattern.compile("\\[(.*?)]");
Matcher matcher = pattern.matcher(object.getJSONArray("choices").getJSONObject(0).getJSONObject("message").getString("content"));
if (matcher.find()) return matcher.group(1);
} catch (Exception e){
throw new RuntimeException("analysis failed!please check or try again!");
}
return "";
}
}
@@ -0,0 +1,23 @@
package org.example.init;
import org.example.constpool.ModuleName;
import org.example.constpool.PluginName;
import org.example.core.analysis.EmotionAnalysisPlugin;
import org.example.core.gpt.ChatGPTPlugin;
import org.example.plugin.annotation.Plugin;
import org.springframework.stereotype.Component;
/**
* @Date 2023/10/16
* @Author xiaochun
*/
@Plugin(moduleName = ModuleName.ACCOUNT,
pluginName = PluginName.EMOTION_ANALYSIS,
pluginName_CN = "情感分析插件",
needPlugin = {PluginName.CHAT_GPT},
pluginClass= EmotionAnalysisPlugin.class,
springBootPlugin = true
)
@Component
public class EmotionAnalysisPluginMachine extends SpringPlugInitMachine{
}
@@ -0,0 +1,11 @@
package org.example.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.example.pojo.AnalysisScheme;
/**
* @Date 2023/10/16
* @Author xiaochun
*/
public interface AnalysisSchemeMapper extends BaseMapper<AnalysisScheme> {
}
@@ -0,0 +1,30 @@
package org.example.pojo;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* @Date 2023/10/16
* @Author xiaochun
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
@TableName("analysis_scheme")
public class AnalysisScheme {
@TableId(value = "id")
private Long id;
private String labels;
private String system;
private String comment;
public String getSystem(){
return system + " 标签:[" + labels + "]";
}
}
@@ -1,51 +0,0 @@
package org.example.utils;
import com.alibaba.fastjson.JSONObject;
import lombok.AllArgsConstructor;
import lombok.Data;
import netscape.javascript.JSObject;
import okhttp3.Headers;
import okhttp3.MediaType;
import okhttp3.RequestBody;
import org.example.core.config.GPTConfig;
import java.util.List;
import java.util.Map;
/**
* @Date 2023/10/13
* @Author xiaochun
*/
public class GPTUtil {
// public static RequestBody getRequestBodyJSON(String[] labels){
// StringBuilder label = new StringBuilder();
// label.append("标签:[").append(labels[0]);
// for (int i = 1;i < labels.length; i++){
// label.append(",").append(labels[i]);
// }
// label.append("]");
// String videoInfo = "弹幕:帅,这操作我要学一年,太6了";
// String systemInfo = "请你作为一个专门看直播的观众,对下列的观众发送的弹幕内容进行分析,然后根据弹幕内容返回从以下几个标签返回给我最合适的一个标签来形容这段内容标签:[搞笑,秀操作,破防,泪目]";
//
// Map<String, Object> message = Map.of(
// "messages", List.of(new Role("system", "???"), new Role("user", "hello")),
// "ai", "gpt-3.5-turbo-16k-0613",
// "stream", false
// );
// String requestBodyString = JSONObject.toJSONString(message);
// RequestBody requestBody = RequestBody.create(requestBodyString, MediaType.parse("application/json"));
// return requestBody;
// }
// public static Headers getHeaders(String key){
//
// }
//
// @Data
// @AllArgsConstructor
// static class Role{
// private String role;
// private String content;
// }
}
@@ -44,6 +44,8 @@ public class PluginName {
public static final String CHAT_GPT = "ChatGPT";
public static final String EMOTION_ANALYSIS = "EmotionAnalysis";
public static final String LIVE_CONFIG_PLUGIN= "LiveConfig";
@@ -1,15 +1,25 @@
// 参考:https://github.com/rain-dl/real-url-proxy-server/blob/master/huya.py
package org.example.core.parser.impl;
import cn.hutool.json.JSONException;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.example.core.creeper.loadconfig.HuyaLiveOnlineConfig;
import org.example.core.parser.PlatformVideoUrlParser;
import org.example.utils.HttpClientUtil;
import org.example.utils.RegexUtil;
import org.json.JSONArray;
import org.json.JSONObject;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static cn.hutool.crypto.SecureUtil.md5;
@@ -18,77 +28,173 @@ import static cn.hutool.crypto.SecureUtil.md5;
* @date 2023/10/9 20:04
*/
public class HuyaFlvUrlParser implements PlatformVideoUrlParser<HuyaLiveOnlineConfig> {
public static String live(String e){
String[] parts = e.split("\\?");
String i = parts[0];
String b = parts[1];
private String room_id;
private String user_id;
private int mode;
private Map<String,String> header;
private Map<String, Map<String, String>> live_url_infos = new HashMap<>();
String[] r = i.split("/");
String s = r[r.length - 1].replaceAll("\\.(flv|m3u8)", "");
String[] c = b.split("&", 4);
c = Arrays.stream(c).filter(item -> !item.isEmpty()).toArray(String[]::new);
private Map<String, String> decodeLiveUrlInfo(String srcAntiCode) {
srcAntiCode = StringEscapeUtils.unescapeHtml4(srcAntiCode);
String[] c = srcAntiCode.split("&");
Map<String, String> n = new HashMap<>();
for (String item : c) {
String[] keyValue = item.split("=");
if (keyValue.length == 2) {
n.put(keyValue[0], keyValue[1]);
for (String i : c) {
if (!i.isEmpty()) {
String[] parts = i.split("=");
if (parts.length == 2) {
n.put(parts[0], parts[1]);
}
}
}
String fm = java.net.URLDecoder.decode(n.get("fm"));
byte[] fmBytes = Base64.getDecoder().decode(fm);
String u = new String(fmBytes, java.nio.charset.StandardCharsets.UTF_8);
String[] uParts = u.split("_");
String p = uParts[0];
String f = String.valueOf(System.currentTimeMillis() * 10000);
String l = n.get("wsTime");
String t = "0";
String h = p + "_" + t + "_" + s + "_" + f + "_" + l;
String m = md5(h);
String y = c[c.length - 1];
String url = i + "?wsSecret=" + m + "&wsTime=" + l + "&u=" + t + "&seqid=" + f + "&" + y;
System.out.println(url); // 打印生成的URL
return url;
byte[] decodedBytes = Base64.getDecoder().decode(fm);
String u = new String(decodedBytes);
Map<String, String> liveUrlInfo = new HashMap<>();
liveUrlInfo.put("hash_prefix", u.split("_")[0]);
liveUrlInfo.put("uuid", n.get("uuid"));
liveUrlInfo.put("ctype", n.get("ctype"));
liveUrlInfo.put("txyp", n.get("txyp"));
liveUrlInfo.put("fs", n.get("fs"));
liveUrlInfo.put("t", n.get("t"));
return liveUrlInfo;
}
@Override
public String getUrl(HuyaLiveOnlineConfig huyaLiveOnlineConfig) throws Exception {
String roomId = huyaLiveOnlineConfig.getRoomId();
Map<String,String> header = new HashMap<>();
header.put("Content-Type","application/x-www-form-urlencoded");
header.put("User-Agent","Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Mobile Safari/537.36 ");
try{
String resp = HttpClientUtil.get("https://www.huya.com/"+roomId,header);
String[] res = RegexUtil.match(resp,"\"liveLineUrl\":\"([\\s\\S]*?)\",");
String liveLineUrl = "";
if(res.length!=0){
liveLineUrl = res[0];
private void clearLiveUrlInfos() {
live_url_infos.clear();
}
private void updateLiveUrlInfo() {
try {
if (mode == 0) {
String room_url = "https://m.huya.com/" + room_id;
String responseText = HttpClientUtil.get(room_url,header);
Pattern pattern = Pattern.compile("\"liveLineUrl\":\"([^\"]*?)\"");
Matcher matcher = pattern.matcher(responseText);
if (matcher.find()) {
String livelineurl_base64 = matcher.group(1);
String livelineurl;
try {
livelineurl = new String(Base64.getDecoder().decode(livelineurl_base64), StandardCharsets.UTF_8);
} catch (Exception e) {
livelineurl = livelineurl_base64;
}
if (!livelineurl.contains("replay")) {
String[] parts = livelineurl.split("\\?");
if (parts.length == 2) {
String url = parts[0];
String anti_code = parts[1];
Map<String, String> live_url_info = new HashMap<>();
live_url_info.put("stream_name", url.substring(url.lastIndexOf('/') + 1).replace(".flv", "").replace(".m3u8", ""));
live_url_info.put("base_url", "http:" + url.substring(0, url.lastIndexOf('/' + live_url_info.get("stream_name"))));
live_url_info.put("hls_url", "http:" + url);
Map<String, String> decodedInfo = decodeLiveUrlInfo(anti_code);
live_url_info.putAll(decodedInfo);
live_url_infos.put("TX", live_url_info);
}
}
}
} else if (mode == 1) {
String room_url = "https://www.huya.com/" + room_id;
Map<String,String> header2 = new HashMap<>();
header2.put("Content-Type", "application/x-www-form-urlencoded");
header2.put("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
String responseText = HttpClientUtil.get(room_url,header2);
String streamInfo = null;
Pattern streamPattern = Pattern.compile("stream: ([^\\n]*?)\\n");
Matcher streamMatcher = streamPattern.matcher(responseText);
if (streamMatcher.find()) {
streamInfo = streamMatcher.group(1);
} else {
Pattern base64Pattern = Pattern.compile("\"stream\": \"([^\"]*?)\"");
Matcher base64Matcher = base64Pattern.matcher(responseText);
if (base64Matcher.find()) {
String liveDataBase64 = base64Matcher.group(1);
streamInfo = new String(Base64.getDecoder().decode(liveDataBase64), StandardCharsets.UTF_8);
}
}
if (streamInfo != null) {
JSONObject liveData = new JSONObject(streamInfo);
JSONArray streamInfoList = liveData.getJSONArray("data").getJSONObject(0).getJSONArray("gameStreamInfoList");
for (int i = 0; i < streamInfoList.length(); i++) {
JSONObject streamInfoObj = streamInfoList.getJSONObject(i);
Map<String, String> live_url_info = new HashMap<>();
String sCdnType = streamInfoObj.getString("sCdnType");
live_url_info.put("stream_name", streamInfoObj.getString("sStreamName"));
live_url_info.put("base_url", streamInfoObj.getString("sHlsUrl"));
live_url_info.put("hls_url", streamInfoObj.getString("sHlsUrl") + "/" + streamInfoObj.getString("sStreamName") + "." + streamInfoObj.getString("sHlsUrlSuffix"));
String sHlsAntiCode = streamInfoObj.getString("sHlsAntiCode");
Map<String, String> decodedInfo = decodeLiveUrlInfo(sHlsAntiCode);
live_url_info.putAll(decodedInfo);
live_url_infos.put(sCdnType, live_url_info);
}
}
} else if (mode == 2) {
String room_url = "https://mp.huya.com/cache.php?m=Live&do=profileRoom&roomid=" + room_id;
CloseableHttpClient httpClient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(room_url);
httpGet.setHeader("Content-Type", "application/x-www-form-urlencoded");
httpGet.setHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
CloseableHttpResponse response = httpClient.execute(httpGet);
String responseText = org.apache.commons.io.IOUtils.toString(response.getEntity().getContent(), "UTF-8");
JSONObject liveData = new JSONObject(responseText);
if (liveData.has("data") && liveData.getJSONObject("data").has("stream") && liveData.getJSONObject("data").getJSONObject("stream").has("baseSteamInfoList")) {
JSONArray streamInfoList = liveData.getJSONObject("data").getJSONObject("stream").getJSONArray("baseSteamInfoList");
for (int i = 0; i < streamInfoList.length(); i++) {
JSONObject streamInfoObj = streamInfoList.getJSONObject(i);
Map<String, String> live_url_info = new HashMap<>();
String sCdnType = streamInfoObj.getString("sCdnType");
live_url_info.put("stream_name", streamInfoObj.getString("sStreamName"));
live_url_info.put("base_url", streamInfoObj.getString("sHlsUrl"));
live_url_info.put("hls_url", streamInfoObj.getString("sHlsUrl") + "/" + streamInfoObj.getString("sStreamName") + "." + streamInfoObj.getString("sHlsUrlSuffix"));
String sHlsAntiCode = streamInfoObj.getString("sHlsAntiCode");
Map<String, String> decodedInfo = decodeLiveUrlInfo(sHlsAntiCode);
live_url_info.putAll(decodedInfo);
live_url_infos.put(sCdnType, live_url_info);
}
}
}
String liveline = "";
if(!liveLineUrl.isEmpty()){
liveLineUrl = liveLineUrl.substring(liveLineUrl.indexOf(":")+2,liveLineUrl.indexOf(",")-1);
liveline = new String(Base64.getDecoder().decode(liveLineUrl), StandardCharsets.UTF_8);
}
if(!liveline.isEmpty()){
return liveline;
}
else{
liveline = live(liveline);
liveline = "https:" + liveline.replace("hls","flv").replace("m3u8","flv");
return liveline;
}
}catch (Exception e){
} catch (IOException | JSONException e) {
e.printStackTrace();
}
return null;
}
public List<String> getRealUrl() {
String ratio = "";
List<String> urls = new ArrayList<>();
String seqid = String.valueOf((System.currentTimeMillis() / 1000) + Long.parseLong(user_id));
String wsTime = Long.toHexString(System.currentTimeMillis() / 1000 + 3600);
for (Map<String, String> live_url_info : live_url_infos.values()) {
String hash0 = org.apache.commons.codec.digest.DigestUtils.md5Hex(seqid + "|" + live_url_info.get("ctype") + "|" + live_url_info.get("t"));
String hash1 = org.apache.commons.codec.digest.DigestUtils.md5Hex(StringUtils.join(new String[]{live_url_info.get("hash_prefix"), user_id, live_url_info.get("stream_name"), hash0, wsTime}, '_'));
String url;
if (live_url_info.get("ctype").contains("mobile")) {
url = String.format("%s?wsSecret=%s&wsTime=%s&uuid=%s&uid=%s&seqid=%s&ratio=%s&txyp=%s&fs=%s&ctype=%s&ver=1&t=%s",
live_url_info.get("hls_url"), hash1, wsTime, live_url_info.get("uuid"), user_id, seqid, ratio, live_url_info.get("txyp"),
live_url_info.get("fs"), live_url_info.get("ctype"), live_url_info.get("t"));
} else {
url = String.format("%s?wsSecret=%s&wsTime=%s&seqid=%s&ctype=%s&ver=1&txyp=%s&fs=%s&ratio=%s&u=%s&t=%s&sv=2107230339",
live_url_info.get("hls_url"), hash1, wsTime, seqid, live_url_info.get("ctype"), live_url_info.get("txyp"), live_url_info.get("fs"),
ratio, user_id, live_url_info.get("t"));
}
urls.add(url);
}
return urls;
}
@Override
public String getUrl(HuyaLiveOnlineConfig LoadConfig) throws Exception {
this.room_id = LoadConfig.getRoomId();
this.user_id = "1463389097687";
this.mode = 1;
this.header = LoadConfig.getHeader();
this.updateLiveUrlInfo();
List<String> list = getRealUrl();
return list.get(0).replace("hls","flv").replace("m3u8","flv");
}
}
Binary file not shown.
+16 -16
View File
@@ -102,22 +102,22 @@
<version>portable-1.7.8</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.squareup.okhttp3</groupId>-->
<!-- <artifactId>okhttp</artifactId>-->
<!-- <version>4.9.0</version>-->
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <groupId>com.squareup.okio</groupId>-->
<!-- <artifactId>okio</artifactId>-->
<!-- </exclusion>-->
<!-- </exclusions>-->
<!-- </dependency>-->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>4.9.0</version>
<exclusions>
<exclusion>
<groupId>com.squareup.okio</groupId>
<artifactId>okio</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.squareup.okio</groupId>-->
<!-- <artifactId>okio</artifactId>-->
<!-- <version>3.2.0</version>-->
<!-- </dependency>-->
<dependency>
<groupId>com.squareup.okio</groupId>
<artifactId>okio</artifactId>
<version>3.2.0</version>
</dependency>
</dependencies>
</project>
@@ -0,0 +1,36 @@
package org.example.account;
import org.example.ConsoleApplication;
import org.example.bean.Barrage;
import org.example.core.analysis.AnalysisSchemeBuilder;
import org.example.core.analysis.EmotionAnalysisPlugin;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
/**
* @Date 2023/10/16
* @Author xiaochun
*/
@RunWith(SpringRunner.class)
@SpringBootTest(classes = ConsoleApplication.class,webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT)
public class AnalysisTest {
@Resource
EmotionAnalysisPlugin plugin;
@Test
public void AnalysisTest(){
List<Barrage> barrages = new ArrayList<>();
barrages.add(new Barrage("1", 1L, 1L,"太帅啦"));
barrages.add(new Barrage("2", 1L, 1L,"666"));
barrages.add(new Barrage("3", 1L, 1L,"无敌"));
AnalysisSchemeBuilder builder = new AnalysisSchemeBuilder()
.barrages(barrages);
System.out.println(plugin.analysis(builder));
}
}
@@ -30,6 +30,7 @@ public class gptTest {
.system("请你作为一个专门看直播的观众,对下列的观众发送的弹幕内容进行分析,然后根据弹幕内容返回从以下几个标签返回给我最合适的一个标签来形容这段内容标签:[搞笑,秀操作,破防,泪目]")
.user("弹幕:帅,这操作我要学一年,太6了")
.stream(false);
System.out.println(builder.done());
ChatGPTPlugin plugin1 = InitPluginRegister.getPlugin(PluginName.CHAT_GPT, ChatGPTPlugin.class);
System.out.println(plugin.reqGPT(builder));
System.out.println(plugin1.reqGPT(builder));
@@ -63,7 +63,7 @@ public class LiveTest {
@Test
public void HuyaLive(){
HuyaLiveOnlineConfig liveConfig = new HuyaLiveOnlineConfig("mumu123456", "C:\\Users\\admin\\Desktop\\douyu\\", "猪猪公主",false);
HuyaLiveOnlineConfig liveConfig = new HuyaLiveOnlineConfig("294359", "C:\\Users\\admin\\Desktop\\douyu\\", "猪猪公主",false);
// 创建下载任务管理器
LiveDownloadManager liveDownLoadManager = new LiveDownloadManager(5);