From 3e7ea14a4a03e441eee404ccbdecd60d9ff45b5b Mon Sep 17 00:00:00 2001 From: shinas <97829569+shinas101@users.noreply.github.com> Date: Tue, 3 Nov 2026 08:57:14 +0530 Subject: [PATCH] add tamilmv a public Indian site (#17227) resolves #2421 --- README.md | 1 + .../Indexers/Definitions/TamilMV.cs | 774 ++++++++++++++++++ .../Common/Indexers/TamilMVTests.cs | 306 +++++++ 3 files changed, 1081 insertions(+) create mode 100644 src/Jackett.Common/Indexers/Definitions/TamilMV.cs create mode 100644 src/Jackett.Test/Common/Indexers/TamilMVTests.cs diff --git a/README.md b/README.md index b59ceebce..cbc1c8342 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,7 @@ Prior versions of Jackett are no longer supported. * ØMagnet * 1337x + * 1TamilMV * 52BT * ACG.RIP * Anibt diff --git a/src/Jackett.Common/Indexers/Definitions/TamilMV.cs b/src/Jackett.Common/Indexers/Definitions/TamilMV.cs new file mode 100644 index 000000000..fe338dcab --- /dev/null +++ b/src/Jackett.Common/Indexers/Definitions/TamilMV.cs @@ -0,0 +1,774 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using AngleSharp.Dom; +using AngleSharp.Html.Dom; +using AngleSharp.Html.Parser; +using Jackett.Common.Extensions; +using Jackett.Common.Helpers; +using Jackett.Common.Models; +using Jackett.Common.Models.IndexerConfig; +using Jackett.Common.Services.Interfaces; +using Jackett.Common.Utils.Clients; +using Newtonsoft.Json.Linq; +using NLog; +using static Jackett.Common.Models.IndexerConfig.ConfigurationData; +using WebClient = Jackett.Common.Utils.Clients.WebClient; + +namespace Jackett.Common.Indexers.Definitions +{ + [ExcludeFromCodeCoverage] + public class TamilMV : IndexerBase + { + public override string Id => "tamilmv"; + public override string Name => "1TamilMV"; + public override string Description => "1TamilMV is a Public Indian torrent tracker for Tamil, Telugu, Hindi, Malayalam, and Kannada releases."; + public override string SiteLink { get; protected set; } = "https://www.1tamilmv.style/"; + + public override string[] AlternativeSiteLinks => new[] + { + "https://www.1tamilmv.style/", + }; + + public override string[] LegacySiteLinks => new[] + { + "https://www.1tamilmv.ac/", + "https://www.1tamilmv.eu/", + "https://www.1tamilmv.pizza/", + "https://www.1tamilmv.wtf/", + "https://www.1tamilmv.unblockninja.com/", + }; + + public override string Language => "ta-IN"; + public override string Type => "public"; + + public override TorznabCapabilities TorznabCaps => SetCapabilities(); + + private static readonly Regex TopicRegex = new Regex(@"/forums/topic/(\d+)", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex BtihRegex = new Regex(@"urn:btih:([a-zA-Z0-9]+)", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex SizeRegex = new Regex(@"(?:^|[^\w.])(\d+(?:\.\d+)?)\s*(GB|MB|KB)(?![a-zA-Z])", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex SitePrefixRegex = new Regex(@"^(?:www\.)?(?:1?tamilmv|tamilblasters|tamilrockers)\.[a-z0-9]+\s*-\s*", RegexOptions.Compiled | RegexOptions.IgnoreCase); + public static readonly Regex ExcludedTitles = new Regex( + @"\b(?:official\s+)?(?:theatrical\s+)?trailers?\b(?!\s*park\b)" + + @"|\bteasers?\b" + + @"|\bsamples?\b" + + @"|\bvideo\s+songs?\b" + + @"|\blyric(?:al)?\s+videos?\b" + + @"|\bmusic\s+videos?\b" + + @"|\b(?:full\s+)?albums?\b" + + @"|\b(?:audio|music)\s+(?:launch|release|jukebox)\b" + + @"|\b(?:audio|music|single)\s+(?:tracks?|singles?)\b" + + @"|\b\d+(?:st|nd|rd|th)\s+single\b" + + @"|\b(?:motion\s+picture\s+)?soundtracks?\b" + + @"|\b(?:original\s+)?background\s+scores?\b" + + @"|\boriginal\s+(?:motion\s+picture\s+)?score\b" + + @"|\b(?:first\s+look|glimpse|promos?|sneak\s+peek)\b" + + @"|\b(?:launch|launched)\s+videos?\b" + + @"|\b(?:success\s+meet|press\s+meet|pre[- ]?release\s+event)\b" + + @"|\b(?:making|behind[- ]the[- ]scenes|BTS)\s+videos?\b" + + @"|\b(?:exclusive\s+|public\s+|press\s+|cast\s+|special\s+)?interview\s+with\b" + + @"|\b(?:exclusive|press\s+meet|cast)\s+interviews?\b" + + @"|\b(?:movie|public|theatre|theater|audience|trailer|teaser)\s+(?:review|reviews|reaction|reactions)\b" + + @"|\b(?:full|awards?|award)\s+show\b" + + @"|\b(?:speech|speeches)\b", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex QualityRegex = new Regex( + @"\b(?:2160p|1080p|720p|480p|4k|uhd|hdrip|web-dl|predvd|dvdrip|x264|x265|hevc|avc|aac)\b", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex YearRegex = new Regex( + @"\b(?:19\d{2}|20\d{2})\b", + RegexOptions.Compiled); + private static readonly Regex TvRegex = new Regex(@"\b(?:s\d+|ep\s*\(?\d+|season\s*\d+|series|episode)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex AudioRegex = new Regex(@"\b(?:ost|soundtrack|flac|mp3\s*songs?|audio\s*songs?)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex UhdRegex = new Regex(@"\b(?:2160p|4k|uhd)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex HdRegex = new Regex(@"\b(?:1080p|fhd|720p|hd)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex SdRegex = new Regex(@"\b(?:480p|sd)\b", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex ExcludedForumRegex = new Regex( + @"video[- ]songs?|mp3[- ]songs?|cinema[- ]news|general[- ]videos|watch[- ].*?[- ]online|post[- &]*share[- ]here|request[- ]section|members[- ]lounge|/forums?/forum/(?:15|16|18|28|30|32|40|41|43|51|52|54|62|63|65|73|74|75|76|89|92|93|95|102|103|104)\b", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + private static readonly ConcurrentDictionary releases)> _topicCache = new(); + private const int MaxReleases = 100; + private const int MaxTopics = 20; + + public TamilMV(IIndexerConfigurationService configService, WebClient wc, Logger l, IProtectionService ps, ICacheService cs) + : base(configService: configService, + client: wc, + logger: l, + p: ps, + cacheService: cs, + configData: new ConfigurationData()) + { + // Avoid Cloudflare too many requests limiter / site thrashing + webclient.requestDelay = 1.1; + + var stripPrefix = new BoolConfigurationItem("Strip website prefix from torrent titles") { Value = true }; + configData.AddDynamic("stripPrefix", stripPrefix); + + var filterSamples = new BoolConfigurationItem("Filter out trailers, samples, and promo releases") { Value = true }; + configData.AddDynamic("filterSamples", filterSamples); + } + + private static TorznabCapabilities SetCapabilities() + { + var caps = new TorznabCapabilities + { + TvSearchParams = new List + { + TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep + }, + MovieSearchParams = new List + { + MovieSearchParam.Q + }, + MusicSearchParams = new List + { + MusicSearchParam.Q + } + }; + + caps.Categories.AddCategoryMapping(2000, TorznabCatType.Movies, "Movies"); + caps.Categories.AddCategoryMapping(2040, TorznabCatType.MoviesHD, "Movies/HD"); + caps.Categories.AddCategoryMapping(2045, TorznabCatType.MoviesUHD, "Movies/UHD"); + caps.Categories.AddCategoryMapping(2030, TorznabCatType.MoviesSD, "Movies/SD"); + caps.Categories.AddCategoryMapping(2010, TorznabCatType.MoviesForeign, "Movies/Foreign"); + + caps.Categories.AddCategoryMapping(5000, TorznabCatType.TV, "TV"); + caps.Categories.AddCategoryMapping(5040, TorznabCatType.TVHD, "TV/HD"); + caps.Categories.AddCategoryMapping(5045, TorznabCatType.TVUHD, "TV/UHD"); + caps.Categories.AddCategoryMapping(5030, TorznabCatType.TVSD, "TV/SD"); + caps.Categories.AddCategoryMapping(5020, TorznabCatType.TVForeign, "TV/Foreign"); + + caps.Categories.AddCategoryMapping(3000, TorznabCatType.Audio, "Audio"); + caps.Categories.AddCategoryMapping(3010, TorznabCatType.AudioMP3, "Audio/MP3"); + caps.Categories.AddCategoryMapping(3020, TorznabCatType.AudioLossless, "Audio/Lossless"); + caps.Categories.AddCategoryMapping(3040, TorznabCatType.AudioVideo, "Audio/Video"); + + return caps; + } + + public override async Task ApplyConfiguration(JToken configJson) + { + LoadValuesFromJson(configJson); + var releases = await PerformQuery(new TorznabQuery()); + + await ConfigureIfOK(string.Empty, releases.Any(), () => + throw new Exception("Could not find releases from 1TamilMV")); + + return IndexerConfigurationStatus.Completed; + } + + public override async Task Download(Uri link) + { + var uncleanLink = UncleanLink(link); + return await Download(uncleanLink, RequestType.GET, referer: SiteLink); + } + + protected override async Task> PerformQuery(TorznabQuery query) + { + var stripPrefix = ((BoolConfigurationItem)configData.GetDynamic("stripPrefix"))?.Value ?? true; + var filterSamples = ((BoolConfigurationItem)configData.GetDynamic("filterSamples"))?.Value ?? true; + + var topicUrls = new List(); + var searchTerm = query.GetQueryString(); + + if (string.IsNullOrWhiteSpace(searchTerm)) + { + topicUrls = await ExtractHomepageTopicsAsync(SiteLink, MaxTopics); + } + else + { + topicUrls = await SearchTopicsAsync(SiteLink, searchTerm); + } + + var releases = new List(); + var parser = new HtmlParser(); + + foreach (var topicUrl in topicUrls) + { + if (releases.Count >= MaxReleases) + { + break; + } + + if (IsExcludedForum(null, topicUrl)) + { + continue; + } + + if (_topicCache.TryGetValue(topicUrl, out var cached) && DateTime.UtcNow < cached.expiresAt) + { + var matchingCached = searchTerm.IsNotNullOrWhiteSpace() + ? cached.releases.Where(r => query.MatchQueryStringAND(r.Title)) + : cached.releases; + + releases.AddRange(matchingCached); + continue; + } + + try + { + var topicResponse = await RequestWithCookiesAndRetryAsync(topicUrl, referer: SiteLink); + if (topicResponse.IsRedirect) + { + topicResponse = await FollowIfRedirect(topicResponse, referrer: SiteLink, maxRedirects: 1); + } + if (topicResponse.Status != HttpStatusCode.OK) + { + logger.Warn($"Failed to fetch 1TamilMV topic {topicUrl}: HTTP {topicResponse.Status}"); + continue; + } + + using var doc = parser.ParseDocument(topicResponse.ContentString); + var topicReleases = ExtractTopicReleases(doc, topicUrl, SiteLink, stripPrefix, filterSamples, logger); + + _topicCache[topicUrl] = (DateTime.UtcNow.AddDays(1), topicReleases); + + var matchingReleases = searchTerm.IsNotNullOrWhiteSpace() + ? topicReleases.Where(r => query.MatchQueryStringAND(r.Title)) + : topicReleases; + + releases.AddRange(matchingReleases); + } + catch (Exception ex) + { + logger.Error(ex, $"Error extracting releases from 1TamilMV topic {topicUrl}"); + } + } + + if (_topicCache.Count > 500) + { + var now = DateTime.UtcNow; + foreach (var kvp in _topicCache) + { + if (kvp.Value.expiresAt < now) + _topicCache.TryRemove(kvp.Key, out _); + } + } + + return releases.OrderByDescending(r => r.PublishDate).ToList(); + } + + private async Task> ExtractHomepageTopicsAsync(string siteLink, int limit = MaxTopics) + { + var response = await RequestWithCookiesAndRetryAsync(siteLink); + if (response.IsRedirect) + { + response = await FollowIfRedirect(response, referrer: siteLink, maxRedirects: 1); + } + if (response.Status != HttpStatusCode.OK) + { + throw new Exception($"Failed to fetch 1TamilMV homepage ({siteLink}): HTTP {response.Status}"); + } + + var parser = new HtmlParser(); + using var doc = parser.ParseDocument(response.ContentString); + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + var bangerContainers = doc.QuerySelectorAll(".banger-container"); + var topReleases = new List(); + var recentlyAdded = new List(); + + foreach (var container in bangerContainers) + { + var headerText = container.QuerySelector(".banger-header")?.TextContent?.ToUpperInvariant() ?? string.Empty; + var links = ExtractTopicLinksFromElement(container, siteLink); + + if (headerText.Contains("TOP RELEASES")) + { + topReleases.AddRange(links); + } + else if (headerText.Contains("RECENTLY ADDED")) + { + recentlyAdded.AddRange(links); + } + } + + var topicUrls = topReleases.Where(seen.Add).ToList(); + topicUrls.AddRange(recentlyAdded.Where(seen.Add)); + + if (topicUrls.Count == 0) + { + var fallbackElements = doc.QuerySelectorAll(".banger-container, .ipsType_break, .ipsContained, .cTopicList, .ipsDataList"); + foreach (var el in fallbackElements) + { + topicUrls.AddRange(ExtractTopicLinksFromElement(el, siteLink).Where(seen.Add)); + } + + if (topicUrls.Count == 0) + { + foreach (var a in doc.QuerySelectorAll("a[href]")) + { + var href = a.GetAttribute("href"); + var clean = NormalizeTopicUrl(href, siteLink); + if (!string.IsNullOrEmpty(clean) && seen.Add(clean)) + topicUrls.Add(clean); + } + } + } + + return topicUrls.Take(limit).ToList(); + } + + private async Task> SearchTopicsAsync(string siteLink, string searchTerm) + { + var searchUrl = $"{siteLink.TrimEnd('/')}/search/api/search.php?q={Uri.EscapeDataString(searchTerm)}&page=1&per_page=100&sort=newest"; + var response = await RequestWithCookiesAndRetryAsync(searchUrl, referer: siteLink); + if (response.IsRedirect) + { + response = await FollowIfRedirect(response, referrer: siteLink, maxRedirects: 1); + } + if (response.Status != HttpStatusCode.OK) + { + logger.Warn($"Failed to search 1TamilMV API: HTTP {response.Status}"); + return new List(); + } + + return ParseSearchApiResponse(response.ContentString, siteLink); + } + + private static List ExtractTopicLinksFromElement(IElement element, string siteLink) + { + var list = new List(); + foreach (var a in element.QuerySelectorAll("a[href]")) + { + var href = a.GetAttribute("href"); + var clean = NormalizeTopicUrl(href, siteLink); + if (!string.IsNullOrEmpty(clean) && !list.Contains(clean)) + { + list.Add(clean); + } + } + return list; + } + + public static List ParseSearchApiResponse(string jsonString, string siteLink) + { + var topicUrls = new List(); + if (string.IsNullOrWhiteSpace(jsonString)) + { + return topicUrls; + } + + var json = JObject.Parse(jsonString); + var results = json["results"] as JArray; + if (results == null || results.Count == 0) + { + return topicUrls; + } + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var item in results) + { + // Only scrape if starter_role is not null + var starterRole = item["starter_role"]; + if (starterRole == null || starterRole.Type == JTokenType.Null || string.IsNullOrWhiteSpace(starterRole.ToString())) + { + continue; + } + + var tid = item["tid"]?.Value(); + if (tid == null || tid <= 0) + { + continue; + } + + var rawTitle = item["title"]?.Value() ?? string.Empty; + if (ExcludedTitles.IsMatch(rawTitle)) + { + continue; + } + var topicUrl = $"{siteLink.TrimEnd('/')}/index.php?/forums/topic/{tid}-{Slugify(rawTitle)}/"; + var cleanUrl = NormalizeTopicUrl(topicUrl, siteLink); + if (!string.IsNullOrEmpty(cleanUrl) && seen.Add(cleanUrl)) + { + topicUrls.Add(cleanUrl); + } + } + + return topicUrls; + } + + public static string Slugify(string title) + { + if (string.IsNullOrWhiteSpace(title)) + { + return "t"; + } + + var decoded = WebUtility.HtmlDecode(title); + // Invision Community strips non-alphanumeric punctuation completely (e.g. DD+5.1 -> dd51, 1.4GB -> 14gb) + var slug = Regex.Replace(decoded, @"[^\w\s-]", "", RegexOptions.None); + // Replace spaces, underscores, and consecutive hyphens with a single hyphen + slug = Regex.Replace(slug, @"[\s_-]+", "-").Trim('-').ToLowerInvariant(); + + return string.IsNullOrEmpty(slug) ? "t" : slug; + } + + public static string NormalizeTopicUrl(string href, string siteLink) + { + if (string.IsNullOrWhiteSpace(href)) + return string.Empty; + + if (ExcludedForumRegex.IsMatch(href)) + return string.Empty; + + var m = TopicRegex.Match(href); + if (!m.Success || !int.TryParse(m.Groups[1].Value, out var topicId) || topicId < 1000) + return string.Empty; + + var url = href.Split('#')[0]; + if (url.Contains("&do=findComment")) + { + url = url.Split(new[] { "&do=" }, StringSplitOptions.None)[0]; + } + if (url.Contains("?do=findComment")) + { + url = url.Split(new[] { "?do=" }, StringSplitOptions.None)[0]; + } + + if (url.StartsWith("/")) + { + return siteLink.TrimEnd('/') + url; + } + + if (!url.StartsWith("http", StringComparison.OrdinalIgnoreCase)) + { + return new Uri(new Uri(siteLink), url).ToString(); + } + + return url; + } + + public static List ExtractTopicReleases( + IHtmlDocument doc, + string topicUrl, + string siteLink, + bool stripPrefix = true, + bool filterSamples = true, + Logger logger = null) + { + var results = new List(); + + if (IsExcludedForum(doc, topicUrl)) + { + logger?.Debug($"1TamilMV skipping topic in excluded forum: {topicUrl}"); + return results; + } + + var magnetLinks = doc.QuerySelectorAll("a[href^='magnet:?']") + .Select(a => a.GetAttribute("href")?.Trim()) + .Where(m => !string.IsNullOrEmpty(m)) + .ToList(); + + var torrentLinks = doc.QuerySelectorAll("a[data-fileext='torrent'][href]").ToList(); + if (torrentLinks.Count == 0) + { + torrentLinks = doc.QuerySelectorAll("a[href]") + .Where(a => + { + var h = (a.GetAttribute("href") ?? string.Empty).ToLowerInvariant(); + var txt = (a.TextContent ?? string.Empty).Trim().ToLowerInvariant(); + return h.Contains("attachment.php?id=") || h.EndsWith(".torrent") || txt.EndsWith(".torrent"); + }) + .ToList(); + } + + var dateStr = doc.QuerySelector("time[datetime]")?.GetAttribute("datetime"); + var publishDate = DateTime.TryParse(dateStr, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out var parsedDate) + ? parsedDate + : DateTime.UtcNow; + + for (var i = 0; i < torrentLinks.Count; i++) + { + var a = torrentLinks[i]; + var href = a.GetAttribute("href")?.Trim(); + var rawTitle = a.TextContent?.Trim(); + if (string.IsNullOrEmpty(rawTitle)) + { + rawTitle = a.GetAttribute("title")?.Trim(); + } + + if (string.IsNullOrEmpty(href) || string.IsNullOrEmpty(rawTitle)) + continue; + + if (!rawTitle.EndsWith(".torrent", StringComparison.OrdinalIgnoreCase) && + href.IndexOf("attachment.php?id=", StringComparison.OrdinalIgnoreCase) < 0 && + !href.EndsWith(".torrent", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (filterSamples && ExcludedTitles.IsMatch(rawTitle)) + { + logger?.Debug($"1TamilMV skipping filtered title: {rawTitle}"); + continue; + } + + if (href.StartsWith("/")) + { + href = siteLink.TrimEnd('/') + href; + } + else if (!href.StartsWith("http", StringComparison.OrdinalIgnoreCase)) + { + href = new Uri(new Uri(siteLink), href).ToString(); + } + + var cleanTitle = CleanTitle(rawTitle, stripPrefix); + var sizeBytes = ParseSize(cleanTitle); + + // Fallback: If size cannot be parsed from the torrent link filename (e.g. 0B), + // inspect the preceding heading element in the post for size and full release title + if (sizeBytes <= 0) + { + var header = FindPrecedingHeader(a); + if (!string.IsNullOrEmpty(header)) + { + var headerSize = ParseSize(header); + if (headerSize > 0) + { + sizeBytes = headerSize; + } + + var cleanHeader = Regex.Replace(header, @"[\s:]+$", string.Empty).Trim(); + cleanHeader = CleanTitle(cleanHeader, stripPrefix); + if (!string.IsNullOrEmpty(cleanHeader) && (YearRegex.IsMatch(cleanHeader) || QualityRegex.IsMatch(cleanHeader))) + { + var extMatch = Regex.Match(cleanTitle, @"\.(?![0-9]+$)[a-zA-Z0-9]{2,4}$"); + if (extMatch.Success && !cleanHeader.EndsWith(extMatch.Value, StringComparison.OrdinalIgnoreCase)) + { + cleanHeader += extMatch.Value; + } + cleanTitle = cleanHeader; + } + } + } + + var magnetUrl = i < magnetLinks.Count ? magnetLinks[i] : (magnetLinks.Count > 0 ? magnetLinks[0] : string.Empty); + string infoHash = null; + if (!string.IsNullOrEmpty(magnetUrl)) + { + var mHash = BtihRegex.Match(magnetUrl); + if (mHash.Success) + { + infoHash = mHash.Groups[1].Value.ToLowerInvariant(); + } + } + + var categories = DetermineCategories(cleanTitle); + + var release = new ReleaseInfo + { + Title = cleanTitle, + Details = new Uri(topicUrl), + Guid = new Uri(href), + Link = new Uri(href), + MagnetUri = !string.IsNullOrEmpty(magnetUrl) ? new Uri(magnetUrl) : null, + InfoHash = infoHash, + PublishDate = publishDate, + Category = categories, + Size = sizeBytes, + Seeders = 1, + Peers = 1, + DownloadVolumeFactor = 0, + UploadVolumeFactor = 1 + }; + + results.Add(release); + } + + return results; + } + + public static string FindPrecedingHeader(IElement torrentElement) + { + if (torrentElement == null) + { + return null; + } + + var current = (INode)torrentElement; + while (current != null) + { + var prev = current.PreviousSibling; + while (prev != null) + { + var text = prev.TextContent?.Trim(); + if (!string.IsNullOrEmpty(text) && + !text.EndsWith(".torrent", StringComparison.OrdinalIgnoreCase) && + !text.Contains("MAGNET") && + !text.Contains("DIRECT LINK") && + text.Length > 15) + { + if (SizeRegex.IsMatch(text) || QualityRegex.IsMatch(text)) + { + return text; + } + } + + prev = prev.PreviousSibling; + } + + current = current.ParentElement; + if (current == null || + current.NodeName.Equals("BODY", StringComparison.OrdinalIgnoreCase) || + current.NodeName.Equals("HTML", StringComparison.OrdinalIgnoreCase) || + (current is IElement elem && (elem.ClassList.Contains("cPost_contentWrap") || elem.GetAttribute("data-role") == "commentContent"))) + { + break; + } + } + + return null; + } + + public static string CleanTitle(string rawTitle, bool stripPrefix = true) + { + if (string.IsNullOrWhiteSpace(rawTitle)) + return string.Empty; + + var title = WebUtility.HtmlDecode(rawTitle).Trim(); + + if (stripPrefix) + { + title = SitePrefixRegex.Replace(title, string.Empty); + } + + if (title.EndsWith(".torrent", StringComparison.OrdinalIgnoreCase)) + { + title = title.Substring(0, title.Length - ".torrent".Length).Trim(); + } + + title = Regex.Replace(title, @"\s+", " ").Trim(); + return title; + } + + public static long ParseSize(string title) + { + if (string.IsNullOrWhiteSpace(title)) + { + return 0; + } + + var m = SizeRegex.Match(title); + if (!m.Success) + { + return 0; + } + + if (!double.TryParse(m.Groups[1].Value, NumberStyles.Float, CultureInfo.InvariantCulture, out var sizeVal)) + { + return 0; + } + + var unit = m.Groups[2].Value.ToUpperInvariant(); + return unit switch + { + "GB" => sizeVal.Gigabytes(), + "MB" => sizeVal.Megabytes(), + "KB" => (long)(sizeVal * 1024L), + _ => 0 + }; + } + + public static List DetermineCategories(string title) + { + var cats = new List(); + + if (AudioRegex.IsMatch(title)) + { + cats.Add(TorznabCatType.Audio.ID); + + if (title.IndexOf("flac", StringComparison.OrdinalIgnoreCase) >= 0) + { + cats.Add(TorznabCatType.AudioLossless.ID); + } + else + { + cats.Add(TorznabCatType.AudioMP3.ID); + } + + return cats; + } + + var isUhd = UhdRegex.IsMatch(title); + var isHd = HdRegex.IsMatch(title); + var isSd = SdRegex.IsMatch(title); + + if (TvRegex.IsMatch(title)) + { + cats.Add(TorznabCatType.TV.ID); + cats.Add(TorznabCatType.TVForeign.ID); + + if (isUhd) + { + cats.Add(TorznabCatType.TVUHD.ID); + } + else if (isHd) + { + cats.Add(TorznabCatType.TVHD.ID); + } + else if (isSd) + { + cats.Add(TorznabCatType.TVSD.ID); + } + } + else + { + cats.Add(TorznabCatType.Movies.ID); + cats.Add(TorznabCatType.MoviesForeign.ID); + + if (isUhd) + { + cats.Add(TorznabCatType.MoviesUHD.ID); + } + else if (isHd) + { + cats.Add(TorznabCatType.MoviesHD.ID); + } + else if (isSd) + { + cats.Add(TorznabCatType.MoviesSD.ID); + } + } + + return cats; + } + + public static bool IsExcludedForum(IHtmlDocument doc, string topicUrl = null) + { + if (!string.IsNullOrEmpty(topicUrl) && ExcludedForumRegex.IsMatch(topicUrl)) + { + return true; + } + + if (doc != null) + { + var breadcrumbLinks = doc.QuerySelectorAll(".ipsBreadcrumb a[href], [data-role='breadcrumbList'] a[href], .ipsType_breadcrumb a[href]"); + foreach (var a in breadcrumbLinks) + { + var href = a.GetAttribute("href") ?? string.Empty; + if (ExcludedForumRegex.IsMatch(href)) + { + return true; + } + + var text = a.TextContent ?? string.Empty; + if (ExcludedForumRegex.IsMatch(text)) + { + return true; + } + } + } + + return false; + } + } +} diff --git a/src/Jackett.Test/Common/Indexers/TamilMVTests.cs b/src/Jackett.Test/Common/Indexers/TamilMVTests.cs new file mode 100644 index 000000000..1b979415c --- /dev/null +++ b/src/Jackett.Test/Common/Indexers/TamilMVTests.cs @@ -0,0 +1,306 @@ +using System.Collections.Generic; +using FluentAssertions; +using Jackett.Common.Helpers; +using Jackett.Common.Indexers.Definitions; +using Jackett.Common.Models; +using NUnit.Framework; + +namespace Jackett.Test.Common.Indexers +{ + [TestFixture] + public class TamilMVTests + { + [TestCase("www.1TamilMV.solar - Baththa (2026) Tamil HDRip - 1080p - x264 - AAC - 2.9GB - HC-ESub.mkv.torrent", + "Baththa (2026) Tamil HDRip - 1080p - x264 - AAC - 2.9GB - HC-ESub.mkv")] + [TestCase("1TamilMV.pm - The Marked Woman (2026) HD - 720p - x264 - 1.4GB.torrent", + "The Marked Woman (2026) HD - 720p - x264 - 1.4GB")] + [TestCase("Teach You a Lesson (2026) S01 EP (01-08) TRUE WEB-DL - 1080p.mkv.torrent", + "Teach You a Lesson (2026) S01 EP (01-08) TRUE WEB-DL - 1080p.mkv")] + public void TestCleanTitle(string raw, string expected) + { + var clean = TamilMV.CleanTitle(raw, stripPrefix: true); + clean.Should().Be(expected); + } + + [Test] + public void TestParseSize() + { + var gbBytes = TamilMV.ParseSize("Baththa (2026) Tamil HDRip - 2.9GB - HC-ESub.mkv"); + gbBytes.Should().Be(2.9.Gigabytes()); + + var mbBytes = TamilMV.ParseSize("Release - 700MB - x264.mkv"); + mbBytes.Should().Be(700.Megabytes()); + + var kbBytes = TamilMV.ParseSize("Sample - 500KB.mkv"); + kbBytes.Should().Be(500L * 1024L); + } + + [Test] + public void TestDetermineCategoriesMovieHD() + { + var cats = TamilMV.DetermineCategories("Baththa (2026) Tamil HDRip - 1080p - x264 - 2.9GB.mkv"); + cats.Should().Contain(TorznabCatType.Movies.ID); + cats.Should().Contain(TorznabCatType.MoviesHD.ID); + cats.Should().Contain(TorznabCatType.MoviesForeign.ID); + } + + [Test] + public void TestDetermineCategoriesMovieUHD() + { + var cats = TamilMV.DetermineCategories("Baththa (2026) Tamil WEB-DL - 4K - 2160p - 20GB.mkv"); + cats.Should().Contain(TorznabCatType.Movies.ID); + cats.Should().Contain(TorznabCatType.MoviesUHD.ID); + } + + [Test] + public void TestDetermineCategoriesTV() + { + var cats = TamilMV.DetermineCategories("Teach You a Lesson (2026) S01 EP (01-08) TRUE WEB-DL - 720p - 1.4GB"); + cats.Should().Contain(TorznabCatType.TV.ID); + cats.Should().Contain(TorznabCatType.TVHD.ID); + cats.Should().Contain(TorznabCatType.TVForeign.ID); + } + + [Test] + public void TestDetermineCategoriesAudio() + { + var cats = TamilMV.DetermineCategories("Baththa (2026) Original Motion Picture Soundtrack Hi-Fi FLAC - 190 MB"); + cats.Should().Contain(TorznabCatType.Audio.ID); + cats.Should().Contain(TorznabCatType.AudioLossless.ID); + } + + [TestCase("https://www.1tamilmv.style/index.php?/forums/forum/92-post-share-here/", true)] + [TestCase("https://www.1tamilmv.style/forums/forum/92-post-share-here/", true)] + [TestCase("/forums/forum/92-post-share-here/", true)] + [TestCase("/forum/92-post-share-here/", true)] + [TestCase("/16-video-songs/", true)] + [TestCase("/15-mp3-songs/", true)] + [TestCase("/18-cinema-news-reviews-general-videos/", true)] + [TestCase("/102-watch-tamil-movies-online/", true)] + [TestCase("104-watch-malayalam-movies-online/", true)] + [TestCase("https://www.1tamilmv.style/index.php?/forums/forum/95-watch-telugu-movies-online/", true)] + [TestCase("https://www.1tamilmv.style/index.php?/forums/forum/75-watch-kannada-movies-online/", true)] + [TestCase("https://www.1tamilmv.style/index.php?/forums/forum/52-video-songs/", true)] + [TestCase("https://www.1tamilmv.style/index.php?/forums/forum/62-mp3-songs/", true)] + [TestCase("https://www.1tamilmv.style/index.php?/forums/forum/89-request-section/", true)] + [TestCase("https://www.1tamilmv.style/index.php?/forums/forum/92/", true)] + [TestCase("https://www.1tamilmv.style/index.php?/forums/topic/12345-some-topic/", false)] + [TestCase("https://www.1tamilmv.style/index.php?/forums/forum/11-web-hd-itunes-hd-bluray/", false)] + [TestCase("https://www.1tamilmv.style/index.php?/forums/forum/12-tamil-movies/", false)] + public void TestIsExcludedForumUrl(string url, bool expected) + { + var result = TamilMV.IsExcludedForum(null, url); + result.Should().Be(expected); + } + + [Test] + public void TestIsExcludedForumBreadcrumbs() + { + var parser = new AngleSharp.Html.Parser.HtmlParser(); + var docExcluded = parser.ParseDocument(@" + + + "); + TamilMV.IsExcludedForum(docExcluded, "https://www.1tamilmv.style/index.php?/forums/topic/99999-shared-release/").Should().BeTrue(); + + var docAllowed = parser.ParseDocument(@" + + + "); + TamilMV.IsExcludedForum(docAllowed, "https://www.1tamilmv.style/index.php?/forums/topic/99999-official-release/").Should().BeFalse(); + } + + [TestCase("Leo (2023) Tamil Proper TRUE WEB-DL", "leo-2023-tamil-proper-true-web-dl")] + [TestCase("Jackie Chan's The Tuxedo (2002)", "jackie-chans-the-tuxedo-2002")] + [TestCase("Movie @ 2024 #Special!", "movie-2024-special")] + [TestCase("Baththa (2026) Tamil TRUE WEB-DL - [4K, 1080p & 720p - AVC / HEVC - (DD+5.1 - 640Kbps) - 17GB - 4GB - 1.4GB & 950MB] - [x264 - 700MB - 400MB & 250MB] - ESub", + "baththa-2026-tamil-true-web-dl-4k-1080p-720p-avc-hevc-dd51-640kbps-17gb-4gb-14gb-950mb-x264-700mb-400mb-250mb-esub")] + [TestCase("", "t")] + public void TestSlugify(string rawTitle, string expectedSlug) + { + var slug = TamilMV.Slugify(rawTitle); + slug.Should().Be(expectedSlug); + } + + [Test] + public void TestParseSearchApiResponse_OnlyScrapesNonNullStarterRole() + { + var json = @"{ + ""results"": [ + { + ""tid"": 176409, + ""title"": ""Leo (2023) Official Release"", + ""starter_role"": ""Moderator"" + }, + { + ""tid"": 193696, + ""title"": ""Leo (2023) User Post"", + ""starter_role"": null + }, + { + ""tid"": 176106, + ""title"": ""Leo (2023) Elite Release"", + ""starter_role"": ""Elite"" + }, + { + ""tid"": 189999, + ""title"": ""Leo (2023) Empty Role Post"", + ""starter_role"": """" + }, + { + ""tid"": 188888, + ""title"": ""Leo (2023) Missing Role Post"" + } + ], + ""total"": 5, + ""page"": 1, + ""per_page"": 50, + ""page_count"": 1 + }"; + + var urls = TamilMV.ParseSearchApiResponse(json, "https://www.1tamilmv.style/"); + urls.Should().HaveCount(2); + urls[0].Should().Be("https://www.1tamilmv.style/index.php?/forums/topic/176409-leo-2023-official-release/"); + urls[1].Should().Be("https://www.1tamilmv.style/index.php?/forums/topic/176106-leo-2023-elite-release/"); + } + + [Test] + public void TestParseSearchApiResponse_EmptyOrInvalid() + { + TamilMV.ParseSearchApiResponse("", "https://www.1tamilmv.style/").Should().BeEmpty(); + TamilMV.ParseSearchApiResponse(" ", "https://www.1tamilmv.style/").Should().BeEmpty(); + TamilMV.ParseSearchApiResponse(@"{""results"":[], ""total"":0}", "https://www.1tamilmv.style/").Should().BeEmpty(); + } + + + [TestCase("The Truman Show (1998) 1080p BluRay", false)] + [TestCase("The Daily Show S29E50 720p", false)] + [TestCase("Regular Show S01 1080p", false)] + [TestCase("The Great Indian Kapil Show S01", false)] + [TestCase("The Interview (2014) 1080p BluRay", false)] + [TestCase("Song of the Sea (2014) 1080p", false)] + [TestCase("Swan Song (2021) 720p", false)] + [TestCase("Trailer Park Boys (2006) 720p", false)] + [TestCase("Show Dogs (2018) 1080p", false)] + [TestCase("Leo (2023) Tamil Proper TRUE WEB-DL - [4K, 1080p & 720p - AVC / HEVC]", false)] + [TestCase("Sundarakanda - Official Trailer | Nara Rohith", true)] + [TestCase("#Single Trailer | #SingleOnMay9th", true)] + [TestCase("Dragon Telugu Trailer", true)] + [TestCase("Leo Success Meet (2023) Tamil Full Show - [1080p & 720p]", true)] + [TestCase("Leo Success Meet (2023) Thalapathy Vijay Speech - [1080p]", true)] + [TestCase("Leo (2023) Tamil Full Album - [FLAC - 24Bit & 16Bit]", true)] + [TestCase("Dragon (2025) Original Motion Picture Soundtrack Master Quality 24bit - FLAC", true)] + [TestCase("Leo (2023) - Video Songs - TRUE WEB-DL - 4K SDR - HEVC", true)] + [TestCase("Leo - Villain Yaaru Video Song - 1080P - DD5.1", true)] + [TestCase("Vazhithunaiye Video Song | Dragon | Pradeep Ranganathan", true)] + [TestCase("Anbenum (From Leo) 3rd Single - (2023) Tamil MP3", true)] + [TestCase("Badass (From Leo) 2nd Single - (2023) Tamil MP3", true)] + [TestCase("Tu Mera Lover (From Mass Jathara) Telugu - Single Track - FLAC", true)] + [TestCase("Movie Teaser", true)] + [TestCase("Movie Sample.mkv", true)] + [TestCase("Movie Promo", true)] + public void TestExcludedTitles(string title, bool shouldExclude) + { + TamilMV.ExcludedTitles.IsMatch(title).Should().Be(shouldExclude); + } + + [Test] + public void TestExtractTopicReleases_FallbackToHeaderWhenSizeMissing() + { + var parser = new AngleSharp.Html.Parser.HtmlParser(); + var html = @" +
+

+ Trouble (2024) WEB-DL - 1080p - AVC - UNTOUCHED - Original Audios (DD+5.1 - 640Kbps) [Tamil + Telugu + Hindi + Eng] - 3.2GB - ESub : +
+ + + www.1TamilMV.tf - Trouble (2024) WEB-DL - 1080p - AVC - (DD+5.1 - 640Kbps) [Tam + Tel + Hin + Eng].mkv.torrent + + +
+ MAGNET +

+
"; + + using (var doc = parser.ParseDocument(html)) + { + var releases = TamilMV.ExtractTopicReleases(doc, "https://www.1tamilmv.style/index.php?/forums/topic/182865-trouble/", "https://www.1tamilmv.style/"); + + releases.Should().HaveCount(1); + var release = releases[0]; + release.Size.Should().Be(3.2.Gigabytes()); + release.Title.Should().Be("Trouble (2024) WEB-DL - 1080p - AVC - UNTOUCHED - Original Audios (DD+5.1 - 640Kbps) [Tamil + Telugu + Hindi + Eng] - 3.2GB - ESub.mkv"); + release.Category.Should().Contain(TorznabCatType.MoviesHD.ID); + } + } + + [Test] + public void TestExtractTopicReleases_StandardAnchorWithSizePreserved() + { + var parser = new AngleSharp.Html.Parser.HtmlParser(); + var html = @" +
+

+ Resident Evil (2026) HQ HDRip - 1080p - x264 - [Tamil + Telugu + Hindi + Eng] - (DD+5.1 - 640Kbps) - 3GB - ESub : +
+ + + www.1TamilMV.style - Resident Evil (2026) HQ HDRip - 1080p - x264 - [Tamil + Telugu + Hindi + Eng] - (DD+5.1 - 640Kbps) - 3GB - ESub.mkv.torrent + + +

+
"; + + using (var doc = parser.ParseDocument(html)) + { + var releases = TamilMV.ExtractTopicReleases(doc, "https://www.1tamilmv.style/index.php?/forums/topic/200720-resident-evil/", "https://www.1tamilmv.style/"); + + releases.Should().HaveCount(1); + var release = releases[0]; + release.Size.Should().Be(3.0.Gigabytes()); + release.Title.Should().Be("Resident Evil (2026) HQ HDRip - 1080p - x264 - [Tamil + Telugu + Hindi + Eng] - (DD+5.1 - 640Kbps) - 3GB - ESub.mkv"); + } + } + + [Test] + public void TestExtractTopicReleases_FallbackPreservesAnyExtension() + { + var parser = new AngleSharp.Html.Parser.HtmlParser(); + var html = @" +
+

+ Trouble (2024) WEB-DL - 720p - x264 - (AAC 2.0) - 850MB : +
+ + + www.1TamilMV.tf - Trouble (2024) WEB-DL - 720p - x264 - (AAC 2.0).mp4.torrent + + +

+
"; + + using (var doc = parser.ParseDocument(html)) + { + var releases = TamilMV.ExtractTopicReleases(doc, "https://www.1tamilmv.style/index.php?/forums/topic/182865-trouble/", "https://www.1tamilmv.style/"); + + releases.Should().HaveCount(1); + var release = releases[0]; + release.Size.Should().Be(850.Megabytes()); + release.Title.Should().Be("Trouble (2024) WEB-DL - 720p - x264 - (AAC 2.0) - 850MB.mp4"); + } + } + } +}