Try to fix block by adding cookie.

azhuge233 · Jan 14, 2025 · 81050d3 · 81050d3
1 parent 1646049
commit 81050d3
Show file tree

Hide file tree

Showing 2 changed files with 46 additions and 11 deletions.
diff --git a/RedditFreeGamesNotifier/Services/Scraper.cs b/RedditFreeGamesNotifier/Services/Scraper.cs
@@ -1,27 +1,40 @@
-using Microsoft.Extensions.Logging;
+using System.Net;
+using Microsoft.Extensions.Logging;
 using RedditFreeGamesNotifier.Strings;
 
 namespace RedditFreeGamesNotifier.Services {
 	internal class Scraper: IDisposable {
 		private readonly ILogger<Scraper> _logger;
 
-		internal HttpClient Client { get; set; } = new HttpClient();
+		internal HttpClient RedditClient { get; set; }
+		internal HttpClient OthersClient { get; set; } = new HttpClient();
 
 		public Scraper(ILogger<Scraper> logger) {
 			_logger = logger;
-			Client.DefaultRequestHeaders.Add("User-Agent", ScrapeStrings.UAs[new Random().Next(0, ScrapeStrings.UAs.Length - 1)]);
 		}
 
 		internal async Task<Dictionary<string, string>> GetSource() {
 			try {
 				_logger.LogDebug(ScrapeStrings.debugGetSource);
 				var results = new Dictionary<string, string>();
 
+				RedditClient = new HttpClient(
+					new HttpClientHandler() {
+						CookieContainer = CreateCookie()
+					}
+				);
+
+				RedditClient.DefaultRequestHeaders.Add("User-Agent", ScrapeStrings.UAs[new Random().Next(0, ScrapeStrings.UAs.Length - 1)]);
+
 				foreach (var url in ScrapeStrings.RedditUrls) {
 					_logger.LogDebug(ScrapeStrings.debugGetSourceWithUrl, url);
-					var response = await Client.GetAsync(url);
-					results.Add(url, await response.Content.ReadAsStringAsync());
-					Task.Delay(new Random().Next(500, 800)).Wait();
+					var response = await RedditClient.GetAsync(url);
+					var content = await response.Content.ReadAsStringAsync();
+					results.Add(url, content);
+
+					// _logger.LogDebug($"response content: {content}");
+
+					await Task.Delay(new Random().Next(500, 800));
 				}
 
 				_logger.LogDebug($"Done: {ScrapeStrings.debugGetSource}");
@@ -39,7 +52,7 @@ internal async Task<string> GetSource(string url) {
 				_logger.LogDebug(ScrapeStrings.debugGetSource);
 
 				_logger.LogDebug(ScrapeStrings.debugGetSourceWithUrl, url);
-				var response = await Client.GetAsync(url);
+				var response = await OthersClient.GetAsync(url);
 				var result = await response.Content.ReadAsStringAsync();
 
 				_logger.LogDebug($"Done: {ScrapeStrings.debugGetSource}");
@@ -52,6 +65,24 @@ internal async Task<string> GetSource(string url) {
 			}
 		}
 
+		private static CookieContainer CreateCookie() { 
+			var cookieContainer = new CookieContainer();
+
+			var redditCookie = new Cookie() {
+				Name = ScrapeStrings.RedditCookieName,
+				Value = string.Empty,
+				Domain = ScrapeStrings.RedditCookieDomain,
+				Path = ScrapeStrings.RedditCookiePath,
+				Expires = DateTime.Now.AddYears(1),
+				Secure = true,
+				HttpOnly = true
+			};
+
+			cookieContainer.Add(redditCookie);
+
+			return cookieContainer;
+		}
+
 		public void Dispose() { 
 			GC.SuppressFinalize(this);
 		}

diff --git a/RedditFreeGamesNotifier/Strings/ScrapeStrings.cs b/RedditFreeGamesNotifier/Strings/ScrapeStrings.cs
@@ -1,15 +1,15 @@
 namespace RedditFreeGamesNotifier.Strings {
 	internal class ScrapeStrings {
-		internal static List<string> RedditUrls { get; set; } = new() {
+		internal static List<string> RedditUrls { get; set; } = [
 			"https://old.reddit.com/r/FreeGamesOnSteam/new/",
 			"https://old.reddit.com/r/FreeGameFindings/new/",
 			"https://old.reddit.com/r/freegames/new/",
 			//"https://old.reddit.com/r/FreeGamesOnSteam/",
 			//"https://old.reddit.com/r/FreeGameFindings/",
 			//"https://old.reddit.com/r/freegames/"
-		};
+		];
 
-		internal static string[] UAs = new string[] {
+		internal static string[] UAs = [
 			//"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
 			//"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/112.0.0.0 Safari/537.36",
 			//"RedditApp/614658 CFNetwork/1494.0.7 Darwin/23.4.0",
@@ -18,7 +18,11 @@ internal class ScrapeStrings {
 			"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36 OPR/65.0.3467.48",
 			"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
 			//"Reddit/Version 2024.18.0/Build 614658/iOS Version 17.4.1 (Build 21E236)"
-		};
+		];
+
+		internal static readonly string RedditCookieName = "reddit_session";
+		internal static readonly string RedditCookieDomain = ".reddit.com";
+		internal static readonly string RedditCookiePath = "/";
 
 		#region debug strings
 		internal static readonly string debugGetSource = "Get source";