From 81050d3d9c32e156db42666d6deb5e69beb7a7a0 Mon Sep 17 00:00:00 2001
From: azhuge233 <17763056+azhuge233@users.noreply.github.com>
Date: Tue, 14 Jan 2025 14:56:44 +0800
Subject: [PATCH] Try to fix block by adding cookie.

---
 RedditFreeGamesNotifier/Services/Scraper.cs   | 45 ++++++++++++++++---
 .../Strings/ScrapeStrings.cs                  | 12 +++--
 2 files changed, 46 insertions(+), 11 deletions(-)

diff --git a/RedditFreeGamesNotifier/Services/Scraper.cs b/RedditFreeGamesNotifier/Services/Scraper.cs
index 351bfb7..16b2f5d 100644
--- a/RedditFreeGamesNotifier/Services/Scraper.cs
+++ b/RedditFreeGamesNotifier/Services/Scraper.cs
@@ -1,15 +1,16 @@
-using Microsoft.Extensions.Logging;
+using System.Net;
+using Microsoft.Extensions.Logging;
 using RedditFreeGamesNotifier.Strings;
 
 namespace RedditFreeGamesNotifier.Services {
 	internal class Scraper: IDisposable {
 		private readonly ILogger<Scraper> _logger;
 
-		internal HttpClient Client { get; set; } = new HttpClient();
+		internal HttpClient RedditClient { get; set; }
+		internal HttpClient OthersClient { get; set; } = new HttpClient();
 
 		public Scraper(ILogger<Scraper> logger) {
 			_logger = logger;
-			Client.DefaultRequestHeaders.Add("User-Agent", ScrapeStrings.UAs[new Random().Next(0, ScrapeStrings.UAs.Length - 1)]);
 		}
 
 		internal async Task<Dictionary<string, string>> GetSource() {
@@ -17,11 +18,23 @@ internal async Task<Dictionary<string, string>> GetSource() {
 				_logger.LogDebug(ScrapeStrings.debugGetSource);
 				var results = new Dictionary<string, string>();
 
+				RedditClient = new HttpClient(
+					new HttpClientHandler() {
+						CookieContainer = CreateCookie()
+					}
+				);
+
+				RedditClient.DefaultRequestHeaders.Add("User-Agent", ScrapeStrings.UAs[new Random().Next(0, ScrapeStrings.UAs.Length - 1)]);
+
 				foreach (var url in ScrapeStrings.RedditUrls) {
 					_logger.LogDebug(ScrapeStrings.debugGetSourceWithUrl, url);
-					var response = await Client.GetAsync(url);
-					results.Add(url, await response.Content.ReadAsStringAsync());
-					Task.Delay(new Random().Next(500, 800)).Wait();
+					var response = await RedditClient.GetAsync(url);
+					var content = await response.Content.ReadAsStringAsync();
+					results.Add(url, content);
+
+					// _logger.LogDebug($"response content: {content}");
+
+					await Task.Delay(new Random().Next(500, 800));
 				}
 
 				_logger.LogDebug($"Done: {ScrapeStrings.debugGetSource}");
@@ -39,7 +52,7 @@ internal async Task<string> GetSource(string url) {
 				_logger.LogDebug(ScrapeStrings.debugGetSource);
 
 				_logger.LogDebug(ScrapeStrings.debugGetSourceWithUrl, url);
-				var response = await Client.GetAsync(url);
+				var response = await OthersClient.GetAsync(url);
 				var result = await response.Content.ReadAsStringAsync();
 
 				_logger.LogDebug($"Done: {ScrapeStrings.debugGetSource}");
@@ -52,6 +65,24 @@ internal async Task<string> GetSource(string url) {
 			}
 		}
 
+		private static CookieContainer CreateCookie() { 
+			var cookieContainer = new CookieContainer();
+
+			var redditCookie = new Cookie() {
+				Name = ScrapeStrings.RedditCookieName,
+				Value = string.Empty,
+				Domain = ScrapeStrings.RedditCookieDomain,
+				Path = ScrapeStrings.RedditCookiePath,
+				Expires = DateTime.Now.AddYears(1),
+				Secure = true,
+				HttpOnly = true
+			};
+
+			cookieContainer.Add(redditCookie);
+
+			return cookieContainer;
+		}
+
 		public void Dispose() { 
 			GC.SuppressFinalize(this);
 		}
diff --git a/RedditFreeGamesNotifier/Strings/ScrapeStrings.cs b/RedditFreeGamesNotifier/Strings/ScrapeStrings.cs
index 716fa13..eaa5814 100644
--- a/RedditFreeGamesNotifier/Strings/ScrapeStrings.cs
+++ b/RedditFreeGamesNotifier/Strings/ScrapeStrings.cs
@@ -1,15 +1,15 @@
 namespace RedditFreeGamesNotifier.Strings {
 	internal class ScrapeStrings {
-		internal static List<string> RedditUrls { get; set; } = new() {
+		internal static List<string> RedditUrls { get; set; } = [
 			"https://old.reddit.com/r/FreeGamesOnSteam/new/",
 			"https://old.reddit.com/r/FreeGameFindings/new/",
 			"https://old.reddit.com/r/freegames/new/",
 			//"https://old.reddit.com/r/FreeGamesOnSteam/",
 			//"https://old.reddit.com/r/FreeGameFindings/",
 			//"https://old.reddit.com/r/freegames/"
-		};
+		];
 
-		internal static string[] UAs = new string[] {
+		internal static string[] UAs = [
 			//"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
 			//"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/112.0.0.0 Safari/537.36",
 			//"RedditApp/614658 CFNetwork/1494.0.7 Darwin/23.4.0",
@@ -18,7 +18,11 @@ internal class ScrapeStrings {
 			"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36 OPR/65.0.3467.48",
 			"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
 			//"Reddit/Version 2024.18.0/Build 614658/iOS Version 17.4.1 (Build 21E236)"
-		};
+		];
+
+		internal static readonly string RedditCookieName = "reddit_session";
+		internal static readonly string RedditCookieDomain = ".reddit.com";
+		internal static readonly string RedditCookiePath = "/";
 
 		#region debug strings
 		internal static readonly string debugGetSource = "Get source";