diff --git a/backend/fastapi/Autocomplete.py b/backend/fastapi/Autocomplete.py index ac03981c..6d8c13a9 100644 --- a/backend/fastapi/Autocomplete.py +++ b/backend/fastapi/Autocomplete.py @@ -2,7 +2,7 @@ import pymongo -from helpers import get_guild_collection +from helpers import get_blacklisted_user_ids, get_guild_collection def autocomplete_categories(guild_id: str, partial_category: str, limit: int): @@ -208,6 +208,8 @@ def autocomplete_users(guild_id: str, partial_user_name: str, limit: int): collection_authors = get_guild_collection(guild_id, "authors") print("collection_authors", collection_authors) + blacklisted_user_ids = get_blacklisted_user_ids() + query = { "$or": [ { @@ -220,7 +222,10 @@ def autocomplete_users(guild_id: str, partial_user_name: str, limit: int): "$regex": partial_user_name, "$options": "i" } } - ] + ], + "_id": { + "$nin": blacklisted_user_ids + } } cursor = collection_authors.find(query, { "names": 1, diff --git a/backend/fastapi/app.py b/backend/fastapi/app.py index 7dcee26e..1acbdd25 100644 --- a/backend/fastapi/app.py +++ b/backend/fastapi/app.py @@ -1,4 +1,5 @@ import datetime +import shutil from dateutil.relativedelta import relativedelta import functools import json @@ -12,7 +13,7 @@ from pydantic import BaseModel import Autocomplete -from helpers import get_global_collection, get_whitelisted_guild_ids, is_db_online, pad_id, get_guild_collection +from helpers import get_blacklisted_user_ids, get_global_collection, get_whitelisted_guild_ids, is_db_online, pad_id, get_guild_collection # fix PIPE encoding error on Windows, auto flush print sys.stdout.reconfigure(encoding='utf-8') @@ -137,28 +138,48 @@ async def get_message_ids(channel_id: str, guild_id: str): """ collection_messages = get_guild_collection(guild_id, "messages") if is_compiled(): - cache_path = f"../../storage/cache/message-ids/{channel_id}.json" + cache_dir = "../../storage/cache/message-ids" else: - cache_path = f"../../release/dcef/storage/cache/message-ids/{channel_id}.json" + cache_dir = "../../release/dcef/storage/cache/message-ids" + + cache_path = f"{cache_dir}/{channel_id}.json" + blacklisted_user_ids_path = f"{cache_dir}/blacklisted_user_ids.json" + + # clear entire cache if blacklisted user ids changed + blacklisted_user_ids = get_blacklisted_user_ids() + if os.path.exists(cache_path): + with open(blacklisted_user_ids_path, "r", encoding="utf-8") as f: + file_content = f.read() + + if file_content != str(blacklisted_user_ids): + shutil.rmtree(cache_dir) + os.makedirs(cache_dir) # read cached ids if available if os.path.exists(cache_path): + print("get_message_ids() cache hit - channel id", channel_id) with open(cache_path, "r", encoding="utf-8") as f: - print("get_message_ids() cache hit - channel id", channel_id) file_content = f.read() return json.loads(file_content) print("get_message_ids() cache miss - channel id", channel_id) query = { - "channelId": channel_id + "channelId": channel_id, + "author._id": { + "$nin": blacklisted_user_ids + } } ids = collection_messages.find(query, {"_id": 1}).sort([("_id", pymongo.ASCENDING)]) new_ids = [str(id["_id"]) for id in ids] # save to cache + file_content = re.sub(r"'", '"', str(new_ids)) with open(cache_path, "w", encoding="utf-8") as f: - file_content = re.sub(r"'", '"', str(new_ids)) + f.write(file_content) + + file_content = re.sub(r"'", '"', str(blacklisted_user_ids)) + with open(blacklisted_user_ids_path, "w", encoding="utf-8") as f: f.write(file_content) return new_ids @@ -179,11 +200,15 @@ async def get_multiple_message_content(message_req_obj: MessageRequest): guild_id = message_req_obj.guild_id collection_messages = get_guild_collection(guild_id, "messages") + blacklisted_user_ids = get_blacklisted_user_ids() messages = collection_messages.find( { "_id": { "$in": message_ids + }, + "author._id": { + "$nin": blacklisted_user_ids } } ) @@ -836,16 +861,21 @@ async def search_messages(guild_id: str, prompt: str = None, only_ids: bool = Tr message_contains = [word.lower() for word in message_contains] message_ids = [pad_id(id) for id in message_ids] + blacklisted_user_ids = get_blacklisted_user_ids() from_user_ids = [pad_id(id) for id in from_user_ids] from_user_ids = extend_users(from_user_ids, from_users, guild_id) + from_user_ids = [user_id for user_id in from_user_ids if user_id not in blacklisted_user_ids] # remove blacklisted users print("from_user_ids", from_user_ids) reaction_from_ids = [pad_id(id) for id in reaction_from_ids] reaction_from_ids = extend_users(reaction_from_ids, reaction_from, guild_id) + reaction_from_ids = [user_id for user_id in mentions_user_ids if user_id not in blacklisted_user_ids] # remove blacklisted users mentions_user_ids = [pad_id(id) for id in mentions_user_ids] mentions_user_ids = extend_users(mentions_user_ids, mentions_users, guild_id) + mentions_user_ids = [user_id for user_id in mentions_user_ids if user_id not in blacklisted_user_ids] # remove blacklisted users + reaction_ids = [pad_id(id) for id in reaction_ids] reactions = [reaction.lower() for reaction in reactions] reaction_ids = extend_reactions(reaction_ids, reactions, guild_id) @@ -881,6 +911,9 @@ async def search_messages(guild_id: str, prompt: str = None, only_ids: bool = Tr if len(from_user_ids) > 0: query["author._id"] = {"$in": from_user_ids} + if len(blacklisted_user_ids) > 0: + query["author._id"] = {"$nin": blacklisted_user_ids} + if len(reaction_from_ids) > 0: query["reactions.users._id"] = {"$in": reaction_from_ids} diff --git a/backend/fastapi/helpers.py b/backend/fastapi/helpers.py index caab566b..025d1dd9 100644 --- a/backend/fastapi/helpers.py +++ b/backend/fastapi/helpers.py @@ -19,6 +19,10 @@ def get_whitelisted_guild_ids(): whitelisted_guild_ids = [pad_id(id) for id in whitelisted_guild_ids] return whitelisted_guild_ids +def get_blacklisted_user_ids(): + blacklisted_user_ids = collection_config.find_one({"key": "blacklisted_user_ids"})["value"] + blacklisted_user_ids = [pad_id(id) for id in blacklisted_user_ids] + return blacklisted_user_ids def get_guild_collection(guild_id, collection_name): diff --git a/backend/preprocess/main_mongo.py b/backend/preprocess/main_mongo.py index a9f6a81b..b2b5e17c 100644 --- a/backend/preprocess/main_mongo.py +++ b/backend/preprocess/main_mongo.py @@ -1,10 +1,7 @@ import os import sys -import requests import functools -from pprint import pprint - from FileFinder import FileFinder from MongoDatabase import MongoDatabase from ChannelCache import ChannelCache @@ -33,6 +30,10 @@ def wipe_database(database: MongoDatabase): if whitelisted_guild_ids is None: config.insert_one({"key": "whitelisted_guild_ids", "value": []}) + blacklisted_user_ids = config.find_one({"key": "blacklisted_user_ids"}) + if blacklisted_user_ids is None: + config.insert_one({"key": "blacklisted_user_ids", "value": []}) + version = config.find_one({"key": "version"}) if version is None: version = {"key": "version", "value": 0}