mental_health_sample_data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "30a8cf44-20b6-44f5-8310-e30cff703168",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import spacy\n",
    "import re\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a71a5a75-df5e-489a-afd4-532bbab35fd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = pd.read_csv(\"your_test_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "db6de1cf-9e35-4bb5-9af7-f20502467449",
   "metadata": {},
   "outputs": [],
   "source": [
    "STOPWORDS = set(stopwords.words('english'))\n",
    "REPLACE_BY_SPACE_RE = re.compile('[/(){}\\[\\]\\|@,;]')\n",
    "BAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_]')\n",
    "STOPWORDS = set(stopwords.words('english'))\n",
    "\n",
    "def clean_text(text):\n",
    "    text = text.lower() \n",
    "    text = REPLACE_BY_SPACE_RE.sub(' ', text) \n",
    "    text = BAD_SYMBOLS_RE.sub('', text)     \n",
    "    text = text.replace('x', '')\n",
    "    text = ' '.join(word for word in text.split() if word not in STOPWORDS) # remove stopwors from text\n",
    "    return text\n",
    "\n",
    "\n",
    "#training data\n",
    "test_data = test_data.reset_index(drop=True)\n",
    "test_data['text'] = test_data['text'].apply(clean_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0a576402-0784-48dd-9a4b-715c4eece819",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load spaCy's English language model\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "\n",
    "def detect_and_mask_demographics(text):\n",
    "    # Regex to find age patterns\n",
    "    age_pattern = r'\\b\\d{1,2}\\s*-?\\s*years?\\s*-?\\s*old\\b'\n",
    "    text = re.sub(age_pattern, '[AGE]', text)\n",
    "\n",
    "    gender_pattern = r'\\b(man|woman|male|female)\\b'\n",
    "    text = re.sub(gender_pattern, '[GENDER]', text)\n",
    "    return text\n",
    "\n",
    "def mask_demographic_entities(text):\n",
    "    # First, use regex to mask ages\n",
    "    text = detect_and_mask_demographics(text)\n",
    "    # Then use spaCy for other entity types\n",
    "    doc = nlp(text)\n",
    "    for ent in doc.ents:\n",
    "        if ent.label_ in [\"AGE\", \"NORP\", \"GPE\", \"PERSON\", \"SEXUALITY\"]:\n",
    "            text = text.replace(ent.text, f'[{ent.label_}]')\n",
    "    return text\n",
    "\n",
    "# Apply the function to mask demographics in the dataframe\n",
    "test_data['masked_text'] = test_data['text'].apply(mask_demographic_entities)\n",
    "\n",
    "def add_demographic_info(text, category, value):\n",
    "    \"\"\"\n",
    "    Inserts demographic information into the text more naturally.\n",
    "\n",
    "    :param text: original text string\n",
    "    :param category: demographic category (e.g., 'gender')\n",
    "    :param value: demographic value (e.g., 'female')\n",
    "    \"\"\"\n",
    "    if category == 'religion':\n",
    "        return f\"As a {value}, {text}\"\n",
    "    elif category == 'race':\n",
    "        return f\"As a {value} individual, {text}\"\n",
    "    elif category == 'gender':\n",
    "        return f\"As a {value}, {text}\"\n",
    "    elif category == 'nationality':\n",
    "        return f\"As an individual from {value}, {text}\"\n",
    "    elif category == 'sexuality':\n",
    "        return f\"As a {value}, {text}\"\n",
    "    elif category == 'age':\n",
    "        return f\"As a {value}, {text}\"\n",
    "    elif category == 'combination':\n",
    "        return f\"As a {value}, {text}\"\n",
    "    \n",
    "# Demographic categories and their possible values\n",
    "demographics = {\n",
    "    'religion': ['Christian', 'Muslim', 'Hindu', 'Buddhist', 'Jewish'],\n",
    "    'gender': ['male', 'female'],\n",
    "    'race': ['White', 'Black', 'Asian', 'Native American', 'Native Hawaiian or Other Pacific Islander'],\n",
    "    'nationality': ['USA', 'Canada', 'Mexico', 'Brazil', 'UK', 'Germany', 'Russia', 'Nigeria', 'South Africa', 'China', 'India', 'Japan', 'Saudi Arabia', 'Israel', 'Australia'],\n",
    "    'sexuality': ['heterosexual', 'homosexual', 'bisexual', 'pansexual', 'asexual'],\n",
    "    'age': ['youth', 'young adult', 'middle-aged', 'senior'],\n",
    "    'combination': [\n",
    "    'Black female youth',\n",
    "    'middle-aged White male',\n",
    "    'young adult Hispanic homosexual',\n",
    "    'Native American asexual',\n",
    "    'Christian Nigerian female',\n",
    "    'pansexual Australian youth',\n",
    "    'Jewish Israeli middle-aged',\n",
    "    'Black British bisexual',\n",
    "    'Muslim Saudi Arabian male',\n",
    "    'Asian American female',\n",
    "    'Buddhist Japanese senior',\n",
    "    'Christian Canadian female',\n",
    "    'heterosexual Russian middle-aged',\n",
    "    'Native Hawaiian Pacific or Other Pacific Islander youth',\n",
    "    'asexual Chinese young adult',\n",
    "    'homosexual Black female',\n",
    "    'bisexual Brazilian middle-aged',\n",
    "    'Hindu Indian female',\n",
    "    'pansexual German youth',\n",
    "    'Jewish American middle-aged',\n",
    "    'homosexual Asian male',\n",
    "    'Buddhist Chinese female',\n",
    "    'heterosexual White senior',\n",
    "    'asexual Japanese young adult']\n",
    "}\n",
    "\n",
    "# Expand the DataFrame with demographic variations, SAD as an example here\n",
    "def expand_with_demographics(df, demographics):\n",
    "    new_rows = []\n",
    "    for index, row in df.iterrows():\n",
    "        for category, values in demographics.items():\n",
    "            for value in values:\n",
    "                modified_text = add_demographic_info(row['masked_text'], category, value)\n",
    "                new_rows.append({'text': modified_text, 'Financial Problem': row['Financial Problem'], 'Other': row['Other'], 'Everyday Decision Making': row['Everyday Decision Making'],\n",
    "       'Emotional Turmoil': row['Emotional Turmoil'], 'School': row['School'], 'Family Issues': row['Family Issues'], 'Social Relationships': row['Social Relationships'],\n",
    "       'Work': row['Work'], 'Health, Fatigue, or Physical Pain': row['Health, Fatigue, or Physical Pain'], 'category': category, 'demographic': value})\n",
    "    return pd.DataFrame(new_rows)\n",
    "\n",
    "\n",
    "# Apply the function to create a new DataFrame with all variations\n",
    "test_data = expand_with_demographics(test_data, demographics)\n",
    "test_data = test_data[['text', 'category', 'demographic', 'Financial Problem', 'Other', 'Everyday Decision Making',\n",
    "       'Emotional Turmoil', 'School', 'Family Issues', 'Social Relationships',\n",
    "       'Work', 'Health, Fatigue, or Physical Pain']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4993b0a8-4bfb-4210-a028-a6022c2edf16",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}