notebook_0_data_generation.ipynb

{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "# Setup\n",
    "\n",
    "If run remotely (e.g. on Google colab) we need to install some packages and also download our libraries"
   ],
   "id": "54ae0924b64b3dc4"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "%%capture\n",
    "!pip install numpy matplotlib seaborn matrixkit tensorflow"
   ],
   "id": "c7559ebcafe9fe27"
  },
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import modellib.io as io\n",
    "\n",
    "from matrixkit.core import MatrixData, ValueProperties, BlockProperties"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "# Generate Synthetic Data\n",
    "Generate a set of `n` square symmetrical and positive semi matrices of dimensions `MATRIX_DIM` to RAM.\n",
    "\n",
    "## Define Constants\n",
    "The following constants are used throughout the rest of the notebook."
   ],
   "id": "31c22a282c4f4a66"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "MATRIX_DIM = 64\n",
    "NUMBER_OF_MATRICES = 2000\n",
    "DIAGONAL_BAND_RADIUS = 10\n",
    "RNG_SEED = 42\n",
    "VALUE_COLORBAR = 'rocket'\n",
    "BLOCK_COLORBAR = 'flare'"
   ],
   "id": "189b0be883cc92f1",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "# Generate matrices\n",
    "bgr_noise_value_props = ValueProperties(density_range=(0.3, 0.5), value_range=(0.0, 0.5))\n",
    "noise_blk_value_props = ValueProperties(density_range=(0.3, 0.5), value_range=(0.3, 1.0))\n",
    "noise_blk_block_props = BlockProperties(size_range=(3, 32), size_average=10, size_std_dev=0.66, gap_chance=0.5)\n",
    "tdata_blk_value_props = ValueProperties(density_range=(0.5, 0.7), value_range=(0.3, 1.0))\n",
    "tdata_blk_block_props = BlockProperties(size_range=(2, 32), size_average=10, size_std_dev=0.66, gap_chance=0)\n",
    "\n",
    "matrix_data = MatrixData(\n",
    "        dimension=MATRIX_DIM,\n",
    "        band_radius=DIAGONAL_BAND_RADIUS,\n",
    "        sample_size=NUMBER_OF_MATRICES,\n",
    "        background_noise_value_properties=bgr_noise_value_props,\n",
    "        block_noise_value_properties=noise_blk_value_props,\n",
    "        block_noise_block_properties=noise_blk_block_props,\n",
    "        block_data_value_properties=tdata_blk_value_props,\n",
    "        block_data_block_properties=tdata_blk_block_props,\n",
    "        seed=42,\n",
    "        determinant_cutoff=0.01,\n",
    "        print_debug=True\n",
    "    )"
   ],
   "id": "93931f17e7c5df2c",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "# Extract matrices, bands and labels\n",
    "bands = matrix_data.bands.reshape(NUMBER_OF_MATRICES, (DIAGONAL_BAND_RADIUS* 2 + 1), MATRIX_DIM, 1)\n",
    "bands = np.nan_to_num(bands) # replace NaNs with 0s\n",
    "labels = matrix_data.tdata_blk_starts\n",
    "print(f\"Matrix Bands Shape: {bands.shape}, Matrix Labels Shape: {labels.shape}\")\n",
    "\n",
    "# Create dataset from matrices and labels\n",
    "dataset = tf.data.Dataset.from_tensor_slices((bands, labels))\n",
    "\n",
    "# Split the dataset\n",
    "train_size = int(0.8 * NUMBER_OF_MATRICES)\n",
    "val_size = int(0.1 * NUMBER_OF_MATRICES)\n",
    "test_size = NUMBER_OF_MATRICES - train_size - val_size\n",
    "\n",
    "train_dataset = dataset.take(train_size).shuffle(buffer_size=10)\n",
    "val_dataset = dataset.skip(train_size).take(val_size)\n",
    "test_dataset = dataset.skip(train_size + val_size)\n",
    "\n",
    "# Create test matrices and labels separately to save\n",
    "test_matrices = matrix_data.matrices[train_size + val_size:]\n",
    "test_labels = labels[train_size + val_size:]\n",
    "\n",
    "# Verify Shape\n",
    "print(f\"Test Matrices Shape: {test_matrices.shape}, Test Labels Shape: {test_labels.shape}\")\n",
    "print(f\"Train size: {train_size}, Val size: {val_size}, Test size: {test_size}\")"
   ],
   "id": "2bb3fbbb381033f5",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Save original matrices",
   "id": "24ea75a668911128"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "# Save original matrices \n",
    "np.save(\"data/matrices/all_matrices_64x64_2000.npy\", matrix_data.matrices)\n",
    "np.save(\"data/matrices/test_matrices_64x64_200.npy\", test_matrices)"
   ],
   "id": "53bb50c086192cf",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Save datasets ",
   "id": "d03c2bfc2748e05c"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "io.save_to_hdf5(train_dataset, \"data/datasets/train_dataset_64_1600.h5\")\n",
    "io.save_to_hdf5(val_dataset, \"data/datasets/val_dataset_64_200.h5\")\n",
    "io.save_to_hdf5(test_dataset, \"data/datasets/test_dataset_64_200.h5\")"
   ],
   "id": "5ff96fe28c115694",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# Load matrices for verification",
   "id": "358da693a860c3f0"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "# Reading the datasets\n",
    "train_bands, train_labels = io.read_from_hdf5(\"data/datasets/train_dataset_64_1600.h5\")\n",
    "val_bands, val_labels = io.read_from_hdf5(\"data/datasets/val_dataset_64_200.h5\")\n",
    "test_bands, test_labels = io.read_from_hdf5(\"data/datasets/test_dataset_64_200.h5\")\n",
    "\n",
    "# Printing shapes to verify\n",
    "print(f\"Train bands shape: {train_bands.shape}, Train labels shape: {train_labels.shape}\")\n",
    "print(f\"Validation bands shape: {val_bands.shape}, Validation labels shape: {val_labels.shape}\")\n",
    "print(f\"Test bands shape: {test_bands.shape}, Test labels shape: {test_labels.shape}\")"
   ],
   "id": "83f41ea5576ec573",
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}