DeepFM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tensorflow.keras.layers import *\n",
    "import tensorflow.keras.backend as K\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.utils import plot_model\n",
    "from tensorflow.keras.callbacks import *\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "# from tensorflow.keras.constraints import *\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/disk/share/criteo/'\n",
    "data = pd.read_csv(path+'criteo_sampled_data.csv')\n",
    "cols = data.columns.values\n",
    "\n",
    "dense_feats = [f for f in cols if f[0] == \"I\"]\n",
    "sparse_feats = [f for f in cols if f[0] == \"C\"]\n",
    "\n",
    "def process_dense_feats(data, feats):\n",
    "    d = data.copy()\n",
    "    d = d[feats].fillna(0.0)\n",
    "    for f in feats:\n",
    "        d[f] = d[f].apply(lambda x: np.log(x+1) if x > -1 else -1)\n",
    "    \n",
    "    return d\n",
    "\n",
    "data_dense = process_dense_feats(data, dense_feats)\n",
    "\n",
    "vocab_sizes = {}\n",
    "def process_sparse_feats(data, feats):\n",
    "    d = data.copy()\n",
    "    d = d[feats].fillna(\"-1\")\n",
    "    for f in feats:\n",
    "        label_encoder = LabelEncoder()\n",
    "        d[f] = label_encoder.fit_transform(d[f])\n",
    "        vocab_sizes[f] = d[f].nunique() + 1\n",
    "    return d\n",
    "\n",
    "data_sparse = process_sparse_feats(data, sparse_feats)\n",
    "total_data = pd.concat([data_dense, data_sparse], axis=1)\n",
    "total_data['label'] = data['label']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 自定义层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SparseEmbedding(Layer):\n",
    "    def __init__(self, sparse_feats, vocab_sizes, embed_dims=8):\n",
    "        super().__init__()\n",
    "        # 离散特征嵌入矩阵\n",
    "        self.sparse_embeds_mat = []\n",
    "        for idx, feat in enumerate(sparse_feats):\n",
    "            # reg = tf.keras.regularizers.l2(0.5)\n",
    "            emb = Embedding(input_dim=vocab_sizes[feat],\n",
    "                            output_dim=embed_dims,\n",
    "                            # embeddings_regularizer=reg,\n",
    "                            name=f'{feat}_emb')\n",
    "            self.sparse_embeds_mat.append(emb)\n",
    "        \n",
    "    def call(self, sparse_inputs):\n",
    "        # FM 部分\n",
    "        sparse_embeds = []\n",
    "        for idx, emb_mat in enumerate(self.sparse_embeds_mat):\n",
    "            emb = emb_mat(sparse_inputs[idx])\n",
    "            sparse_embeds.append(emb)\n",
    "        concat_sparse_embeds = Concatenate(axis=1)(sparse_embeds)\n",
    "        return concat_sparse_embeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Linear(Layer):\n",
    "    def __init__(self, sparse_feats, vocab_sizes):\n",
    "        super().__init__()\n",
    "        \n",
    "        # 离散特1d征嵌入矩阵\n",
    "        self.sparse_1d_embeds = SparseEmbedding(sparse_feats, vocab_sizes, embed_dims=1)\n",
    "        \n",
    "        self.fc_dense = Dense(1)\n",
    "        self.fc_sparse = Dense(1)\n",
    "        \n",
    "    def call(self, inputs):\n",
    "        dense_inputs, sparse_inputs = inputs[0], inputs[1]\n",
    "        # 线性部分\n",
    "        concat_dense_inputs = Concatenate(axis=1)(dense_inputs)\n",
    "        first_order_dense_layer = self.fc_dense(concat_dense_inputs)\n",
    "        \n",
    "        concat_sparse_embeds_1d = self.sparse_1d_embeds(sparse_inputs) \n",
    "        flat_sparse_embeds_1d = Flatten()(concat_sparse_embeds_1d)\n",
    "        first_order_sparse_layer = self.fc_sparse(flat_sparse_embeds_1d)\n",
    "        \n",
    "        linear_output = Add()([first_order_dense_layer, first_order_sparse_layer])\n",
    "        return linear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FM(Layer):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "    def call(self, concat_sparse_embeds):\n",
    "        # 先求和再求平方\n",
    "        sum_embeds = tf.reduce_sum(concat_sparse_embeds, axis=1)\n",
    "        square_sum_embeds = Multiply()([sum_embeds, sum_embeds])\n",
    "        # 先平方再求和\n",
    "        square_embeds = Multiply()([concat_sparse_embeds, concat_sparse_embeds])\n",
    "        sum_square_embeds = tf.reduce_sum(square_embeds, axis=1)\n",
    "        # 相减除以2\n",
    "        sub =  0.5 * Subtract()([square_sum_embeds, sum_square_embeds])\n",
    "        # 相加\n",
    "        snd_order_sparse_output = tf.reduce_sum(sub, axis=1, keepdims=True)\n",
    "        return snd_order_sparse_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DNN(Layer):\n",
    "    def __init__(self, hid_units=[256,256,256], use_dropout=True):\n",
    "        super().__init__()\n",
    "        self.use_dropout = use_dropout\n",
    "        self.Dropout = Dropout(0.3)\n",
    "        self.dense_layers = []\n",
    "        for unit in hid_units:\n",
    "            self.dense_layers.append(Dense(unit, activation='relu'))\n",
    "        self.dense_layers.append(Dense(1))\n",
    "        \n",
    "    def call(self, concat_sparse_embeds):\n",
    "        flat_sparse_embed = Flatten()(concat_sparse_embeds)\n",
    "        \n",
    "        x = self.dense_layers[0](flat_sparse_embed)\n",
    "        for dense in self.dense_layers[1:]:\n",
    "            x = dense(x)\n",
    "            if self.use_dropout:\n",
    "                x = self.Dropout(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 构建模型 (keras函数式)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DeepFM:\n",
    "    def __init__(self, dense_feats, sparse_feats, vocab_sizes, embed_dims=8):\n",
    "        \n",
    "        # 连续特征\n",
    "        self.dense_inputs = []\n",
    "        for feat in dense_feats:\n",
    "            self.dense_inputs.append(Input(shape=1, name=feat))\n",
    "            \n",
    "        # 离散特征\n",
    "        self.sparse_inputs = []\n",
    "        for feat in sparse_feats:\n",
    "            self.sparse_inputs.append(Input(shape=1, name=feat))\n",
    "        \n",
    "        self.Linear = Linear(sparse_feats, vocab_sizes)\n",
    "        self.SparseEmbedding = SparseEmbedding(sparse_feats, vocab_sizes, embed_dims=8)\n",
    "        self.FM = FM()\n",
    "        self.DNN = DNN()\n",
    "        \n",
    "    def bulid_model(self):\n",
    "        all_inputs = [self.dense_inputs, self.sparse_inputs]\n",
    "        \n",
    "        linear_output = self.Linear(all_inputs)\n",
    "        concat_sparse_embeds = self.SparseEmbedding(self.sparse_inputs)\n",
    "        snd_order_sparse_output = self.FM(concat_sparse_embeds)\n",
    "        fc_layer_output = self.DNN(concat_sparse_embeds)\n",
    "        \n",
    "        # 输出部分\n",
    "        output = Add()([linear_output, snd_order_sparse_output, fc_layer_output])\n",
    "        output = Activation('sigmoid')(output)\n",
    "        \n",
    "        model = Model(inputs=all_inputs, outputs=output)\n",
    "        return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_data = total_data.loc[:500000-1]\n",
    "valid_data = total_data.loc[500000:]\n",
    "\n",
    "train_dense_x_all = [train_data[f].values for f in dense_feats]\n",
    "train_sparse_x_all = [train_data[f].values for f in sparse_feats]\n",
    "train_label_all = train_data[['label']].values\n",
    "\n",
    "val_dense_x_all = [valid_data[f].values for f in dense_feats]\n",
    "val_sparse_x_all = [valid_data[f].values for f in sparse_feats]\n",
    "val_label_all = valid_data[['label']].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1954/1954 [==============================] - 27s 14ms/step - loss: 0.5171 - binary_crossentropy: 0.5171 - auc: 0.7346 - val_loss: 0.4816 - val_binary_crossentropy: 0.4816 - val_auc: 0.7663 - lr: 0.0010\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fc618863450>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = DeepFM(dense_feats, sparse_feats, vocab_sizes).bulid_model()\n",
    "model.compile(optimizer='rmsprop', loss='binary_crossentropy', \n",
    "              metrics=['binary_crossentropy', tf.keras.metrics.AUC()])\n",
    "\n",
    "os.makedirs('checkpoints/model.h5', exist_ok=True)\n",
    "checkpoints = ModelCheckpoint('checkpoints', monitor='val_auc', \n",
    "                              mode='max', save_weights_only=True)# , save_best_only=True\n",
    "early_stopping = EarlyStopping(monitor='val_auc', min_delta=0.0001, patience=5)\n",
    "def scheduler(epoch):\n",
    "    thred = 10\n",
    "    if epoch < thred:\n",
    "        return 0.001\n",
    "    else:\n",
    "        return 0.001 * tf.math.exp(0.1 * (thred - epoch))\n",
    "lr_schedule = LearningRateScheduler(scheduler)\n",
    "callbacks = [checkpoints, early_stopping, lr_schedule]\n",
    "\n",
    "\n",
    "model.fit([train_dense_x_all, train_sparse_x_all], train_label_all, batch_size=256,\n",
    "         validation_data=([val_dense_x_all, val_sparse_x_all], val_label_all),\n",
    "         callbacks=callbacks, epochs=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 附：继承 Model 的模型构建方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DeepFM(tf.keras.Model):\n",
    "    def __init__(self, dense_feats, sparse_feats, vocab_sizes, embed_dims=8):\n",
    "        super().__init__()\n",
    "        self.dense_feats = dense_feats\n",
    "        self.sparse_feats = sparse_feats\n",
    "        self.vocab_sizes = vocab_sizes\n",
    "        self.embed_dims = embed_dims\n",
    "        \n",
    "        self.Linear = Linear(sparse_feats, vocab_sizes)\n",
    "        self.SparseEmbedding = SparseEmbedding(sparse_feats, vocab_sizes, embed_dims=8)\n",
    "        self.FM = FM()\n",
    "        self.DNN = DNN()\n",
    "        \n",
    "    \n",
    "    def call(self, inputs, training=True):\n",
    "        dense_inputs, sparse_inputs = inputs[0], inputs[1]\n",
    "        \n",
    "        linear_output = self.Linear(inputs)\n",
    "        concat_sparse_embeds = self.SparseEmbedding(sparse_inputs)\n",
    "        snd_order_sparse_output = self.FM(concat_sparse_embeds)\n",
    "        fc_layer_output = self.DNN(concat_sparse_embeds)\n",
    "        \n",
    "        # 输出部分\n",
    "        output = Add()([linear_output, snd_order_sparse_output, fc_layer_output])\n",
    "        output = Activation('sigmoid')(output)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer deep_fm_12 is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n",
      "train_loss 2.4098854 val_loss 2.483604\n",
      "train_loss 1.3211236 val_loss 1.3194331\n",
      "train_loss 1.169803 val_loss 1.0407462\n",
      "train_loss 0.99219334 val_loss 1.059052\n",
      "train_loss 0.8957213 val_loss 0.92486346\n",
      "train_loss 0.9512948 val_loss 0.9190863\n",
      "train_loss 0.95848316 val_loss 0.8585176\n",
      "train_loss 0.8645132 val_loss 0.83732563\n",
      "train_loss 0.7037978 val_loss 0.805529\n",
      "train_loss 0.67349327 val_loss 0.8104939\n",
      "train_loss 0.7041616 val_loss 0.7876595\n",
      "train_loss 0.6040318 val_loss 0.7762838\n",
      "train_loss 0.6661148 val_loss 0.7730615\n",
      "train_loss 0.62362516 val_loss 0.7610869\n",
      "train_loss 0.7019628 val_loss 0.7555212\n",
      "train_loss 0.66617835 val_loss 0.7523162\n",
      "train_loss 0.7438489 val_loss 0.7429388\n",
      "train_loss 0.65911984 val_loss 0.73776776\n",
      "train_loss 0.7577257 val_loss 0.72724354\n",
      "train_loss 0.5841069 val_loss 0.7345725\n",
      "train_loss 0.69067764 val_loss 0.72640127\n",
      "train_loss 0.70591426 val_loss 0.7106958\n",
      "train_loss 0.5872619 val_loss 0.7009569\n",
      "train_loss 0.68808687 val_loss 0.7069153\n",
      "train_loss 0.71475106 val_loss 0.69875795\n",
      "train_loss 0.66280043 val_loss 0.6927632\n",
      "train_loss 0.5748261 val_loss 0.70920014\n",
      "train_loss 0.51971066 val_loss 0.6790053\n",
      "train_loss 0.6533222 val_loss 0.6743224\n",
      "train_loss 0.5787353 val_loss 0.6695126\n",
      "train_loss 0.6206254 val_loss 0.6677863\n",
      "train_loss 0.6333037 val_loss 0.65689474\n",
      "train_loss 0.6053113 val_loss 0.65412337\n",
      "train_loss 0.5766033 val_loss 0.6488665\n",
      "train_loss 0.58827424 val_loss 0.6418533\n",
      "train_loss 0.5509889 val_loss 0.6426741\n",
      "train_loss 0.586426 val_loss 0.63931245\n",
      "train_loss 0.5573069 val_loss 0.6267758\n",
      "train_loss 0.5612141 val_loss 0.6449642\n",
      "train_loss 0.5308564 val_loss 0.62057\n",
      "train_loss 0.47453913 val_loss 0.6234291\n",
      "train_loss 0.53106457 val_loss 0.6179022\n",
      "train_loss 0.5241479 val_loss 0.6151297\n",
      "train_loss 0.6813842 val_loss 0.6094231\n",
      "train_loss 0.6254232 val_loss 0.608384\n",
      "train_loss 0.5442903 val_loss 0.6020965\n",
      "train_loss 0.48889655 val_loss 0.59901583\n",
      "train_loss 0.5180689 val_loss 0.59522665\n",
      "train_loss 0.57622343 val_loss 0.59929794\n",
      "train_loss 0.5545541 val_loss 0.5887621\n",
      "train_loss 0.55703753 val_loss 0.5847919\n",
      "train_loss 0.5962651 val_loss 0.5867704\n",
      "train_loss 0.6141354 val_loss 0.5895961\n",
      "train_loss 0.44274747 val_loss 0.57974124\n",
      "train_loss 0.53316665 val_loss 0.5761268\n",
      "train_loss 0.498142 val_loss 0.57354474\n",
      "train_loss 0.51469684 val_loss 0.5783324\n",
      "train_loss 0.6005205 val_loss 0.5681326\n",
      "train_loss 0.5296999 val_loss 0.5666884\n",
      "train_loss 0.47100228 val_loss 0.5630498\n",
      "train_loss 0.54450154 val_loss 0.5787666\n",
      "train_loss 0.5576676 val_loss 0.55907017\n",
      "train_loss 0.53484154 val_loss 0.5579099\n",
      "train_loss 0.5364264 val_loss 0.5627258\n",
      "train_loss 0.51488286 val_loss 0.55314064\n",
      "train_loss 0.55697054 val_loss 0.5507671\n",
      "train_loss 0.57481194 val_loss 0.550431\n",
      "train_loss 0.55537355 val_loss 0.5489825\n",
      "train_loss 0.56959105 val_loss 0.5468589\n",
      "train_loss 0.5849153 val_loss 0.5559658\n",
      "train_loss 0.6252235 val_loss 0.5475525\n",
      "train_loss 0.56599003 val_loss 0.5416614\n",
      "train_loss 0.45060825 val_loss 0.5402306\n",
      "train_loss 0.4618894 val_loss 0.5392666\n",
      "train_loss 0.5865662 val_loss 0.53793675\n",
      "train_loss 0.52943933 val_loss 0.5399228\n",
      "train_loss 0.46397635 val_loss 0.53526783\n",
      "train_loss 0.5720602 val_loss 0.53363514\n",
      "train_loss 0.53340673 val_loss 0.53278434\n",
      "train_loss 0.5761976 val_loss 0.5318284\n",
      "train_loss 0.5111011 val_loss 0.5346499\n",
      "train_loss 0.5211141 val_loss 0.53015906\n",
      "train_loss 0.4850332 val_loss 0.5295973\n",
      "train_loss 0.5528135 val_loss 0.5284586\n",
      "train_loss 0.5071026 val_loss 0.52804345\n",
      "train_loss 0.5900887 val_loss 0.5259946\n",
      "train_loss 0.46475852 val_loss 0.5291728\n",
      "train_loss 0.51071024 val_loss 0.53127724\n",
      "train_loss 0.5197077 val_loss 0.52651155\n",
      "train_loss 0.52425003 val_loss 0.5265078\n",
      "train_loss 0.5947874 val_loss 0.52251405\n",
      "train_loss 0.6377196 val_loss 0.52052724\n",
      "train_loss 0.481161 val_loss 0.51937187\n",
      "train_loss 0.52613926 val_loss 0.5188099\n",
      "train_loss 0.54683673 val_loss 0.52657354\n",
      "train_loss 0.4866224 val_loss 0.5167897\n",
      "train_loss 0.49941382 val_loss 0.5169294\n",
      "train_loss 0.5138988 val_loss 0.51508343\n",
      "train_loss 0.5252162 val_loss 0.5134446\n",
      "train_loss 0.54643357 val_loss 0.5159939\n",
      "train_loss 0.50454354 val_loss 0.51795447\n",
      "train_loss 0.5477977 val_loss 0.51163554\n",
      "train_loss 0.48235464 val_loss 0.51226854\n",
      "train_loss 0.46278268 val_loss 0.5106911\n",
      "train_loss 0.55143964 val_loss 0.5130235\n",
      "train_loss 0.49793103 val_loss 0.5086262\n",
      "train_loss 0.55516046 val_loss 0.5116377\n",
      "train_loss 0.526183 val_loss 0.50830406\n",
      "train_loss 0.4525234 val_loss 0.5073179\n",
      "train_loss 0.48127568 val_loss 0.50913733\n",
      "train_loss 0.5003133 val_loss 0.5108921\n",
      "train_loss 0.5246837 val_loss 0.5055629\n",
      "train_loss 0.5484116 val_loss 0.50500053\n",
      "train_loss 0.5513848 val_loss 0.50540334\n",
      "train_loss 0.5670711 val_loss 0.5034824\n",
      "train_loss 0.53560483 val_loss 0.50356776\n",
      "train_loss 0.4458433 val_loss 0.50354636\n",
      "train_loss 0.5142056 val_loss 0.5023199\n",
      "train_loss 0.5943471 val_loss 0.5016766\n",
      "train_loss 0.51107144 val_loss 0.5018072\n",
      "train_loss 0.4835248 val_loss 0.506819\n",
      "train_loss 0.45860666 val_loss 0.50842124\n",
      "train_loss 0.47358721 val_loss 0.49983543\n",
      "train_loss 0.49508673 val_loss 0.5003009\n",
      "train_loss 0.46116623 val_loss 0.49922407\n",
      "train_loss 0.46439755 val_loss 0.49882585\n",
      "train_loss 0.55467176 val_loss 0.49828482\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-53-8f0fdeaec3c8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGradientTape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtape\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m         \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_dense_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_sparse_x\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlosses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbinary_crossentropy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_label\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreduce_mean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    966\u001b[0m           with base_layer_utils.autocast_context_manager(\n\u001b[1;32m    967\u001b[0m               self._compute_dtype):\n\u001b[0;32m--> 968\u001b[0;31m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcast_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    969\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle_activity_regularization\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    970\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_mask_metadata\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_masks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-7-0ecc6b3cc7fa>\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, inputs, training)\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0mlinear_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mconcat_sparse_embeds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSparseEmbedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msparse_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m         \u001b[0msnd_order_sparse_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFM\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconcat_sparse_embeds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m         \u001b[0mfc_layer_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDNN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconcat_sparse_embeds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    966\u001b[0m           with base_layer_utils.autocast_context_manager(\n\u001b[1;32m    967\u001b[0m               self._compute_dtype):\n\u001b[0;32m--> 968\u001b[0;31m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcast_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    969\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle_activity_regularization\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    970\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_mask_metadata\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_masks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-5-743cce7a7281>\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, concat_sparse_embeds)\u001b[0m\n\u001b[1;32m      6\u001b[0m         \u001b[0;31m# 先求和再求平方\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0msum_embeds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreduce_sum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconcat_sparse_embeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m         \u001b[0msquare_sum_embeds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMultiply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msum_embeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msum_embeds\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m         \u001b[0;31m# 先平方再求和\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0msquare_embeds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMultiply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mconcat_sparse_embeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconcat_sparse_embeds\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    807\u001b[0m     \u001b[0;31m# mode when all inputs can be traced back to `keras.Input()` (when building\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    808\u001b[0m     \u001b[0;31m# models using the functional API).\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 809\u001b[0;31m     \u001b[0mbuild_graph\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mare_all_symbolic_tensors\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    810\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    811\u001b[0m     \u001b[0;31m# Accept NumPy and scalar inputs by converting to Tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/utils/tf_utils.py\u001b[0m in \u001b[0;36mare_all_symbolic_tensors\u001b[0;34m(tensors)\u001b[0m\n\u001b[1;32m    324\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    325\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mare_all_symbolic_tensors\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 326\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_symbolic_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtensor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    327\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/utils/tf_utils.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    324\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    325\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mare_all_symbolic_tensors\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 326\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_symbolic_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtensor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    327\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/utils/tf_utils.py\u001b[0m in \u001b[0;36mis_symbolic_tensor\u001b[0;34m(tensor)\u001b[0m\n\u001b[1;32m    352\u001b[0m     return (getattr(tensor, '_keras_history', False) or\n\u001b[1;32m    353\u001b[0m             not context.executing_eagerly())\n\u001b[0;32m--> 354\u001b[0;31m   \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcomposite_tensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCompositeTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    355\u001b[0m     \u001b[0mcomponent_tensors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexpand_composites\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'graph'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcomponent_tensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/abc.py\u001b[0m in \u001b[0;36m__instancecheck__\u001b[0;34m(cls, instance)\u001b[0m\n\u001b[1;32m    137\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0m__instancecheck__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minstance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    138\u001b[0m             \u001b[0;34m\"\"\"Override for isinstance(instance, cls).\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0m_abc_instancecheck\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minstance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0m__subclasscheck__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubclass\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "train_data = total_data.loc[:500000-1]\n",
    "valid_data = total_data.loc[500000:]\n",
    "\n",
    "train_dense_x_all = np.array([train_data[[f]].values for f in dense_feats])\n",
    "train_sparse_x_all = np.array([train_data[[f]].values for f in sparse_feats])\n",
    "train_label_all = train_data[['label']].values\n",
    "\n",
    "val_dense_x_all = np.array([valid_data[[f]].values for f in dense_feats])\n",
    "val_sparse_x_all = np.array([valid_data[[f]].values for f in sparse_feats])\n",
    "val_label_all = valid_data[['label']].values\n",
    "\n",
    "\n",
    "model = DeepFM(dense_feats, sparse_feats, vocab_sizes)\n",
    "opt = tf.keras.optimizers.Adam(learning_rate=1e-3)\n",
    "batch_size = 256\n",
    "for i in range(total_data.shape[0]//batch_size):\n",
    "    train_dense_x = list(train_dense_x_all[:,i*batch_size:(i+1)*batch_size,:])\n",
    "    train_sparse_x = list(train_sparse_x_all[:,i*batch_size:(i+1)*batch_size,:])\n",
    "    train_label = train_label_all[i*batch_size:(i+1)*batch_size]\n",
    "    \n",
    "    with tf.GradientTape() as tape:\n",
    "        pred = model([train_dense_x, train_sparse_x])\n",
    "        loss = tf.keras.losses.binary_crossentropy(train_label, pred)\n",
    "        loss = tf.reduce_mean(loss)\n",
    "    grads = tape.gradient(loss, model.variables)\n",
    "    opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "    \n",
    "    \n",
    "    if i%10 ==0:\n",
    "        val_dense_x = list(val_dense_x_all)\n",
    "        val_sparse_x = list(val_sparse_x_all)\n",
    "        val_label = val_label_all\n",
    "\n",
    "        pred = model([val_dense_x, val_sparse_x])\n",
    "        val_loss = tf.keras.losses.binary_crossentropy(val_label, pred)\n",
    "        val_loss = tf.reduce_mean(val_loss)\n",
    "        print('train_loss', loss.numpy(), 'val_loss', val_loss.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}